diff --git a/ggml/.gitignore b/ggml/.gitignore index 47921f71e..ffabd67db 100644 --- a/ggml/.gitignore +++ b/ggml/.gitignore @@ -37,3 +37,8 @@ __pycache__/ # Model files ggml-model-f16.bin *.bat + +# AGENTS + +AGENTS.local.md +.pi/SYSTEM.md diff --git a/ggml/.pi/SYSTEM.md b/ggml/.pi/SYSTEM.md deleted file mode 120000 index b9d3a0581..000000000 --- a/ggml/.pi/SYSTEM.md +++ /dev/null @@ -1 +0,0 @@ -gg/SYSTEM.md \ No newline at end of file diff --git a/ggml/.pi/gg/APPEND_SYSTEM.md b/ggml/.pi/gg/APPEND_SYSTEM.md new file mode 100644 index 000000000..dd79897e3 --- /dev/null +++ b/ggml/.pi/gg/APPEND_SYSTEM.md @@ -0,0 +1,6 @@ + +Project-specific instructions: + +## Coding + +- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments diff --git a/ggml/.pi/gg/SYSTEM.md b/ggml/.pi/gg/SYSTEM.md deleted file mode 100644 index bd308ea96..000000000 --- a/ggml/.pi/gg/SYSTEM.md +++ /dev/null @@ -1,32 +0,0 @@ -You are a coding agent. Here are some very important rules that you must follow: - -General: -- Be very precise and concise when writing code, comments, explanations, etc. -- If an inline comment exceeds 2 lines, replace it with: `// note: TODO LATER` -- PR and commit titles format: ` : `. Lookup recents for examples -- Don't try to build or run the code unless you are explicitly asked to do so -- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources -- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user - -Coding: -- When in doubt, always refer to the CONTRIBUTING.md file of the project -- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments -- When referencing issues or PRs in comments, use the format: - - C/C++ code: `// ref: <url>` - - Other (CMake, etc.): `# ref: <url>` - -Pull requests (PRs): -- New branch names are prefixed with "gg/" -- Before opening a pull request, ask the user to confirm the description -- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line) -- When creating a pull request, look for the repository's PR template and follow it -- For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]" -- If `PI_MODEL_NAME` env var is not set, ask the user to tell you what model was used and write it in place of [MODEL] -- Always create the pull requests in draft mode -- Never reply to review comments or post comments on issues/PRs without explicit permission from the user - -Commits: -- On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag -- Do not explicitly set the git author in commits - rely on the default git config -- Always use `--no-gpg-sign` when committing -- Never `git push` without explicit confirmation from the user diff --git a/ggml/AUTHORS b/ggml/AUTHORS index d130ab3f5..b03f7e7f6 100644 --- a/ggml/AUTHORS +++ b/ggml/AUTHORS @@ -1,335 +1,957 @@ -# date: Tue Feb 4 13:03:51 EET 2025 +# date: Wed Nov 4 11:19:40 EET 2026 # this file is auto-generated by scripts/gen-authors.sh +旺旺碎冰冰 <38837039+Cyberhan123@users.noreply.github.com> +蕭澧邦 <45505768+shou692199@users.noreply.github.com> +布客飞龙 <562826179@qq.com> +도로로도로또 <60079918+dororodoroddo@users.noreply.github.com> +谢乃闻 <sienaiwun@users.noreply.github.com> +0 <1939455790@qq.com> 0cc4m <picard12@live.de> +0Marble <85058989+0Marble@users.noreply.github.com> 65a <10104049+65a@users.noreply.github.com> -AT <manyoso@users.noreply.github.com> +92es <76127081+92es@users.noreply.github.com> +a-huk <56552991+a-huk@users.noreply.github.com> +a3sh <38979186+A3shTnT@users.noreply.github.com> +Aadeshveer Singh <24b0926@iitb.ac.in> +Aadeshveer Singh <aadeshveer07@gmail.com> +Aaron Teo <57927438+taronaeo@users.noreply.github.com> +Aaron Teo <aaron.teo1@ibm.com> +abhijain1204fujitsu <139222713+abhijain1204fujitsu@users.noreply.github.com> +Abhijit Ramesh <abhijitramesh2k@gmail.com> Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> +Abir Deol <abir.deol2006@gmail.com> +abotsis <github@bots.is> +Acly <aclysia@gmail.com> Adam Tazi <52357206+ad1tazi@users.noreply.github.com> +adavyas <121313528+adavyas@users.noreply.github.com> +adgup-qti <adgup@qti.qualcomm.com> +Adrian Kretz <me@akretz.com> Adrien Gallouët <adrien@gallouet.fr> Adrien Gallouët <angt@huggingface.co> +AesSedai <7980540+AesSedai@users.noreply.github.com> +ag2s20150909 <19373730+ag2s20150909@users.noreply.github.com> +AgoraPete <peter.haughie@agora-thinktanks.org> +agray3 <agray3@users.noreply.github.com> Ahmad Tameem <113388789+Tameem-10xE@users.noreply.github.com> +ai-fonsi <length-amiss-7k@icloud.com> AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> AidanBeltonS <aidan.belton@codeplay.com> Akarshan Biswas <akarshan.biswas@gmail.com> +Akarshan Biswas <akarshan@menlo.ai> Akarshan Biswas <akarshanbiswas@fedoraproject.org> +akleine <alb.kleine@gmx.de> +Alan Gray <agray3@users.noreply.github.com> +Alan Tseng <alan.tseng.cs@gmail.com> +Alan Tseng <alanhc.tseng1999@gmail.com> Albert Jin <albert.jin@gmail.com> +Alberto Cabrera Pérez <1478977+Alcpz@users.noreply.github.com> Alberto Cabrera Pérez <alberto.cabrera@codeplay.com> Alberto Cabrera Pérez <alberto.cabrera@intel.com> +Alberto Cabrera Pérez <alberto.cabrera@liquid.ai> +Aleksei Nikiforov <103434461+AlekseiNikiforovIBM@users.noreply.github.com> Alex Azarov <alex@azarov.by> +Alex Morgan <36094800+amitmi704@users.noreply.github.com> Alex O'Connell <35843486+acon96@users.noreply.github.com> Alex von Gluck IV <kallisti5@unixzen.com> +alex-spacemit <jinghui.huang@spacemit.com> +Alexander Heisler <126129661+heislera763@users.noreply.github.com> +Alexey Kopytko <alexey@kopytko.com> +Alfred <zxu3@clemson.edu> +Amadeus dyw <74057036+SG-Amadeus@users.noreply.github.com> +Aman Chadha(IVIXMMI) <79802170+ac-mmi@users.noreply.github.com> +Aman Gupta <amangupta052@gmail.com> +Aman Karki <amankarki151@gmail.com> +Aman Karki <itsamankarki@gmail.com> AmbientL <107641468+AmbientL@users.noreply.github.com> +amd-dwang <dong.wang@amd.com> AmirAli Mirian <37371367+amiralimi@users.noreply.github.com> +amritahs-ibm <amritahs@linux.vnet.ibm.com> +An Long <aisk@users.noreply.github.com> +AN Long <aisk@users.noreply.github.com> +Anand Patil <126432639+AnandPatil1@users.noreply.github.com> +Anant Shrivastava <anant@anantshri.info> Ananta Bastola <anantarajbastola@gmail.com> +Anav Prasad <anavp@nvidia.com> +anavp-nvidia <anavp@nvidia.com> +Andrea Rossetti <rossettiandrea@proton.me> Andreas (Andi) Kunar <andreask@msn.com> Andreas Kieslinger <47689530+aendk@users.noreply.github.com> Andrei <abetlen@gmail.com> +Andrew Lee <andysl@gmail.com> Andrew Minh Nguyen <40281306+amqdn@users.noreply.github.com> +Andrew Smith <atsmith19@comcast.net> Andrii Ryzhkov <andriiryzhkov@users.noreply.github.com> +andyluo7 <43718156+andyluo7@users.noreply.github.com> +Animesh <animeshsri.official@gmail.com> +Anjielon <angelo.mogavero@gmail.com> +Ankit Khandelwal <ankk98@gmail.com> +Antoine Viallon <antoine@lesviallon.fr> +Anton Mitkov <anton_b_mitkov@abv.bg> +Anton Mitkov <anton.mitkov@codeplay.com> +anubhavfujitsu <Anubhav.Srivastava@fujitsu.com> +anujj <ajalota@nvidia.com> +Aparna M P <aparmp@qti.qualcomm.com> +apcameron <37645737+apcameron@users.noreply.github.com> +appvoid <78444142+appvoid@users.noreply.github.com> +Arianna Method <theariannamethod@gmail.com> +ariez-xyz <41232910+ariez-xyz@users.noreply.github.com> Arjun <ccldarjun@icloud.com> +Arseniy Obolenskiy <good.doog.2012@gmail.com> +asf0 <scorpionspfc@gmail.com> Ashraful Islam <ashraful.meche@gmail.com> Astariul <43774355+astariul@users.noreply.github.com> AsukaMinato <asukaminato@nyan.eu.org> +AT <manyoso@users.noreply.github.com> +Atharva Dubey <atharva.dubey@codeplay.com> +Atomic-Germ <97569476+Atomic-Germ@users.noreply.github.com> +aubreyli <aubreylee@gmail.com> +AuroraRAS <chplee@gmail.com> +automaticcat <daogiatuank54@gmail.com> Avi Lumelsky <avilume@gmail.com> +bagheera <59658056+bghira@users.noreply.github.com> +Bálint Bokros <bbalint96@gmail.com> +bandoti <141645996+bandoti@users.noreply.github.com> Bart Pelle <3662930+Velocity-@users.noreply.github.com> +Bartowski <3266127+bartowski1182@users.noreply.github.com> +BB-fat <45072480+BB-fat@users.noreply.github.com> +Beinsezii <39478211+Beinsezii@users.noreply.github.com> Ben Ashbaugh <ben.ashbaugh@intel.com> +Bernard Ladenthin <bernard.ladenthin@gmail.com> Bernhard M. Wiedemann <githubbmwprimary@lsmod.de> +Bertay Eren <39909689+bertaye@users.noreply.github.com> +Bhargav Krish <bkrishnamurt@nvidia.com> +Bhavik Sharda <10757940+BLSharda@users.noreply.github.com> +Bill Sideris <bill88t@feline.gr> +Bizhao Shi <37729561+shibizhao@users.noreply.github.com> +Björn Ganster <mail@bjoern-ganster.de> +bmwl <brian.marshall@tolko.com> +bobqianic <129547291+bobqianic@users.noreply.github.com> +Bodhi <3882561+BodhiHu@users.noreply.github.com> +Boian Berberov <7432115+bberberov@users.noreply.github.com> Borislav Stanimirov <b.stanimirov@abv.bg> +bosh <98094229+boshjerns@users.noreply.github.com> +Bowen Han <fancycode@gmail.com> Brad Ito <phlogisticfugu@users.noreply.github.com> Brad Murray <59848399+bradmurray-dt@users.noreply.github.com> +Brad Smith <1472326+infinitewarp@users.noreply.github.com> +bri-prism <288398250+bri-prism@users.noreply.github.com> Brian <mofosyne@gmail.com> Bryan Lozano <b.lozano.havoc@gmail.com> +bssrdf <merlintiger@hotmail.com> +budiony <budiony@gmail.com> +Buğra Özgürsoy <13810383+ozgursoy@users.noreply.github.com> +Caladrel <jnf333@gmail.com> +Captain-Tripps <jstaples2@hotmail.com> Carolinabanana <140120812+Carolinabanana@users.noreply.github.com> CarterLi999 <664681047@qq.com> +ccbinn <ccbinn@163.com> Cebtenzzre <cebtenzzre@gmail.com> Changyeon Kim <cyzero.kim@samsung.com> +Chanho Lee <ibear6954@gmail.com> +Charles Duffy <charles@dyfis.net> Charles Xu <63788048+chaxu01@users.noreply.github.com> Charles Xu <charles.xu@arm.com> +chen fan <350211548@qq.com> Chen Xi <xi2.chen@intel.com> Chen Xi <xixichen08@foxmail.com> +Chen Yuan <constant.chen@uwaterloo.ca> +Chen Yuan <constantchen525@gmail.com> +chengchi <davesjoewang@gmail.com> +Chenguang Li <757486878@qq.com> Chenguang Li <87689256+noemotiovon@users.noreply.github.com> Chris Elrod <elrodc@gmail.com> +Chris Lee <clee@mg8.org> +Chris Rohlf <chris.rohlf@gmail.com> Christian Kastner <ckk@kvr.at> +Christoph Reiter <reiter.christoph@gmail.com> +Chyan <163109379+chyan8@users.noreply.github.com> +Clemens Wasser <clemens.wasser@gmail.com> Clint Herron <hanclinto@gmail.com> +cmdr2 <secondary.cmdr2@gmail.com> +cmdr2 <shashank.shekhar.global@gmail.com> +codemonkey <441345965@qq.com> +compilade <113953597+compilade@users.noreply.github.com> +compilade <git@compilade.net> +Congcong Cai <congcongcai0907@163.com> Conrad Kramer <conrad@conradkramer.com> +Copilot <198982749+Copilot@users.noreply.github.com> Cordeiro <1471463+ocordeiro@users.noreply.github.com> +Cordell Blanchard <55163549+CordellBlanchard@users.noreply.github.com> +cphlipot <9103367+cphlipot@users.noreply.github.com> +cqderek <cqderek@gmail.com> +cqderek <cqiang@qti.qualcomm.com> +CrispStrobe <154636388+CrispStrobe@users.noreply.github.com> Cristiano Calcagno <cristianoc@users.noreply.github.com> +cwriter <silvan.niederer@bluewin.ch> DAN™ <dranger003@gmail.com> Dan Forbes <dan@danforbes.dev> +Dan Hoffman <43101339+thedanhoffman@users.noreply.github.com> Dan Johansson <164997844+eddnjjn@users.noreply.github.com> Dan Johansson <dan.johansson@arm.com> Daniel Bevenius <daniel.bevenius@gmail.com> +Daniel Kuts <kutz@ispras.ru> +Daniel Tang <danielzgtg.opensource@gmail.com> +Daniel Varga <vargad88@gmail.com> Daniel Ziegenberg <daniel@ziegenberg.at> Daniele <57776841+daniandtheweb@users.noreply.github.com> +Daniele <daniele.dilotorres@gmail.com> +Danny Milosavljevic <dannym@friendly-machines.com> +Dante <144588016+KafKafrnZ@users.noreply.github.com> Daulet Zhanguzin <daulet@users.noreply.github.com> Dave <dave-fl@users.noreply.github.com> Dave Airlie <airlied@gmail.com> Dave Airlie <airlied@redhat.com> +David Friehs <david@friehs.info> +David Huang <1969802+hjc4869@users.noreply.github.com> +David M. Rogers <predictivestatmech@gmail.com> David Miller <david@patagona.ca> +David Spruill <62445444+Spruill-1@users.noreply.github.com> +David Zhao <90013954+Your-Cheese@users.noreply.github.com> +David366AI <86212041+David366AI@users.noreply.github.com> DavidKorczynski <david@adalogics.com> Davidson Francis <davidsondfgl@gmail.com> +Daya Adianto <addianto@users.noreply.github.com> +ddpasa <112642920+ddpasa@users.noreply.github.com> +deepsek <166548550+deepsek@users.noreply.github.com> +denersc <denerstassun@gmail.com> +Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> +Devedse <2350015+devedse@users.noreply.github.com> +Developer-Ecosystem-Engineering <65677710+Developer-Ecosystem-Engineering@users.noreply.github.com> +diannao <55k@outlook.com> Dibakar Gope <dibakar.gope@arm.com> Didzis Gosko <didzis@users.noreply.github.com> Diego Devesa <slarengh@gmail.com> Diogo <dgcruz983@gmail.com> +distlibs <30928251+distlibs@users.noreply.github.com> Djip007 <3705339+Djip007@users.noreply.github.com> Djip007 <djip.perois@free.fr> +Dmitry Atamanov <data-man@users.noreply.github.com> +Dmytro Minochkin <dmytro.minochkin@gmail.com> +Doctor Shotgun <126566557+DocShotgun@users.noreply.github.com> +Dominik Pantaleoni <95251853+dpantaleoni@users.noreply.github.com> +Dong Won Kim <63934649+ddwkim@users.noreply.github.com> +Donghyeon Jeong <54725479+djeong20@users.noreply.github.com> +Dongsheng Luo <luods1995@outlook.com> Dou Xinpeng <15529241576@163.com> Dou Xinpeng <81913537+Dou-Git@users.noreply.github.com> Dr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com> +drluoto <155452829+drluoto@users.noreply.github.com> +dscripka <dscripka@users.noreply.github.com> +dskwe <dskwelmcy@163.com> +dsproule <dsproule@qti.qualcomm.com> +duduta <simona.gherman@gmail.com> Ebey Abraham <ebey97@gmail.com> +Ed Addario <29247825+EAddario@users.noreply.github.com> +Eden <aa9736195201@gmail.com> +Ehsan Bateni <ebateni@qti.qualcomm.com> Eldar Yusupov <eyusupov@gmail.com> +Emil Askerov <56842174+EmilAskerov@users.noreply.github.com> Emmanuel Durand <emmanueldurand@protonmail.com> Engininja2 <139037756+Engininja2@users.noreply.github.com> +Eric A Stalee <87948564+Eric-A-Stalee@users.noreply.github.com> Eric Zhang <34133756+EZForever@users.noreply.github.com> Erik Scholz <Green-Sky@users.noreply.github.com> +Erik Winter <93544977+ewintr@users.noreply.github.com> +Ervin Áron Tasnádi <etasnadi@protonmail.com> +Ethan Guo <ethanguo.dev@gmail.com> Ettore Di Giacinto <mudler@users.noreply.github.com> +EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com> +Eurekatic <eurekatic@eurekatic.eu> +Evan Huus <eapache@gmail.com> Eve <139727413+netrunnereve@users.noreply.github.com> +Ewan Crawford <ewan.cr@gmail.com> +Ewan Crawford <ewan@codeplay.com> F1L1P <78918286+F1L1Pv2@users.noreply.github.com> +fairydreaming <166155368+fairydreaming@users.noreply.github.com> Faisal Zaghloul <quic_fzaghlou@quicinc.com> +FamousMonsterr <famousmonster@ya.ru> FantasyGmm <16450052+FantasyGmm@users.noreply.github.com> +fanyang <fanyang89@outlook.com> Felix <stenbackfelix@gmail.com> +Felix Ye <felixyjs@gmail.com> +feni6 <fenissova@proton.me> Finn Voorhees <finnvoorhees@gmail.com> FirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com> +fitzsim <fitzsim@fitzsim.org> +fj-y-saito <85871716+fj-y-saito@users.noreply.github.com> +fl0rianr <226492742+fl0rianr@users.noreply.github.com> +fl0rianr <f.reinle@otec.de> +Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com> +forforever73 <63285796+forforever73@users.noreply.github.com> +Francois Dugast <francois.dugast@intel.com> +François-Xavier Gsell <fxgsell@gmail.com> Frankie Robertson <frankier@users.noreply.github.com> +fraxy-v <65565042+fraxy-v@users.noreply.github.com> +Frosty40 <newjordan@gmail.com> +fxzjshm <11426482+fxzjshm@users.noreply.github.com> +Gabe Goodhart <ghart@us.ibm.com> GainLee <perfecter.gen@gmail.com> +gatbontonpc <gatbontonpc@gmail.com> +Gaurav Garg <52341457+gaugarg-nv@users.noreply.github.com> +Gaurav Garg <gaugarg@nvidia.com> +George <35490284+noctrex@users.noreply.github.com> George Hindle <george@georgehindle.com> Georgi Gerganov <ggerganov@gmail.com> +Geramy Loveless <gloveless@jqluv.com> +Gezahegne <gezahegne.yirefu@gmail.com> +Gian-Carlo Pascutto <gcp@sjeng.org> +gianni-cor <gianfrancocordella@gmail.com> +GiantPrince <90118823+GiantPrince@users.noreply.github.com> Gilad S <7817232+giladgd@users.noreply.github.com> Gilad S <giladgd@users.noreply.github.com> Gilad S. <7817232+giladgd@users.noreply.github.com> +GittyBurstein <g0534163997@gmail.com> +Giuseppe Scrivano <gscrivan@redhat.com> +gn64 <yukikaze.jp@gmail.com> +goerch <jhr.walter@t-online.de> +goldwaving <77494627+goldwaving@users.noreply.github.com> +Gregor Jasny <gjasny@googlemail.com> +Guanhuai Zhang <67999475+BiReRa@users.noreply.github.com> +Guantao Wang <103912927+Grantley-W@users.noreply.github.com> Guillaume Wenzek <gwenzek@users.noreply.github.com> +Guus Waals <_@guusw.nl> +guyfischman <138163913+guyfischman@users.noreply.github.com> Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com> +Hale Chan <halechan@qq.com> +Hao-Chen2337 <2113996104@qq.com> +HaoJun ZHANG <neroued@gmail.com> +haopeng <657407891@qq.com> +Harkirat Gill <harkirat.gill@amd.com> +HarrisonSec <gzxharrison@gmail.com> Haus1 <haus.xda@gmail.com> +hebangwen <32662175+hebangwen@users.noreply.github.com> +Henry Linjamäki <henry.linjamaki@gmail.com> +Henry Linjamäki <henry.mikael.linjamaki@intel.com> +Herman Semenoff <GermanAizek@yandex.ru> Herman Semenov <GermanAizek@yandex.ru> +hesiru <siruhe666@gmail.com> +hey-gm <heitorgm@outlook.com> +hidenorly <hidenorly@users.noreply.github.com> HimariO <dsfhe49854@gmail.com> +hipudding <huafengchun@gmail.com> Hirochika Matsumoto <git@hkmatsumoto.com> +Hitesh Chopra <34310832+hiteshchopra11@users.noreply.github.com> +hmirin <hmirin@users.noreply.github.com> +hmscider <201289679+hmscider@users.noreply.github.com> +hokanosekai <69720899+hokanosekai@users.noreply.github.com> +homeofe <154892989+homeofe@users.noreply.github.com> Hong Bo PENG <penghb@cn.ibm.com> +Hongqiang Wang <66336067+wanghqc@users.noreply.github.com> +Hongqiang Wang <wangh@qti.qualcomm.com> +hrushitfujitsu <Hrushit.Kakadia@fujitsu.com> Hugo Rosenkranz-Costa <hugo.rosenkranz@gmail.com> +HumerousGorgon <31957201+HumerousGorgon@users.noreply.github.com> +hydai <z54981220@gmail.com> Hyunsung Lee <ita9naiwa@gmail.com> -IGUILIZ Salah-Eddine <76955987+salahiguiliz@users.noreply.github.com> +I3eg1nner <45710049+I3eg1nner@users.noreply.github.com> +iacopPBK <iacopogiottorossi@gmail.com> Ian Bull <irbull@eclipsesource.com> +Ian Faust <icfaust@gmail.com> +Ian McKellar <ian@mckellar.org> +Icenowy Zheng <uwu@icenowy.me> +Igor Matuszewski <Xanewok@gmail.com> +IGUILIZ Salah-Eddine <76955987+salahiguiliz@users.noreply.github.com> +Ihar Hrachyshka <ihar.hrachyshka@gmail.com> Ihar Hrachyshka <ihrachys@redhat.com> +ihb2032 <40718643+ihb2032@users.noreply.github.com> Ikko Eltociear Ashimine <eltociear@gmail.com> +Ilia Ilmer <iliailmer@users.noreply.github.com> +InflexCZE <martin.pavlicek11@gmail.com> +Intel AI Get-to Market Customer Success and Solutions <ai.gtm.css@gmail.com> +iron <lizhenneng@gmail.com> +Isaac <34376531+init-22@users.noreply.github.com> +Isaac McFadyen <isaac@imcf.me> +Ismail <115064057+AlrIsmail@users.noreply.github.com> +issixx <46835150+issixx@users.noreply.github.com> +itterative <190138728+itterative@users.noreply.github.com> Ivan <nekotekina@gmail.com> Ivan Filipov <159561759+vanaka11@users.noreply.github.com> Ivan Stepanov <ivanstepanovftw@gmail.com> Ivan Zdane <accounts@ivanzdane.com> +ixgbe <1113177880@qq.com> Jack Mousseau <jmousseau@users.noreply.github.com> Jack Vial <vialjack@gmail.com> JacobLinCool <jacoblincool@gmail.com> +Jaden_Mach <88880593+jadenmach2@users.noreply.github.com> +jaeminSon <woalsdnd@gmail.com> +Jake Karnes <jake.karnes@gmail.com> Jakob Frick <jakob.maria.frick@gmail.com> +JamePeng <jame_peng@sina.com> +James Francis <6763899+JamesFranc@users.noreply.github.com> +Jan Ekström <jeebjp@gmail.com> Jan Ploski <jpl@plosquare.com> +Jared Tweed <jaredtwe@gmail.com> Jared Van Bortel <jared@nomic.ai> +Jasmine-tim <403568753@qq.com> +Jason C.H <ctrysbita@outlook.com> +Jason Ni <jason.ni.py@gmail.com> +Jassieluo <130133492+Jassieluo@users.noreply.github.com> +Jay <BusyJay@users.noreply.github.com> +Jay Zenith <162098309+JayZenith@users.noreply.github.com> +Jayant Lohia <rajiblohia@gmail.com> +jbooth <jaybooth@gmail.com> +jdomke <28772296+jdomke@users.noreply.github.com> Jeff Bolz <jbolz@nvidia.com> +Jeffrey Morgan <jmorganca@gmail.com> Jeffrey Quesnelle <jquesnelle@gmail.com> +Jeremy Demeule <jdemeule@users.noreply.github.com> +Jeremy Rand <244188+JeremyRand@users.noreply.github.com> Jeroen Mostert <jeroen.mostert@cm.com> +Jess Sullivan <Jess@sulliwood.org> +Jesse Gross <jesse@kernel.org> +Jesse Symons <155509533+Torchit1@users.noreply.github.com> +Jetson Tan <tanzongyouyi@outlook.com> +Jhen-Jie Hong <iainst0409@gmail.com> +Jiacheng (Jason) Chen <76919340+jiachengjason@users.noreply.github.com> Jiahao Li <liplus17@163.com> +Jiang, Fish <fish.jiang@intel.com> JidongZhang-THU <1119708529@qq.com> +Jie Fu (傅杰) <jiefu@tencent.com> +jiez <373447296@qq.com> +Jim Wu <jimw567@users.noreply.github.com> +Jingxin (Philip) Li <philipaslee@gmail.com> +jingzhou <jinzho@qti.qualcomm.com> +Jinyang He <hejinyang@loongson.cn> +jinzihao <jinzihao1996@gmail.com> Jiří Podivín <66251151+jpodivin@users.noreply.github.com> +Jiwoong Song <55657581+jwsong98@users.noreply.github.com> +JJJYmmm <92386084+JJJYmmm@users.noreply.github.com> +jklincn <985765408@qq.com> Jo Liss <joliss42@gmail.com> Joe Todd <joe.todd@codeplay.com> +Joel <agnel.joel@gmail.com> Johannes Gäßler <johannesg@5d6.de> John Balis <phobossystems@gmail.com> +johnson442 <56517414+johnson442@users.noreply.github.com> +Jonathan Clohessy <jonathan.clohessy@arm.com> +Jonathan Graehl <99024+graehl@users.noreply.github.com> Josh Bleecher Snyder <josharian@gmail.com> +Judd <4046440+foldl@users.noreply.github.com> Judd <foldl@users.noreply.github.com> +Julian Pscheid <julian@pscheid.com> +Julius Tischbein <ju.tischbein@gmail.com> Jun Hee Yoo <contact.jhyoo@gmail.com> +junchao-loongson <68935141+junchao-loongson@users.noreply.github.com> +junchao-zhao <68935141+junchao-loongson@users.noreply.github.com> Junil Kim <logyourself@gmail.com> +JusteLeo <leonard.adamo66@gmail.com> +Justin Bradford <jabradford@gmail.com> +Justin Santa Barbara <justinsb@google.com> Justina Cho <justcho5@gmail.com> Justine Tunney <jtunney@gmail.com> Justine Tunney <jtunney@mozilla.com> +k.h.lai <adrian.k.h.lai@outlook.com> +k4ss4n <128936199+k4ss4n@users.noreply.github.com> +Kai Pastor <dg0yt@darc.de> +Kakaru <97896816+KakaruHayate@users.noreply.github.com> +Kamalesh VS <76260512+kkjjkamal123@users.noreply.github.com> +karavayev <192749314+karavayev@users.noreply.github.com> Karol Kontny <82021046+kkontny@users.noreply.github.com> +Kartik Gulia <kgulia@nvidia.com> +Kartik Sirohi <99896785+sirohikartik@users.noreply.github.com> +Katara <sohaib.harraoui@ostorlab.dev> +Katostrofik <georgiopapairo@gmail.com> +katsu560 <118887472+katsu560@users.noreply.github.com> Kawrakow <48489457+ikawrakow@users.noreply.github.com> +kbenkhaled <khalilbenkhaled01@gmail.com> Kevin Gibbons <bakkot@gmail.com> +Kevin Hannon <kehannon@redhat.com> +Kevin Hopper <93635715+kh0pper@users.noreply.github.com> +Kevin Liu <4396kevinliu@gmail.com> +Kevin Pouget <kpouget@redhat.com> +Kilian Hu <90606809+kilian-hu@users.noreply.github.com> +kimminsu <80271594+kimminsu38oo@users.noreply.github.com> +Kirill A. Korinsky <kirill@korins.ky> +klosax <131523366+klosax@users.noreply.github.com> +kononnable <kononnable@gmail.com> +Konrad Moren <kmoren@nvidia.com> Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com> +Kris <github@selfref.dev> +Krishna Sridhar <99914379+srikris-sridhar@users.noreply.github.com> +kumaal <44551860+kumaal@users.noreply.github.com> +kunnis <kunnis@users.noreply.github.com> +kurquhar <kurquhar@qti.qualcomm.com> +Kusha Gharahi <3326002+kushagharahi@users.noreply.github.com> +KyleHagy <59183061+KyleHagy@users.noreply.github.com> Kylin <56434533+KyL0N@users.noreply.github.com> +l3utterfly <gc.pthzfoldr@gmail.com> +l8bloom <l8bloomapi@gmail.com> +las7 <98077186+las7@users.noreply.github.com> +Laurent Zuijdwijk <laurent.zuijdwijk@gmail.com> +Law Po Ying <30721578+yingying0906@users.noreply.github.com> +le.chang <cljs118@126.com> +Leebr Data Consulting <harrak.amine1987@gmail.com> +leejet <31925346+leejet@users.noreply.github.com> +leejet <leejet714@gmail.com> +leo-pony <nengjunma@outlook.com> +Leon <118652405+LeON-Nie-code@users.noreply.github.com> +Leonard Mosescu <tlemo@users.noreply.github.com> +leonardHONG <2695316095@qq.com> +lhez <lih@qti.qualcomm.com> +lhez <quic_lih@quicinc.com> +liangliangchang <lichang@amd.com> +liminfei-amd <91481003+liminfei-amd@users.noreply.github.com> +lingyezhixing <144504450+lingyezhixing@users.noreply.github.com> +Linux User <linuxusergd@outlook.com> +liuwei-git <14815172+liuwei-git@users.noreply.github.com> +lixing-star <104126818+lixing-star@users.noreply.github.com> +lksj92hs <134250687+lksj92hs@users.noreply.github.com> +lnigam <lnigam@nvidia.com> LoganDark <git@logandark.mozmail.com> LoganDark <github@logandark.mozmail.com> +losewayy <1134285612@qq.com> LostRuins <39025047+LostRuins@users.noreply.github.com> +lovedheart <6277001+lovedheart@users.noreply.github.com> +lucy <154630366+lucyknada@users.noreply.github.com> Lukas Möller <mail@lukas-moeller.ch> +Łukasz Ślusarczyk <112692748+lslusarczyk@users.noreply.github.com> +Łukasz Ślusarczyk <lukasz.slusarczyk@intel.com> +Lukasz Stolcman <4583553+lstolcman@users.noreply.github.com> +LunalFresh <165352784+LunalFresh@users.noreply.github.com> +luoyu-intel <yu.luo@intel.com> +luyhcsu <110711054+luyhcsu@users.noreply.github.com> M Refi D.A <24388107+refinism@users.noreply.github.com> +M. Mediouni <mohamed@unpredictable.fr> M. Yusuf Sarıgöz <yusufsarigoz@gmail.com> +m1el <m1el@ya.ru> Ma Mingfei <mingfei.ma@intel.com> +Mads Marquart <mads@marquart.dk> +magicse <magicse@users.noreply.github.com> +Mahekk Shaikh <118063190+Mahekk357@users.noreply.github.com> Mahesh Madhav <67384846+heshpdx@users.noreply.github.com> +mahorozte <41834471+mahorozte@users.noreply.github.com> MaiHD <maihd.dev@gmail.com> +Marcel Petrick <mail@marcelpetrick.it> +Marco Colombo <ing.marco.colombo@gmail.com> +Marcus <marcusds@users.noreply.github.com> +Mario Limonciello <mario.limonciello@amd.com> +Mario Limonciello <superm1@kernel.org> +Mark Todorovich <mtodorov@qti.qualcomm.com> Mark Zhuang <zhuangqiubin@gmail.com> +Markus Ebner <seijikun@users.noreply.github.com> Markus Tavenrath <mtavenrath@users.noreply.github.com> +Martin Chang <marty1885@users.noreply.github.com> +Martin Klacer <martin.klacer@arm.com> +Marvin Gießing <marvin.giessing@gmail.com> +Masashi Yoshimura <yoshimura.masashi.frbs@gmail.com> +Masato Nakasaka <masato.nakasaka@intel.com> +Masato Nakasaka <rillomas@gmail.com> Masaya, Kato <62578291+msy-kato@users.noreply.github.com> +mashizora <30516315+mashizora@users.noreply.github.com> +Mason Milburn <masonmilby@gmail.com> Mathieu Baudier <mbaudier@argeo.org> Mathijs de Bruin <mathijs@mathijsfietst.nl> +matiaslin <45382001+matiaslin@users.noreply.github.com> +Matt Corallo <649246+TheBlueMatt@users.noreply.github.com> +Matt Jallo <matt@mattjallo.com> Matt Stephenson <mstephenson6@users.noreply.github.com> +matt23654 <193348153+matt23654@users.noreply.github.com> +matt23654 <matthew.webber@protonmail.com> +matteo <matteogeniaccio@yahoo.it> +Matthew Michel <matthew.michel@intel.com> Max Krasnyansky <max.krasnyansky@gmail.com> +Max Krasnyansky <maxk@qti.qualcomm.com> Max Krasnyansky <quic_maxk@quicinc.com> +Maxim Evtush <154841002+maximevtush@users.noreply.github.com> Mayank Kumar Pal <mynkpl1998@gmail.com> +meatposes <computerdork@verizon.net> +MeeMin <74113151+Meet91721@users.noreply.github.com> +Mendy Berger <12537668+MendyBerger@users.noreply.github.com> Meng, Hengyu <hengyu.meng@intel.com> Mengqing Cao <cmq0113@163.com> +Mengsheng Wu <mengshen@qti.qualcomm.com> Metal Whale <45712559+metalwhale@users.noreply.github.com> +mgroeber9110 <45620825+mgroeber9110@users.noreply.github.com> +Miaoqian Lin <linmq006@gmail.com> +Michael de Gans <michael.john.degans@gmail.com> +Michael Huang <15768500+tehsiuhuang@users.noreply.github.com> Michael Klimenko <mklimenko29@gmail.com> +Michael Lamothe <michael.lamothe@gmail.com> Michael Podvitskiy <podvitskiymichael@gmail.com> +Michael Taylor <162068037+mctylr-gh@users.noreply.github.com> Michael Verrilli <msv@pobox.com> +Michael Wand <michael@baybridgeaquarium.com> +michaeltrabalka-tech <michael.trabalka@gmail.com> +midnight <midnightmagic@users.noreply.github.com> +Mike Abbott <furrysalamander@gmail.com> +Mike van Lammeren <mike@van.lammeren.net> +Mishig <dmishig@gmail.com> +miyan <1138989048@qq.com> +mkoker <132301062+mkoker@users.noreply.github.com> +mmalpe-nv <mmalpe@nvidia.com> +mnehete32 <33429707+mnehete32@users.noreply.github.com> +Mohamed Elashri <git@elashri.com> Molly Sophia <mollysophia379@gmail.com> +MoonShadow <moonshadow25@163.com> +Mostafa <mostafaaafaheem@gmail.com> +muggle-stack <promuggle@qq.com> +Muhammad Saad <saadydev@gmail.com> +Naco Siren <naco-siren@users.noreply.github.com> +Nathan Wilson <67372905+Nathanw1014@users.noreply.github.com> +Nathanw1014 <67372905+Nathanw1014@users.noreply.github.com> Natsu <chino@hotococoa.moe> +Nechama Krashinski <n.05567347@gmail.com> +neha-ha <137219201+neha-ha@users.noreply.github.com> Neo Zhang <14088817+arthw@users.noreply.github.com> +Neo Zhang <zhang.jianyu@outlook.com> Neo Zhang Jianyu <jianyu.zhang@intel.com> Neuman Vong <neuman.vong@gmail.com> Nevin <nevinpuri1901@gmail.com> Nicholai Tukanov <nicholaitukanov@gmail.com> +Nick Farrell <nick.farrell@aiven.io> +Nick Lafleur <55208706+nicklafleur@users.noreply.github.com> Nico Bosshard <nico@bosshome.ch> +Nicolas Mowen <nickmowen213@gmail.com> Nicolò Scipione <nicolo.scipione@codeplay.com> +Nikhil Jain <nikhil.jain0987@gmail.com> +Nikhil Sharma <150424968+nikhilfujitsu@users.noreply.github.com> Nikita Sarychev <42014488+sARY77@users.noreply.github.com> +Niklas Wenzel <dev@nikwen.de> +Niko Maroulis <nik.maroulis@gmail.com> +Nils Gladitz <nilsgladitz@gmail.com> +Noah <99681487+NoahOksuz@users.noreply.github.com> +Nothing <anranyus@hotmail.com> Nouamane Tazi <nouamane98@gmail.com> +nullname <chraac@gmail.com> +nuri <yoonuri1@gmail.com> +nwyin <tommynguyen0512@gmail.com> +o7si <32285332+o7si@users.noreply.github.com> +ochafik <ochafik@google.com> +Oleksandr Kuvshynov <661042+okuvshynov@users.noreply.github.com> +Oliver Simons <oliver.simons@posteo.de> +Oliver Simons <osimons@nvidia.com> +Oliver Walsh <owalsh@redhat.com> Olivier Chafik <ochafik@google.com> Olivier Chafik <ochafik@users.noreply.github.com> Ondřej Čertík <ondrej@certik.us> +oobabooga <oobabooga4@gmail.com> +Ori Pekelman <ori@pekelman.com> +otaGran <ujt2h8@gmail.com> Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com> +Ozymandias_EBON <112784549+johnkarlhill@users.noreply.github.com> PAB <pierreantoine.bannier@gmail.com> +Pascal <admin@serveurperso.com> +Pasha Khosravi <khosravipasha@users.noreply.github.com> +Patrick Buckley <eous@users.noreply.github.com> +Patrick Hoffmann <patrick@hoffmann.lol> +Patrick Peng <retr0@retr0.blog> +Patrick00017 <42609007+Patrick00017@users.noreply.github.com> +Patryk Kaminski <kaminpatryk@gmail.com> +Paul Flynn <paul@arkavo.com> +Paul Mairhofer <paul@protom.eu> Paul Tsochantaris <ptsochantaris@icloud.com> +Pavels Zaicenkovs <github@a.pzv.me> +Penglin Cai <1402538448@qq.com> +pengxin99 <pengxin.yuan@intel.com> +Pepper Gray <hello@peppergray.xyz> +Perry Naseck <4472083+DaAwesomeP@users.noreply.github.com> Peter <peter277@users.noreply.github.com> +Peter0x44 <peter0x44@disroot.org> +petterreinholdtsen <pere-github@hungry.com> Philpax <me@philpax.me> +Phylliida Dev <phylliida.dev@gmail.com> Pierre Alexandre SCHEMBRI <pa.schembri@gmail.com> +pikalover6 <49179590+pikalover6@users.noreply.github.com> +PikaPikachu <kangletian@hotmail.com> +Piotr Kubaj <pkubaj@anongoth.pl> +Piotr Wilkin (ilintar) <piotr.wilkin@syndatis.com> +pl752 <pl752@mail.ru> Plamen Minev <pacominev@gmail.com> Playdev <josang1204@gmail.com> +Ploo <239304139+xinitrcn1@users.noreply.github.com> +pmaybank <113125070+pmaybank@users.noreply.github.com> +PMZFX <georgiopapairo@gmail.com> +postmasters <namnguyen@google.com> +Prabhsimran Singh <prabhsimrans@nvidia.com> +Pradeep Rao <pradeep.rao@amd.com> +Prajwal B Mehendarkar <prajwal.b.mehendarkar@ibm.com> +Pranav Dhinakar <pdhinaka@qti.qualcomm.com> +Pranav Dhinakar <pranavdhinakar@gmail.com> +Pranav Uttarkar <122235768+PranavUttarkar@users.noreply.github.com> +Pranesh Gonegandla <pranesh.iitp@gmail.com> Prashant Vithule <119530321+Vithulep@users.noreply.github.com> +pratiknarola-t <pratik.narola@tether.io> +Pratyush Kumar <praty.bachchas@gmail.com> +ProgenyAlpha <loveandhappypaws@gmail.com> Przemysław Pawełczyk <przemoc@gmail.com> +PureJourney <edward.pong@qq.com> +QDelta <60222316+QDelta@users.noreply.github.com> +Qeeweew <68716978+Qeeweew@users.noreply.github.com> +qiao_px <84374584+Qiao12-pixel@users.noreply.github.com> +qiurui144 <39214303+qiurui144@users.noreply.github.com> +QuintinShaw <github@xyt.email> +QuintinShaw <yx6f20@soton.ac.uk> R0CKSTAR <xiaodong.ye@mthreads.com> R0CKSTAR <yeahdongcn@gmail.com> +RachelMantel <rrm85040@gmail.com> Radoslav Gerganov <rgerganov@gmail.com> Radosław Gryta <radek.gryta@gmail.com> +Rafail Giavrimis <47496212+grafail@users.noreply.github.com> +Rafal Lewczuk <rafal.lewczuk@gmail.com> +ragz4125 <65285549+ragz4125@users.noreply.github.com> +Rahul Sathe <150351592+rrsathe@users.noreply.github.com> +Rail Chabdarov <rchabdar@amd.com> +Rajendra Matcha <matcraje@qti.qualcomm.com> +Raman Shinde <raman.shinde15@gmail.com> +RapidMark <32768622+RapidMark@users.noreply.github.com> +Rashid Ul Islam <33536561+Ra5hidIslam@users.noreply.github.com> +Raul Torres <138264735+rauletorresc@users.noreply.github.com> +ravel7524 <58877666+ravel7524@users.noreply.github.com> +Ravi Panchumarthy <ravi.panchumarthy@intel.com> Ravindra Marella <marella@users.noreply.github.com> Ray Cromwell <cromwellian@gmail.com> +RealOrko <45273739+RealOrko@users.noreply.github.com> +redfox <59549776+yaohengxu@users.noreply.github.com> +Reese Levine <reeselevine1@gmail.com> +rehan-10xengineer <rehanbackup0317@gmail.com> Reinforce-II <fate@eastal.com> +Rémy O <remyoudompheng@gmail.com> Rémy Oudompheng <oudomphe@phare.normalesup.org> +ren <189031187+lathrys-at@users.noreply.github.com> Reza Rezvan <reza@rezvan.xyz> +Richard <r-burton@hotmail.co.uk> +Richard Davison <richard.davison1@gmail.com> Rick G <26732651+TheFlipbook@users.noreply.github.com> +Rithik Sharma <rithiksh02@gmail.com> RiverZhou <riverzhou2000@gmail.com> +rmatif <66360289+rmatif@users.noreply.github.com> +rmatif <kingrealriadh@gmail.com> +rmatif <rmatif@proton.me> +Robert Esclapez <Robert.Garcia@amd.com> Robert Ormandi <52251610+ormandi@users.noreply.github.com> +robertomeroni <150194833+robertomeroni@users.noreply.github.com> +Rohanjames1997 <rohan.james4@gmail.com> +Rohit Mahesh <74331568+rohitmahesh1@users.noreply.github.com> Romain Biessy <romain.biessy@codeplay.com> +Ronny Brendel <ronnyb@nvidia.com> Ronsor <ronsor@ronsor.pw> Rotem Dan <rotemdan@gmail.com> +Ruben Ortlam <picard12@live.de> +Ruben Ortlam <rortlam@redhat.com> +Ryan C <ryan5rdx@users.noreply.github.com> Ryan Hitchman <hitchmanr@gmail.com> -SRHMorris <69468379+SRHMorris@users.noreply.github.com> -SXX <sxx1136965276@gmail.com> +s-goto-11 <206795233+s-goto-11@users.noreply.github.com> +s8322 <s0527684199@gmail.com> +Saba Fallah <10401143+sfallah@users.noreply.github.com> +Sachin Sharma <sachin@zettabolt.com> +Safi Ullah <safiullah.3915@gmail.com> +safranowith <bsh155762@gmail.com> +Salman Chishti <salmanmkc@GitHub.com> Salvatore Mesoraca <s.mesoraca16@gmail.com> +Sam Murdoch <hello@burl.au> Sam Spilsbury <smspillaz@gmail.com> +Sam/Samuel <57896620+cern1710@users.noreply.github.com> +SamareshSingh <97642706+ssam18@users.noreply.github.com> Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> +Sandro Hanea <40202887+sandrohanea@users.noreply.github.com> +Sandro Steeger <78495486+Stastez@users.noreply.github.com> Santtu Keskinen <santtu.keskinen@gmail.com> +Sarah Wu <sarahwu185@gmail.com> +Sarah Wu <sarwu@nvidia.com> +SavicStefan <50296686+SavicStefan@users.noreply.github.com> +Scott Fudally <sfudally@nvidia.com> +scutler-nv <scutler@nvidia.com> +SecondSeed <38520721+SecondSeed@users.noreply.github.com> Sergio López <slp@redhat.com> Sergio López <slp@sinrega.org> +Seyoung Jeong <seyoungjeong@gmail.com> +SG-Amadeus <74057036+SG-Amadeus@users.noreply.github.com> +Shagun Bera <141054835+notV3NOM@users.noreply.github.com> +shalinib-ibm <Shalini.Salomi.Bodapati@ibm.com> +Shang Yuanchun <idealities@gmail.com> +shani-f <s0556787439@gmail.com> Shanshan Shen <467638484@qq.com> +shaofeiqi <109865877+shaofeiqi@users.noreply.github.com> +shaofeiqi <shaoqi@qti.qualcomm.com> +Shaw Nguyen <49144872+mrshaw01@users.noreply.github.com> +Shawn Gu <shawngu@qti.qualcomm.com> +Shawn yang <137684499+Yangxiaoz@users.noreply.github.com> +Sheldon Robinson <sheldon.robinson@live.com> +Shenghan Yang <ysharke@sjtu.edu.cn> Shijie <821898965@qq.com> +Shin-myoung-serp <relent95@naver.com> +Shobhit <sobhit.me@gmail.com> +Shouyu <65317431+joeldushouyu@users.noreply.github.com> +Shreya Jain <shreyajn@qti.qualcomm.com> +Shrivas Shankar <86219405+shrivasshankar@users.noreply.github.com> +SHUAI YANG <shuaiyang047@163.com> Shupei Fan <dymarkfan@outlook.com> +Siavash Norouzi <35790025+siavashnorouzi@users.noreply.github.com> Siddharth Ramakrishnan <srr2141@columbia.edu> +Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com> +Simon Redman <simon@ergotech.com> +Simon Teixidor <simon@flaskpost.me> +simon886212 <37953122+simon886212@users.noreply.github.com> +sirus20x6 <sirus20x6@users.noreply.github.com> +sjinzh <sjinzh@gmail.com> +skirodev <57715494+skirodev@users.noreply.github.com> +Sky <Iflyinskyin2013@gmail.com> Skyler Celestinian-Sterling <80314197+Celestinian@users.noreply.github.com> +slaren <slarengh@gmail.com> Slava Primenko <primenko.s@gmail.com> +sliu39 <russell.liu@intel.com> +Slobodan Josic <127323561+slojosic-amd@users.noreply.github.com> +snadampal <87143774+snadampal@users.noreply.github.com> +someone13574 <81528246+someone13574@users.noreply.github.com> +someoneinjd <someoneinjd@outlook.com> +Sophon <strongtz@yeah.net> +Sou-ly <79574807+Sou-ly@users.noreply.github.com> +Sreeram <sreeram.sivasubramony@amd.com> +SRHMorris <69468379+SRHMorris@users.noreply.github.com> Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com> +stduhpf <stephduh@live.fr> +Stephen Cox <scox@ict.co> +Steve Lhomme <robux4@ycbcr.xyz> Steward Garcia <57494570+FSSRepo@users.noreply.github.com> +Strongtut <Strongtut@users.noreply.github.com> +StyMaar <StyMaar@users.noreply.github.com> +sudhiarm <sudhi.sathyavathy@arm.com> +Sundaram krishnan <104441812+sundaram123krishnan@users.noreply.github.com> Supreet Sethi <supreet.sethi@gmail.com> +Svetlozar Georgiev <55534064+sgeor255@users.noreply.github.com> +SXX <song_xiaoxi@126.com> +SXX <sxx1136965276@gmail.com> +taher <8665427+nullhook@users.noreply.github.com> +Taimur Ahmad <taimur.ahmad@10xengineers.ai> +Tak-RS <snosk.t@gmail.com> +takuya kodama <a.s.takuya1026@gmail.com> Takuya Takeuchi <takuya.takeuchi.dev@gmail.com> +Talha Adnan <mkhan387@uic.edu> +Talha Can Havadar <havadartalha@gmail.com> +Tamar <Tamar0812@outlook.co.il> +tamarPal <tamarp3385@gmail.com> Tamotsu Takahashi <ttakah+github@gmail.com> Tanmay <tnmysachan@gmail.com> Tanmay Sachan <tnmysachan@gmail.com> +Tanner Bruhn <66120666+tannerbruhn@users.noreply.github.com> +Tarek Dakhran <t.dakhran@gmail.com> +Tarek Dakhran <tarek@liquid.ai> +Taylor <tnie13@hotmail.com> +TecJesh <qdvm5gl@163.com> +Tekin Ertekin <tekin.ertekin@gmail.com> +texasich <101962694+texasich@users.noreply.github.com> +texmex76 <40733439+texmex76@users.noreply.github.com> +the-crypt-keeper <84680712+the-crypt-keeper@users.noreply.github.com> +TheArchitectit <roger@vroger.com> +Theera K. <tkittich@hotmail.com> +thelittlefireman <5165783+thelittlefireman@users.noreply.github.com> +theo77186 <theo77186@users.noreply.github.com> +thewh1teagle <61390950+thewh1teagle@users.noreply.github.com> +Thiago Padilha <thiago@padilha.cc> +Thore Koritzius <thorekoritzius@outlook.de> +TianHao324 <854531745@qq.com> +TianHao324 <tianhao42@huawei.com> Timothy Cronin <40186632+4imothy@users.noreply.github.com> +Titaniumtown <titaniumtown@proton.me> +Todd Malsbary <todd.malsbary@intel.com> +Todor Boinovski <tboinovski@gmail.com> +Todor Boinovski <todorb@qti.qualcomm.com> Tom Bailey <tombailey@users.noreply.github.com> Tom Jobbins <784313+TheBloke@users.noreply.github.com> +Tom Overlund <tomov@dilacero.org> Tony Wasserka <4840017+neobrain@users.noreply.github.com> Tristan Druyen <tristan@vault81.mozmail.com> +Trivalik <3148279+trivalik@users.noreply.github.com> +Trivikram Reddy <127072883+trivikram-reddy1@users.noreply.github.com> Tyé singwa <92231658+tye-singwa@users.noreply.github.com> +tyronecai <tyronecai@163.com> +Uber <20715993+YagUber@users.noreply.github.com> +ucag.li <ucag@qq.com> UEXTM.com <84163508+uextm@users.noreply.github.com> +ulatekh <ulatekh@yahoo.com> +unraido <127105806+unraido@users.noreply.github.com> +uvos <carl@uvos.xyz> +uvos <devnull@uvos.xyz> +uvos <philipp@uvos.xyz> +Valeriy Dubov <dvv101111@gmail.com> +Vedran Miletić <vedran@miletic.net> +Vexxie <rainandriamusic@gmail.com> +Ville Vesilehto <ville@vesilehto.fi> +vinay <94120295+vinaykulk621@users.noreply.github.com> +Vineel Abhinav <131174187+vineelabhinav@users.noreply.github.com> +virajwad <84867530+virajwad@users.noreply.github.com> +Vishal Singh <vishal@zettabolt.com> +Vladimir Vuksanovic <109677816+vvuksanovic@users.noreply.github.com> +Vladislav <vladplotnikov34@gmail.com> +vmobilis <75476228+vmobilis@users.noreply.github.com> +Wagner Bruna <wbruna@users.noreply.github.com> +WakeUpMorty <balogkarlo4@gmail.com> +Wallentri <wallentridan88@proton.me> +WalterCool <waltercool@slash.cl> +Wang Weixuan <wangweixvan@gmail.com> +wangshuai09 <391746016@qq.com> +Wei Wang <w10493wang@163.com> +Weizhao Ouyang <o451686892@gmail.com> +WenqiangJia2026 <wenqijia@amd.com> +wes <huangwei858@126.com> +will-lms <will@lmstudio.ai> WillCorticesAI <150854901+WillCorticesAI@users.noreply.github.com> William Tambellini <william.tambellini@gmail.com> William Tambellini <wtambellini@sdl.com> +Winston Ma <winstonma@ymail.com> +woachk <24752637+woachk@users.noreply.github.com> +wsbagnsv1 <sclumpfpapa36@gmail.com> +Wyatt Caldwell <218154709+Detensable@users.noreply.github.com> +xctan <axunlei@gmail.com> +xctan <xc-tan@outlook.com> +Xiang Chen <46052474+x14ngch3n@users.noreply.github.com> +Xiangyan Sun <wishstudio@gmail.com> +xiaobing318 <71554036+xiaobing318@users.noreply.github.com> +xiaofei <hbuxiaofei@gmail.com> XiaotaoChen <chenxiaotao1234@gmail.com> +Xinpeng Dou <15529241576@163.com> Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com> Xuan Son Nguyen <thichthat@gmail.com> +Xuan-Son Nguyen <son@huggingface.co> +Xuan-Son Nguyen <thichthat@gmail.com> +XZiar <15145384+XZiar@users.noreply.github.com> +y198 <90976397+y198nt@users.noreply.github.com> +yael-works <106673277+yael-works@users.noreply.github.com> +YaelGitAccount <38328157276@mby.co.il> +YaelLogic <y0548591250@gmail.com> +YangLe <smilingpoplar@gmail.com> +yangyaofei <yangyaofei@gmail.com> +Yangyu Chen <cyy@cyyself.name> +Yanzhao Wang <yanzhaow@qti.qualcomm.com> +Yanzhao Wang <yzwang1996@gmail.com> +Yarden Tal <yardent@qti.qualcomm.com> +YardenTal44 <yardent@qti.qualcomm.com> +Yash Raj Pandey <55940078+devYRPauli@users.noreply.github.com> +Yavor Ivanov <yavorgenadiev@gmail.com> Yavor Ivanov <yivanov@viewray.com> YavorGIvanov <yivanov@viewray.com> +YehuditE <y8703470@gmail.com> +Yibo Cai <cyb70289@gmail.com> +Yibo Cai <yibo.cai@arm.com> +YiChen Lv <63285796+forforever73@users.noreply.github.com> +Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Yilong Guo <vfirst218@gmail.com> Yilong Guo <yilong.guo@intel.com> +Yiwei Shao <44545837+njsyw1997@users.noreply.github.com> +ymcki <84055651+ymcki@users.noreply.github.com> +ynankani <ynankani@nvidia.com> +yomi <3527628634@qq.com> +Yonghye Kwon <developer.0hye@gmail.com> +Yongmin Yoo 유용민 <yymin1022@gmail.com> +Yoshi_likes_e4 <104140648+pt13762104@users.noreply.github.com> +yulo <77381088+zhang-hui-yulo@users.noreply.github.com> Yuri Khrustalev <ykhrustalev@users.noreply.github.com> +yuri@FreeBSD <yuri@FreeBSD> +yzyyzyhhh <96101183+happyyzy@users.noreply.github.com> +z <faichou.zh@gmail.com> +Zack Li <39573601+zhiyuan8@users.noreply.github.com> +zduford <z.duford@gmail.com> +zhentaoyu <zhentao.yu@intel.com> Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com> +Zheyuan Chen <sephirotheca17@gmail.com> Zhiyuan Li <lizhiyuan@uniartisan.com> Zhiyuan Li <uniartisan2017@gmail.com> -a3sh <38979186+A3shTnT@users.noreply.github.com> -ag2s20150909 <19373730+ag2s20150909@users.noreply.github.com> -agray3 <agray3@users.noreply.github.com> -amd-dwang <dong.wang@amd.com> -amritahs-ibm <amritahs@linux.vnet.ibm.com> -apcameron <37645737+apcameron@users.noreply.github.com> -appvoid <78444142+appvoid@users.noreply.github.com> -ariez-xyz <41232910+ariez-xyz@users.noreply.github.com> -automaticcat <daogiatuank54@gmail.com> -bandoti <141645996+bandoti@users.noreply.github.com> -bmwl <brian.marshall@tolko.com> -bobqianic <129547291+bobqianic@users.noreply.github.com> -bssrdf <merlintiger@hotmail.com> -chengchi <davesjoewang@gmail.com> -compilade <113953597+compilade@users.noreply.github.com> -compilade <git@compilade.net> -ddpasa <112642920+ddpasa@users.noreply.github.com> -denersc <denerstassun@gmail.com> -dscripka <dscripka@users.noreply.github.com> -fitzsim <fitzsim@fitzsim.org> -fj-y-saito <85871716+fj-y-saito@users.noreply.github.com> -fraxy-v <65565042+fraxy-v@users.noreply.github.com> -gn64 <yukikaze.jp@gmail.com> -goerch <jhr.walter@t-online.de> -goldwaving <77494627+goldwaving@users.noreply.github.com> -haopeng <657407891@qq.com> -hidenorly <hidenorly@users.noreply.github.com> -hipudding <huafengchun@gmail.com> -hydai <z54981220@gmail.com> -issixx <46835150+issixx@users.noreply.github.com> -jaeminSon <woalsdnd@gmail.com> -jdomke <28772296+jdomke@users.noreply.github.com> -jiez <373447296@qq.com> -johnson442 <56517414+johnson442@users.noreply.github.com> -junchao-loongson <68935141+junchao-loongson@users.noreply.github.com> -k.h.lai <adrian.k.h.lai@outlook.com> -katsu560 <118887472+katsu560@users.noreply.github.com> -klosax <131523366+klosax@users.noreply.github.com> -kunnis <kunnis@users.noreply.github.com> -l3utterfly <gc.pthzfoldr@gmail.com> -le.chang <cljs118@126.com> -leejet <31925346+leejet@users.noreply.github.com> -leejet <leejet714@gmail.com> -leo-pony <nengjunma@outlook.com> -lhez <quic_lih@quicinc.com> -liuwei-git <14815172+liuwei-git@users.noreply.github.com> -luoyu-intel <yu.luo@intel.com> -magicse <magicse@users.noreply.github.com> -mahorozte <41834471+mahorozte@users.noreply.github.com> -mashizora <30516315+mashizora@users.noreply.github.com> -matt23654 <matthew.webber@protonmail.com> -matteo <matteogeniaccio@yahoo.it> -ochafik <ochafik@google.com> -otaGran <ujt2h8@gmail.com> -pengxin99 <pengxin.yuan@intel.com> -pikalover6 <49179590+pikalover6@users.noreply.github.com> -postmasters <namnguyen@google.com> -sjinzh <sjinzh@gmail.com> -skirodev <57715494+skirodev@users.noreply.github.com> -slaren <slarengh@gmail.com> -snadampal <87143774+snadampal@users.noreply.github.com> -someone13574 <81528246+someone13574@users.noreply.github.com> -stduhpf <stephduh@live.fr> -taher <8665427+nullhook@users.noreply.github.com> -texmex76 <40733439+texmex76@users.noreply.github.com> -the-crypt-keeper <84680712+the-crypt-keeper@users.noreply.github.com> -thewh1teagle <61390950+thewh1teagle@users.noreply.github.com> -ucag.li <ucag@qq.com> -ulatekh <ulatekh@yahoo.com> -uvos <devnull@uvos.xyz> -uvos <philipp@uvos.xyz> -wangshuai09 <391746016@qq.com> -woachk <24752637+woachk@users.noreply.github.com> -xctan <axunlei@gmail.com> -yangyaofei <yangyaofei@gmail.com> -yuri@FreeBSD <yuri@FreeBSD> -zhentaoyu <zhentao.yu@intel.com> zhouwg <6889919+zhouwg@users.noreply.github.com> zhouwg <zhouwg2000@gmail.com> -谢乃闻 <sienaiwun@users.noreply.github.com> -布客飞龙 <562826179@qq.com> -旺旺碎冰冰 <38837039+Cyberhan123@users.noreply.github.com> +ZihaoMu <zmu@amd.com> +Zijun Yu <zijun.yu.joey@gmail.com> +Zijun Yu <zijun.yu@intel.com> +zyang-dev <267119621+zyang-dev@users.noreply.github.com> +zzzzwc <tiddar@foxmail.com> diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 3eacd4cd5..b1018cbbb 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -3,8 +3,8 @@ cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit project("ggml" C CXX ASM) ### GGML Version -set(GGML_VERSION_MAJOR 0) -set(GGML_VERSION_MINOR 26) +set(GGML_VERSION_MAJOR 1) +set(GGML_VERSION_MINOR 0) set(GGML_VERSION_PATCH 0) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") @@ -205,7 +205,7 @@ option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copie option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF) option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON) option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF) -set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING +set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16;f8_e4m3-f8_e4m3" CACHE STRING "ggml: FlashAttention K-V type combinations to compile, \"all\" or a list such as \"q8_0-q8_0;q8_0-q4_0\"") option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT}) option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON) @@ -250,6 +250,7 @@ option(GGML_OPENMP_FETCH "ggml: fetch LLVM OpenMP" option(GGML_RPC "ggml: use RPC" OFF) option(GGML_SYCL "ggml: use SYCL" OFF) option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF) +option(GGML_SYCL_SUPPORT_XMX "ggml: build the XMX (DPAS) code paths in the SYCL backend" ON) option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON) option(GGML_SYCL_HOST_MEM_FALLBACK "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON) option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API "ggml: use Level Zero API in SYCL backend" ON) diff --git a/ggml/UPSTREAM b/ggml/UPSTREAM index 80d7c8933..3f7239e30 100644 --- a/ggml/UPSTREAM +++ b/ggml/UPSTREAM @@ -1,5 +1,5 @@ repo: git@github.com:ggml-org/ggml.git -sha: d7cb574130e6f01ad25b3289685489200febcd74 +sha: 417a024b4fb6bc3116d41af7c067166a039bdf12 patches: patches/ggml/0001-fix-threadpool-oversubscription.patch patches/ggml/0002-backend-reg-filter.patch diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake index f3610298f..c8d232294 100644 --- a/ggml/cmake/common.cmake +++ b/ggml/cmake/common.cmake @@ -2,10 +2,16 @@ function(ggml_get_flags CCID CCVER) set(C_FLAGS "") set(CXX_FLAGS "") - if (CCID MATCHES "Clang") + if (CCID MATCHES "Clang" OR CCID MATCHES "IntelLLVM") set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return) set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi) + if (CCID STREQUAL "IntelLLVM") + # for Intel DPC++ and LLVM SYCL, using debug does not disable optimization + list(APPEND C_FLAGS -Rno-debug-disables-optimization) + list(APPEND CXX_FLAGS -Rno-debug-disables-optimization) + endif() + if ( (CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0) @@ -52,7 +58,7 @@ endfunction() # Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS. function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS) - set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16) + set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16 f8_e4m3) string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS) string(STRIP "${FA_QUANTS}" FA_QUANTS) diff --git a/ggml/docs/gguf.md b/ggml/docs/gguf.md index 21c5e8a2e..cafc1c270 100644 --- a/ggml/docs/gguf.md +++ b/ggml/docs/gguf.md @@ -150,6 +150,8 @@ Fields, including arrays, are written sequentially without alignment unless othe Models are little-endian by default. They can also come in big-endian for use with big-endian computers; in this case, all values (including metadata values and tensors) will also be big-endian. At the time of writing, there is no way to determine if a model is big-endian; this may be rectified in future versions. If no additional information is provided, assume the model is little-endian. +The `ggml_type` values below mirror `enum ggml_type` in [`include/ggml.h`](../include/ggml.h), which is authoritative and gains new values over time, so readers should take type codes and block sizes from there. + ```c enum ggml_type: uint32_t { GGML_TYPE_F32 = 0, @@ -192,7 +194,11 @@ enum ggml_type: uint32_t { // GGML_TYPE_IQ4_NL_4_8 = 37, // GGML_TYPE_IQ4_NL_8_8 = 38, GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) - GGML_TYPE_COUNT = 40, + GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_Q2_0 = 42, + GGML_TYPE_F8_E4M3 = 43, + GGML_TYPE_COUNT = 44, }; enum gguf_metadata_value_type: uint32_t { diff --git a/ggml/examples/gpt-2/main-alloc.cpp b/ggml/examples/gpt-2/main-alloc.cpp index 172d3eb6e..311bff1e2 100644 --- a/ggml/examples/gpt-2/main-alloc.cpp +++ b/ggml/examples/gpt-2/main-alloc.cpp @@ -430,6 +430,14 @@ struct ggml_cgraph * gpt2_graph( ggml_get_rows(ctx, model.wte, embd), ggml_get_rows(ctx, model.wpe, position)); + // KQ_mask = mask_past() + // [n_past + N, N, 1] + const int n_kv = n_past + N; + + struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_kv, N, 1); + ggml_set_name(KQ_mask, "KQ_mask"); + ggml_set_input(KQ_mask); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur; @@ -521,7 +529,7 @@ struct ggml_cgraph * gpt2_graph( // KQ_masked = mask_past(KQ_scaled) // [n_past + N, N, 12] - struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past); + struct ggml_tensor * KQ_masked = ggml_add(ctx, KQ_scaled, KQ_mask); // KQ = soft_max(KQ_masked) // [n_past + N, N, 12] @@ -698,6 +706,14 @@ bool gpt2_eval( ((int32_t *) position->data)[i] = n_past + i; } + struct ggml_tensor * KQ_mask = ggml_graph_get_tensor(gf, "KQ_mask"); + const int n_kv = n_past + N; + for (int i1 = 0; i1 < N; ++i1) { + for (int i0 = 0; i0 < n_kv; ++i0) { + ((float *) KQ_mask->data)[i0 + i1*n_kv] = i0 > n_past + i1 ? -INFINITY : 0.0f; + } + } + // run the computation struct ggml_cplan plan = ggml_graph_plan(gf, n_threads, nullptr); static std::vector<uint8_t> work_buffer; diff --git a/ggml/examples/gpt-2/main-backend.cpp b/ggml/examples/gpt-2/main-backend.cpp index 6c6871247..5ada26724 100644 --- a/ggml/examples/gpt-2/main-backend.cpp +++ b/ggml/examples/gpt-2/main-backend.cpp @@ -488,6 +488,14 @@ struct ggml_cgraph * gpt2_graph( ggml_get_rows(ctx, model.wte, embd), ggml_get_rows(ctx, model.wpe, position)); + // KQ_mask = mask_past() + // [n_past + N, N, 1] + const int n_kv = n_past + N; + + struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_kv, N, 1); + ggml_set_name(KQ_mask, "KQ_mask"); + ggml_set_input(KQ_mask); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur; @@ -579,7 +587,7 @@ struct ggml_cgraph * gpt2_graph( // KQ_masked = mask_past(KQ_scaled) // [n_past + N, N, 12] - struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past); + struct ggml_tensor * KQ_masked = ggml_add(ctx, KQ_scaled, KQ_mask); // KQ = soft_max(KQ_masked) // [n_past + N, N, 12] @@ -757,6 +765,17 @@ bool gpt2_eval( ggml_backend_tensor_set(position, &v, i*sizeof(int32_t), sizeof(v)); } + struct ggml_tensor * KQ_mask = ggml_graph_get_tensor(gf, "KQ_mask"); + const int n_kv = n_past + N; + + std::vector<float> KQ_mask_data(n_kv*N); + for (int i1 = 0; i1 < N; ++i1) { + for (int i0 = 0; i0 < n_kv; ++i0) { + KQ_mask_data[i0 + i1*n_kv] = i0 > n_past + i1 ? -INFINITY : 0.0f; + } + } + ggml_backend_tensor_set(KQ_mask, KQ_mask_data.data(), 0, KQ_mask_data.size()*sizeof(float)); + // set backend options if (ggml_backend_is_cpu(model.backend)) { ggml_backend_cpu_set_n_threads(model.backend, n_threads); diff --git a/ggml/examples/gpt-2/main-ctx.cpp b/ggml/examples/gpt-2/main-ctx.cpp index fe099977a..b7bf0619d 100644 --- a/ggml/examples/gpt-2/main-ctx.cpp +++ b/ggml/examples/gpt-2/main-ctx.cpp @@ -445,6 +445,17 @@ bool gpt2_eval( ggml_get_rows(ctx0, model.wte, embd), ggml_get_rows(ctx0, model.wpe, position)); + // KQ_mask = mask_past() + // [n_past + N, N, 1] + const int n_kv = n_past + N; + + struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, N, 1); + for (int i1 = 0; i1 < N; ++i1) { + for (int i0 = 0; i0 < n_kv; ++i0) { + ((float *) KQ_mask->data)[i0 + i1*n_kv] = i0 > n_past + i1 ? -INFINITY : 0.0f; + } + } + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur; @@ -535,7 +546,7 @@ bool gpt2_eval( // KQ_masked = mask_past(KQ_scaled) // [n_past + N, N, 12] - struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past); + struct ggml_tensor * KQ_masked = ggml_add(ctx0, KQ_scaled, KQ_mask); // KQ = soft_max(KQ_masked) // [n_past + N, N, 12] diff --git a/ggml/examples/gpt-2/main-sched.cpp b/ggml/examples/gpt-2/main-sched.cpp index 7d789d68a..d3774baad 100644 --- a/ggml/examples/gpt-2/main-sched.cpp +++ b/ggml/examples/gpt-2/main-sched.cpp @@ -596,6 +596,14 @@ struct ggml_cgraph * gpt2_graph( ggml_set_name(inpL->src[0], "wte"); ggml_set_name(inpL->src[1], "wpe"); + // KQ_mask = mask_past() + // [n_past + N, N, 1] + const int n_kv = n_past + N; + + struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_kv, N, 1); + ggml_set_name(KQ_mask, "KQ_mask"); + ggml_set_input(KQ_mask); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur; @@ -697,7 +705,7 @@ struct ggml_cgraph * gpt2_graph( // KQ_masked = mask_past(KQ_scaled) // [n_past + N, N, 12] - struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past); + struct ggml_tensor * KQ_masked = ggml_add(ctx, KQ_scaled, KQ_mask); ggml_format_name(KQ_masked, "l%d.KQ_masked", il); // KQ = soft_max(KQ_masked) @@ -878,8 +886,21 @@ bool gpt2_eval( struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp); - // run the computation ggml_backend_sched_reset(sched); + ggml_backend_sched_alloc_graph(sched, gf); + + struct ggml_tensor * KQ_mask = ggml_graph_get_tensor(gf, "KQ_mask"); + const int n_kv = n_past + N; + + std::vector<float> KQ_mask_data(n_kv*N); + for (int i1 = 0; i1 < N; ++i1) { + for (int i0 = 0; i0 < n_kv; ++i0) { + KQ_mask_data[i0 + i1*n_kv] = i0 > n_past + i1 ? -INFINITY : 0.0f; + } + } + ggml_backend_tensor_set(KQ_mask, KQ_mask_data.data(), 0, KQ_mask_data.size()*sizeof(float)); + + // run the computation ggml_backend_sched_graph_compute(sched, gf); //if (n_past%100 == 0) { diff --git a/ggml/examples/gpt-j/main.cpp b/ggml/examples/gpt-j/main.cpp index 2c175772d..e95379b59 100644 --- a/ggml/examples/gpt-j/main.cpp +++ b/ggml/examples/gpt-j/main.cpp @@ -441,6 +441,17 @@ bool gptj_eval( // wte struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd); + // KQ_mask = mask_past() + // [n_past + N, N, 1] + const int n_kv = n_past + N; + + struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, N, 1); + for (int i1 = 0; i1 < N; ++i1) { + for (int i0 = 0; i0 < n_kv; ++i0) { + ((float *) KQ_mask->data)[i0 + i1*n_kv] = i0 > n_past + i1 ? -INFINITY : 0.0f; + } + } + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur; @@ -500,7 +511,7 @@ bool gptj_eval( 1.0f/sqrt(float(n_embd)/n_head)); // KQ_masked = mask_past(KQ_scaled) - struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past); + struct ggml_tensor * KQ_masked = ggml_add(ctx0, KQ_scaled, KQ_mask); // KQ = soft_max(KQ_masked) struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked); diff --git a/ggml/examples/sam/sam.cpp b/ggml/examples/sam/sam.cpp index e5a58dcab..f6ceccb89 100644 --- a/ggml/examples/sam/sam.cpp +++ b/ggml/examples/sam/sam.cpp @@ -1267,33 +1267,25 @@ struct ggml_cgraph * sam_encode_image( cur = ggml_mul_mat(ctx0, layer.qkv_w, cur); cur = ggml_add_inplace(ctx0, cur, layer.qkv_b); - // split qkv into separate tensors + // qkv is stored as three n_enc_state-wide segments (q, k, v) along dim 0, each + // holding n_enc_head heads of n_enc_head_dim values - slice them as strided views + // instead of materializing the split (same approach as clip.cpp in llama.cpp) // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L225-L229 const int B = cur->ne[3]; - cur = ggml_reshape_4d(ctx0, cur, n_enc_state, 3, W*H, B); - cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 3, 1, 2)); + const size_t nb_head = n_enc_head_dim * cur->nb[0]; // stride between heads inside a segment + const size_t nb_seg = n_enc_state * cur->nb[0]; // stride between the q, k and v segments - struct ggml_tensor * Q; - struct ggml_tensor * K; - struct ggml_tensor * V; + struct ggml_tensor * Q = ggml_view_4d(ctx0, cur, n_enc_head_dim, W*H, n_enc_head, B, cur->nb[1], nb_head, cur->nb[3], 0*nb_seg); + struct ggml_tensor * K = ggml_view_4d(ctx0, cur, n_enc_head_dim, W*H, n_enc_head, B, cur->nb[1], nb_head, cur->nb[3], 1*nb_seg); + struct ggml_tensor * V = ggml_view_4d(ctx0, cur, n_enc_head_dim, W*H, n_enc_head, B, cur->nb[1], nb_head, cur->nb[3], 2*nb_seg); - Q = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 0*cur->nb[3]); - Q = ggml_reshape_4d(ctx0, Q, n_enc_head_dim, n_enc_head, W*H, B); - Q = ggml_cont (ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3)); - Q = ggml_reshape_3d(ctx0, Q, n_enc_head_dim, W*H, B*n_enc_head); + // ggml_mul_mat consumes the strided K directly + struct ggml_tensor * KQ = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, K, Q), W*H, W*H, B*n_enc_head); - K = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 1*cur->nb[3]); - K = ggml_reshape_4d(ctx0, K, n_enc_head_dim, n_enc_head, W*H, B); - K = ggml_cont (ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3)); - K = ggml_reshape_3d(ctx0, K, n_enc_head_dim, W*H, B*n_enc_head); - - V = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 2*cur->nb[3]); - V = ggml_reshape_4d(ctx0, V, n_enc_head_dim, n_enc_head, W*H, B); - V = ggml_cont (ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3)); // transposed - V = ggml_reshape_3d(ctx0, V, W*H, n_enc_head_dim, B*n_enc_head); - - struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + // rel_pos needs a contiguous q; V must be transposed for the second mul_mat + Q = ggml_reshape_3d(ctx0, ggml_cont(ctx0, Q), n_enc_head_dim, W*H, B*n_enc_head); + V = ggml_reshape_3d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 0, 2, 3)), W*H, n_enc_head_dim, B*n_enc_head); struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, @@ -1810,8 +1802,8 @@ bool sam_decode_mask( ggml_build_forward_expand(gf, ggml_cpy(ctx0, out, ggml_view_2d(ctx0, hyper_in, hyper_in->ne[0], hyper_in->ne[2], hyper_in->nb[1], i*hyper_in->nb[1]))); } - struct ggml_tensor * masks = ggml_mul_mat(ctx0, hyper_in, upscaled_embedding); - masks = ggml_cont(ctx0, ggml_transpose(ctx0, masks)); // TODO: Shouldn't be needed + // swapping the operands yields [W*H, num_mask_tokens, batch] directly (no transpose needed) + struct ggml_tensor * masks = ggml_mul_mat(ctx0, upscaled_embedding, hyper_in); masks = ggml_reshape_4d(ctx0, masks, keys->ne[0], keys->ne[1], masks->ne[1], keys->ne[3]); // Generate mask quality predictions diff --git a/ggml/examples/test-cmake/CMakeLists.txt b/ggml/examples/test-cmake/CMakeLists.txt index 713dc7179..aa4b79c54 100644 --- a/ggml/examples/test-cmake/CMakeLists.txt +++ b/ggml/examples/test-cmake/CMakeLists.txt @@ -3,7 +3,7 @@ project(ggml-simple) set(CMAKE_CXX_STANDARD 17) -find_package(ggml 0.19.0 REQUIRED) +find_package(ggml 1.0.0 REQUIRED) set(TEST_TARGET test-cmake) add_executable(test-cmake test-cmake.cpp) diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index 0b51ed14a..324160f0a 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -70,7 +70,8 @@ extern "C" { GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); // tensor copy between different backends - GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy (const struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy_n(const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors); // // Backend (stream) @@ -95,6 +96,8 @@ extern "C" { GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_set_n ( struct ggml_tensor * const * tensors, const void * data, const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors); + GGML_API void ggml_backend_tensor_get_n (const struct ggml_tensor * const * tensors, void * data, const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors); GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); GGML_API void ggml_backend_synchronize(ggml_backend_t backend); @@ -208,6 +211,18 @@ extern "C" { // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + + // Builds the communicator for the case where each rank is a separate process, so a process + // owns one backend of the communicator rather than all of them. + // + // The caller provides bootstrap_callback, which moves bytes between the ranks before the + // communicator exists. Each rank passes size bytes in send_buf, and recv_buf receives + // world_size * size bytes in rank order. The backend decides what to exchange and how many + // times to call it, and every rank must make the same calls in the same order. + // comm_init_rank blocks until all world_size ranks have joined. + typedef bool (*ggml_backend_comm_bootstrap_callback)(void * user_data, const void * send_buf, void * recv_buf, size_t size); + typedef void * (*ggml_backend_comm_init_rank_t)(ggml_backend_t backend, int rank, int world_size, ggml_backend_comm_bootstrap_callback bootstrap_callback, void * bootstrap_callback_data); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); @@ -324,6 +339,23 @@ extern "C" { // typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); + // Callback while copying input weights of a split + // if the user returns false the scheduler simply copies the entire weight + // the callback is called only for input weights in host buffers + // the callback is called after all non-weight inputs of the split have been copied + // `src` is the tensor in the previous split + // `dst` is the copy of `src` in the split + // `graph` is the compute graph of the split + typedef bool (*ggml_backend_sched_copy_callback)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst, struct ggml_cgraph * graph, void * user_data); + + // Callback when the buffers reserved for a graph are not sufficient and have to be reserved again + // it happens on the graph allocation path and costs a synchronization of the backends + // `sched` is the scheduler + // `graph` is the compute graph being allocated + // `backend_ids_changed` is true if the backend assignment of the graph nodes changed since the last allocation + // `graph_size_changed` is true if the graph size changed since the last allocation + typedef void (*ggml_backend_sched_realloc_callback)(ggml_backend_sched_t sched, struct ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data); + // Initialize a backend scheduler, backends with low index are given priority over backends with high index GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload); GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); @@ -362,6 +394,12 @@ extern "C" { // Set a callback to be called for each resulting node during graph compute GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + // Set a callback to be called when the inputs weights of a split are being copied + GGML_API void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data); + + // Set a callback to be called when the compute buffers of a graph are being reallocated + GGML_API void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data); + // // Meta backend // diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index 1f8cb7906..da1d4e1f1 100644 --- a/ggml/include/ggml-rpc.h +++ b/ggml/include/ggml-rpc.h @@ -6,26 +6,31 @@ extern "C" { #endif -#define RPC_PROTO_MAJOR_VERSION 7 +#define RPC_PROTO_MAJOR_VERSION 10 #define RPC_PROTO_MINOR_VERSION 0 #define RPC_PROTO_PATCH_VERSION 0 #ifdef __cplusplus -static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); +static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); #endif #define GGML_RPC_MAX_SERVERS 16 // backend API GGML_BACKEND_API ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device); + GGML_BACKEND_API bool ggml_backend_is_rpc(ggml_backend_t backend); GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, uint32_t device); GGML_BACKEND_API void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total); -GGML_BACKEND_API void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir, - size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices); +GGML_BACKEND_API void ggml_backend_rpc_start_server( + const char * endpoint, + const char * cache_dir, + size_t n_threads, + size_t n_devices, + ggml_backend_dev_t * devices); GGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_reg(void); GGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint); diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 224bdef92..2341fa6c1 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -1,178 +1,5 @@ #pragma once -// -// GGML Tensor Library -// -// This documentation is still a work in progress. -// If you wish some specific topics to be covered, feel free to drop a comment: -// -// https://github.com/ggml-org/whisper.cpp/issues/40 -// -// ## Overview -// -// This library implements: -// -// - a set of tensor operations -// - automatic differentiation -// - basic optimization algorithms -// -// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes, -// but is not limited to, the following: -// -// - linear regression -// - support vector machines -// - neural networks -// -// The library allows the user to define a certain function using the available tensor operations. This function -// definition is represented internally via a computation graph. Each tensor operation in the function definition -// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the -// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized -// using one of the available optimization algorithms. -// -// For example, here we define the function: f(x) = a*x^2 + b -// -// { -// struct ggml_init_params params = { -// .mem_size = 16*1024*1024, -// .mem_buffer = NULL, -// }; -// -// // memory allocation happens here -// struct ggml_context * ctx = ggml_init(params); -// -// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); -// -// ggml_set_param(ctx, x); // x is an input variable -// -// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); -// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); -// struct ggml_tensor * x2 = ggml_mul(ctx, x, x); -// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b); -// -// ... -// } -// -// Notice that the function definition above does not involve any actual computation. The computation is performed only -// when the user explicitly requests it. For example, to compute the function's value at x = 2.0: -// -// { -// ... -// -// struct ggml_cgraph * gf = ggml_new_graph(ctx); -// ggml_build_forward_expand(gf, f); -// -// // set the input variable and parameter values -// ggml_set_f32(x, 2.0f); -// ggml_set_f32(a, 3.0f); -// ggml_set_f32(b, 4.0f); -// -// ggml_graph_compute_with_ctx(ctx, &gf, n_threads); -// -// printf("f = %f\n", ggml_get_f32_1d(f, 0)); -// -// ... -// } -// -// The actual computation is performed in the ggml_graph_compute() function. -// -// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the -// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know -// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory -// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was -// actually needed. -// -// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic -// differentiation and optimization algorithms. -// -// The described approach allows to define the function graph once and then compute its forward or backward graphs -// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way -// the user can avoid the memory allocation overhead at runtime. -// -// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class -// citizens, but in theory the library can be extended to support FP8 and integer data types. -// -// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary -// and binary operations. Most of the available operations fall into one of these two categories. With time, it became -// clear that the library needs to support more complex operations. The way to support these operations is not clear -// yet, but a few examples are demonstrated in the following operations: -// -// - ggml_permute() -// - ggml_conv_1d_1s() -// - ggml_conv_1d_2s() -// -// For each tensor operator, the library implements a forward and backward computation function. The forward function -// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the -// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a -// calculus class, or watch the following video: -// -// What is Automatic Differentiation? -// https://www.youtube.com/watch?v=wG_nF1awSSY -// -// -// ## Tensor data (struct ggml_tensor) -// -// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of -// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains -// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example: -// -// { -// struct ggml_tensor * c = ggml_add(ctx, a, b); -// -// assert(c->src[0] == a); -// assert(c->src[1] == b); -// } -// -// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the -// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows -// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and -// permutation. All tensor operations have to take the stride into account and not assume that the tensor is -// contiguous in memory. -// -// The data of the tensor is accessed via the "data" pointer. For example: -// -// { -// const int nx = 2; -// const int ny = 3; -// -// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny); -// -// for (int y = 0; y < ny; y++) { -// for (int x = 0; x < nx; x++) { -// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y; -// } -// } -// -// ... -// } -// -// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used. -// -// ## The matrix multiplication operator (ggml_mul_mat) -// -// TODO -// -// -// ## Multi-threading -// -// TODO -// -// -// ## Overview of ggml.c -// -// TODO -// -// -// ## SIMD optimizations -// -// TODO -// -// -// ## Debugging ggml -// -// TODO -// -// - #ifdef GGML_SHARED # if defined(_WIN32) && !defined(__MINGW32__) # ifdef GGML_BUILD @@ -381,6 +208,8 @@ extern "C" { GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t); GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t); + typedef struct { uint8_t bits; } ggml_fp8_e4m3_t; + struct ggml_object; struct ggml_context; struct ggml_cgraph; @@ -430,7 +259,8 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, - GGML_TYPE_COUNT = 43, + GGML_TYPE_F8_E4M3 = 43, + GGML_TYPE_COUNT = 44, }; // [TAG_GGML_PREC] @@ -442,7 +272,6 @@ extern "C" { // the precision parameters are stored as ggml_tensor.op_params to the respective ops enum ggml_prec { GGML_PREC_UNDEFINED = 0, - GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED GGML_PREC_F32 = 10, GGML_PREC_BF16 = 15, GGML_PREC_F16 = 20, @@ -486,6 +315,7 @@ extern "C" { GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors + GGML_FTYPE_MOSTLY_F8_E4M3 = 29, // except 1d tensors }; // available tensor operations: @@ -495,7 +325,6 @@ extern "C" { GGML_OP_DUP, GGML_OP_ADD, GGML_OP_ADD_ID, - GGML_OP_ADD1, GGML_OP_ACC, GGML_OP_SUB, GGML_OP_MUL, @@ -537,8 +366,6 @@ extern "C" { GGML_OP_GET_ROWS_BACK, GGML_OP_SET_ROWS, GGML_OP_DIAG, - GGML_OP_DIAG_MASK_INF, - GGML_OP_DIAG_MASK_ZERO, GGML_OP_SOFT_MAX, GGML_OP_SOFT_MAX_BACK, GGML_OP_ROPE, @@ -585,6 +412,7 @@ extern "C" { GGML_OP_DSV4_HC_COMB, GGML_OP_DSV4_HC_PRE, GGML_OP_DSV4_HC_POST, + GGML_OP_TOP_K_MASK, GGML_OP_UNARY, @@ -760,10 +588,6 @@ extern "C" { GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row - GGML_DEPRECATED( - GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float - "use ggml_row_size() instead"); - GGML_API const char * ggml_type_name(enum ggml_type type); GGML_API const char * ggml_op_name (enum ggml_op op); GGML_API const char * ggml_op_symbol(enum ggml_op op); @@ -925,24 +749,13 @@ extern "C" { enum ggml_type type); // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]] + // an id of -1 adds nothing: dst[i0, i1, i2] = a[i0, i1, i2] GGML_API struct ggml_tensor * ggml_add_id( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, struct ggml_tensor * ids); - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b), - "use ggml_add instead"); - - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b), - "use ggml_add_inplace instead"); - // dst = a // view(dst, nb1, nb2, nb3, offset) += b // return dst @@ -1476,6 +1289,16 @@ extern "C" { enum ggml_prec prec, int idx); + // [TAG_GGML_PREC] + // get the accumulator / src[idx] precision set by the functions above + // return GGML_PREC_UNDEFINED when the op does not carry the requested precision + GGML_API enum ggml_prec ggml_prec_get_acc( + const struct ggml_tensor * a); + + GGML_API enum ggml_prec ggml_prec_get_src( + const struct ggml_tensor * a, + int idx); + // A: k columns, n rows => [ne03, ne02, n, k] // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] @@ -1484,19 +1307,19 @@ extern "C" { struct ggml_tensor * a, struct ggml_tensor * b); - // change the precision of a matrix multiplication - // set to GGML_PREC_F32 for higher precision (useful for phi-2) - GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec( - struct ggml_tensor * a, - enum ggml_prec prec), - "use ggml_prec_set_acc() instead"); - // change the hint of a matrix multiplication GGML_API void ggml_mul_mat_set_hint( struct ggml_tensor * a, enum ggml_op_hint hint); // indirect matrix multiplication + // an id of -1 skips the slot and zeroes the matching row of the result + // + // [TAG_MUL_MAT_ID_DUPLICATE_ID] + // note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b` + // (i.e. duplicate experts per token are not guaranteed to be supported in all backends) + // TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533 + // the CPU backend skips negative ids, their rows of the result are zero GGML_API struct ggml_tensor * ggml_mul_mat_id( struct ggml_context * ctx, struct ggml_tensor * as, @@ -1755,30 +1578,6 @@ extern "C" { struct ggml_context * ctx, struct ggml_tensor * a); - // set elements above the diagonal to -INF - GGML_API struct ggml_tensor * ggml_diag_mask_inf( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past); - - // in-place, returns view(a) - GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past); - - // set elements above the diagonal to 0 - GGML_API struct ggml_tensor * ggml_diag_mask_zero( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past); - - // in-place, returns view(a) - GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past); - GGML_API struct ggml_tensor * ggml_clamp( struct ggml_context * ctx, struct ggml_tensor * a, @@ -1982,40 +1781,6 @@ extern "C" { float beta_fast, float beta_slow); - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - int mode, - int n_ctx_orig, - float freq_base, - float freq_scale, - float ext_factor, - float attn_factor, - float beta_fast, - float beta_slow), - "use ggml_rope_ext instead"); - - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - int mode, - int n_ctx_orig, - float freq_base, - float freq_scale, - float ext_factor, - float attn_factor, - float beta_fast, - float beta_slow), - "use ggml_rope_ext_inplace instead"); - - // compute correction dims for YaRN RoPE scaling - GGML_API void ggml_rope_yarn_corr_dims( - int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]); - // rotary position embedding backward, i.e compute dx from dy // a - dy GGML_API struct ggml_tensor * ggml_rope_ext_back( @@ -2058,6 +1823,11 @@ extern "C" { struct ggml_tensor * a, int n_offs); + // control rounding of YaRN correction bounds (default: true) + GGML_API struct ggml_tensor * ggml_rope_set_truncate( + struct ggml_tensor * a, + bool truncate); + // im2col // converts data into a format that effectively results in a convolution when combined with matrix multiplication GGML_API struct ggml_tensor * ggml_im2col( @@ -2316,7 +2086,7 @@ extern "C" { GGML_SCALE_MODE_BILINEAR = 1, GGML_SCALE_MODE_BICUBIC = 2, - GGML_SCALE_MODE_COUNT + GGML_SCALE_MODE_COUNT = 3 }; enum ggml_scale_flag { @@ -2332,18 +2102,6 @@ extern "C" { int scale_factor, enum ggml_scale_mode mode); - // interpolate - // interpolate scale to specified dimensions - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext( - struct ggml_context * ctx, - struct ggml_tensor * a, - int ne0, - int ne1, - int ne2, - int ne3, - enum ggml_scale_mode mode), - "use ggml_interpolate instead"); - // Up- or downsamples the input to the specified size. // 2D scale modes (eg. bilinear) are applied to the first two dimensions. GGML_API struct ggml_tensor * ggml_interpolate( @@ -2494,10 +2252,23 @@ extern "C" { float max_bias, float logit_softcap); - GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec( - struct ggml_tensor * a, - enum ggml_prec prec), - "use ggml_prec_set_acc() instead"); + // same as ggml_flash_attn_ext, but each q slice attends to its own list of K/V rows + // q: [n_embd_k, n_batch, n_head, ne3] + // k: [n_embd_k, n_rows, n_head_kv, 1 ] shared by all q slices + // v: [n_embd_v, n_rows, n_head_kv, 1 ] + // mask: [n_kv, n_batch, ne32, ne3] + // kv_rows: [n_kv, ne3] I32, mask column i of slice i3 uses row kv_rows[i, i3] of k and v + // negative rows are padding, their mask entries must be -inf + GGML_API struct ggml_tensor * ggml_flash_attn_ext_rows( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + struct ggml_tensor * kv_rows, + float scale, + float max_bias, + float logit_softcap); GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( const struct ggml_tensor * a); @@ -2649,6 +2420,7 @@ extern "C" { // the output packs the attention scores [S_v, H_v, n_tokens, n_seqs] followed by K state // snapshots, most-recent first (slot 0 = final state, slot s = state s tokens back). K == 1 // keeps only the final state; when n_tokens < K only slots 0..n_tokens-1 are written. + // when n_tokens > K only slot 0 is guaranteed; backends may leave slots 1..K-1 unwritten. GGML_API struct ggml_tensor * ggml_gated_delta_net( struct ggml_context * ctx, struct ggml_tensor * q, @@ -2726,6 +2498,26 @@ extern "C" { struct ggml_tensor * post, struct ggml_tensor * comb); + // top-k sparse attention mask: keep the mask cells of the live picks and the extra cells, set the rest to -inf + // + // mask: [n_kv, m1, m2, m3] f16 or f32, broadcast to the rows of top_k + // top_k: [n_top, ne1, ne2, ne3] i32, picks of each row + // score: [n_pick, ne1, ne2, ne3] f32, can be NULL: a pick is live when its score is above -inf, else every pick is live + // blocks: [n_blk, n_pick] i32, can be NULL: the member cells of each pick, else a pick is a cell + // extra: [n_extra, ne1, ne2, ne3] i32, can be NULL: cells of each row kept without a pick + // res: [n_kv, ne1, ne2, ne3] same type as mask + // + // res[c, i] = mask[c, i % m] when c is a cell of a live pick of row i or an extra cell of row i, else -inf + // picks and cells out of range are skipped + // + GGML_API struct ggml_tensor * ggml_top_k_mask( + struct ggml_context * ctx, + struct ggml_tensor * mask, + struct ggml_tensor * top_k, + struct ggml_tensor * score, + struct ggml_tensor * blocks, + struct ggml_tensor * extra); + // custom operators typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata); diff --git a/ggml/requirements.txt b/ggml/requirements.txt index 4151ef245..096dfe256 100644 --- a/ggml/requirements.txt +++ b/ggml/requirements.txt @@ -1,5 +1,6 @@ accelerate==0.19.0 -numpy~=2.2.6 +numpy~=2.2.6; python_version < "3.14" +numpy~=2.4.6; python_version >= "3.14" sentencepiece>=0.1.98,<0.3.0 torchvision~=0.26.0 transformers==5.5.1 diff --git a/ggml/scripts/check-apiabi-compat.sh b/ggml/scripts/check-apiabi-compat.sh new file mode 100755 index 000000000..229602f44 --- /dev/null +++ b/ggml/scripts/check-apiabi-compat.sh @@ -0,0 +1,292 @@ +#!/bin/sh +# Check for backwards-incompatible API and ABI changes between two builds +# +# Backwards-incompatible API changes, such as removing a value from an enum, +# are checked by abi-compliance-checker. Such changes can break compilation of +# existing programs. +# +# Backwards-incompatible ABI changes, such as the removal of a public function, +# are checked by libigail-tools. Such changes could break run-time dynamic +# linking of existing binaries. (We don't use a-c-c for ABI checks because it +# needs a debug build, whereas abigail does not.) +# +# Commands: +# --generate <build-dir>: Creates API/ABI dumps in <build-dir> +# <build-dir> is expected to be a CMake build result +# --check <dir1> <dir2>: Compares dumps in <dir1> and <dir2> +# Comparison exit codes +# 0: all good +# 1: backwards-incompatible changes found +# +# Options: +# --include-path <dir>: a-c-c calls gcc on headers; use this option to add +# directories to gcc's search path +# +# +# This script would typically be used before cutting a release: +# +# 1. Generate API/ABI dump for the old version +# +# $ check-apiabi-compat.sh --generate <build-dir-old> libfoo [ libbar ...] +# +# 2. <update source> +# +# 3. Generate API/ABI dump for the new version +# +# $ check-apiabi-compat.sh --generate <build-dir-new> libfoo [ libbar ...] +# +# 4. Compare the two dumps +# +# $ check-apiabi-compat.sh --check <old-build-dir> <new-build-dir> +# +# If the check exits 0, all is fine. Otherwise, backwards-incompatible +# changes were found, and the librar(ies) need a SOVER bump. +set -eu + +# Preconditions +if ! command -v abi-compliance-checker >/dev/null 2>&1; then + echo "abi-compliance-checker is not installed." >&2 + exit 1 +elif ! command -v abidw >/dev/null 2>&1; then + echo "abigail-tools are not installed." >&2 + exit 1 +fi + +# Some generic functions +usage() { + echo "Usage: $0 [ --include-path <dir> ] --generate <build-dir> libXXX [ libYYY ... ]" >&2 + echo " $0 [ --strict ] --check <old-build-dir> <new-build-dir>" >&2 + echo "" >&2 + echo " --strict: fail on any API/ABI change, including backwards-compatible additions" >&2 +} + +get_cmake_project_name() { + sed -nr 's/^project\("(.*)".*$/\1/p' CMakeLists.txt +} + +get_cmake_version() { + major="$(sed -nr 's/^set\([A-Z]+_VERSION_MAJOR ([0-9]+)\)$/\1/p' CMakeLists.txt)" + minor="$(sed -nr 's/^set\([A-Z]+_VERSION_MINOR ([0-9]+)\)$/\1/p' CMakeLists.txt)" + patch="$(sed -nr 's/^set\([A-Z]+_VERSION_PATCH ([0-9]+)\)$/\1/p' CMakeLists.txt)" + echo "$major.$minor.$patch" +} + +# Option parsing and validation +DO_GEN=0 +DO_CHECK=0 +STRICT=0 +BUILD_DIR= +BUILD_DIR_NEW= +INCLUDE_PATHS= +while [ "$#" -gt 0 ]; do + case "$1" in + --generate=*) + DO_GEN=1 + BUILD_DIR="${1#*=}" + shift + ;; + --generate) + DO_GEN=1 + if [ -z "${2:-}" ]; then + usage + exit 1 + fi + BUILD_DIR="$2" + shift 2 + ;; + --check) + DO_CHECK=1 + if [ -z "${2:-}" ] || [ -z "${3:-}" ]; then + usage + exit 1 + elif ! [ -d "$2" ]; then + echo "$2 is not a directory." >&2 + exit 1 + elif ! [ -d "$3" ]; then + echo "$3 is not a directory." >&2 + exit 1 + fi + BUILD_DIR="$2" + BUILD_DIR_NEW="$3" + shift 3 + ;; + --include-path=*) + INCLUDE_PATHS="$INCLUDE_PATHS ${1#*=}" + shift + ;; + --include-path) + if [ -z "${2:-}" ]; then + usage + exit 1 + fi + INCLUDE_PATHS="$INCLUDE_PATHS $2" + shift 2 + ;; + --strict) + STRICT=1 + shift + ;; + -h | --help) + usage + exit 1 + ;; + -?*) + usage + exit 1 + ;; + *) + break + ;; + esac +done +if [ $((DO_GEN + DO_CHECK)) -gt 1 ]; then + echo "Can only use one --command." >&2 + exit +fi +PROJECT_NAME="$(get_cmake_project_name)" +PROJECT_VERSION="$(get_cmake_version)" +LIB_NAMES="" +while [ "$#" -gt 0 ]; do + if [ "${1#lib}" = "$1" ]; then + echo "Library to check must start with libXXX." >&2 + exit 1 + fi + LIB_NAMES="$LIB_NAMES $1" + shift +done + +dump_current_api() { + echo "Dumping API..." + + DESCRIPTOR="$BUILD_DIR/apiabi/acc-descriptor.xml" + mkdir -p "$BUILD_DIR/apiabi" + cat >"$DESCRIPTOR" <<EOF +<version>$PROJECT_VERSION</version> +<headers>include</headers> +<add_include_paths>$INCLUDE_PATHS</add_include_paths> +EOF + + # This addresses a bug between a-c-c and universal-ctags, manifested when + # a name is use both for a tag and a function name + mkdir -p "$BUILD_DIR/apiabi/.ctags.d" + echo "--fields=-t" >"$BUILD_DIR/apiabi/.ctags.d/acc.ctags" + + # Change HOME so that .ctags.d gets picked up by universal-ctags, if used + HOME="$BUILD_DIR/apiabi" abi-compliance-checker \ + -headers-only \ + -lib "$PROJECT_NAME" \ + -dump "$DESCRIPTOR" \ + -log-path "$BUILD_DIR/apiabi/acc.log" \ + -dump-path "$BUILD_DIR/apiabi/api.dump" + # acc generates this file with an ancient timestamp, which confuses gzip + touch "$BUILD_DIR/apiabi/api.dump" +} + +dump_current_abi() { + echo "Dumping ABIs ..." + mkdir -p "$BUILD_DIR/apiabi" + # The suppressions are needed to avoid including all the internal C++ + # symbols, and system types + cat >"$BUILD_DIR/apiabi/abidw.suppress" <<EOF +[suppress_function] + +label = suppress internal C++ mangled functions +symbol_name_regexp = ^_Z +drop = yes + +[suppress_variable] +label = suppress internal C++ mangled variables +symbol_name_regexp = ^_Z +drop = yes + +[suppress_type] +label = Suppress types outside of our own source +source_location_not_regexp = ^include/ +drop = yes +EOF + + # In abidw 2.5, handling of undefined stuff was changed a bit + abidw_version="$(abidw --version | sed -r 's/^abidw: ([0-9]+\.[0-9]+).*$/\1/')" + abidw_major="${abidw_version%.*}" + abidw_minor="${abidw_version#*.}" + if [ "$abidw_major" -gt 2 ] || [ "$abidw_minor" -gt 4 ]; then + abidw_undefined_syms_options="--no-load-undefined-interfaces" + else + abidw_undefined_syms_options="--drop-undefined-syms" + fi + + for lib_name in $LIB_NAMES; do + # Depending on where add_library resides, the libraries can end up in + # build/src or build/bin + lib_path="$BUILD_DIR/src/$lib_name.so" + if ! [ -f "$lib_path" ]; then + lib_path="$BUILD_DIR/bin/$lib_name.so" + if ! [ -f "$lib_path" ]; then + echo "Cannot find library $lib_name.so" >&2 + exit 1 + fi + fi + abidw \ + --headers-dir include \ + "$abidw_undefined_syms_options" \ + --suppressions "$BUILD_DIR/apiabi/abidw.suppress" \ + --out-file "${BUILD_DIR}/apiabi/$lib_name.abi.xml" \ + "$lib_path" + done +} + +# Run the actual commands +if [ "$DO_GEN" -eq 1 ]; then + dump_current_api + dump_current_abi + exit 0 +elif [ "$DO_CHECK" -eq 1 ]; then + # From here on, we don't want to exit on first error + set +e + + ACC_STRICT= + [ "$STRICT" -eq 0 ] || ACC_STRICT="-strict" + abi-compliance-checker \ + $ACC_STRICT \ + -source \ + -library "$PROJECT_NAME" \ + -old "$BUILD_DIR/apiabi/api.dump" \ + -new "$BUILD_DIR_NEW/apiabi/api.dump" \ + -src-report-path "$BUILD_DIR_NEW/apiabi/api_compat_report.html" + API_RESULT=$? + + ABI_RESULT=0 + for xml_file in "$BUILD_DIR/apiabi/"lib*.abi.xml; do + xml_file_new="$BUILD_DIR_NEW/apiabi/$(basename "$xml_file")" + + if ! [ -f "$xml_file_new" ]; then + echo "Cannot compare, missing file: $xml_file_new" >&2 + exit 1 + fi + + abidiff "$xml_file" "$xml_file_new" + res=$? + [ "$((res & 8))" -ne 0 ] && ABI_RESULT=1 + # check bit 2 for compatible ABI changes (like new symbols) and if + # STRICT is set then handle this as an error. + [ "$STRICT" -eq 1 ] && [ "$((res & 4))" -ne 0 ] && ABI_RESULT=1 + done + + if [ "$API_RESULT" -gt 0 ]; then + if [ "$STRICT" -eq 1 ]; then + echo "ERROR: API changed — a minor version bump is required." >&2 + else + echo "ERROR: API changed with backwards-incompatible problems — a major version bump is required." >&2 + fi + fi + if [ "$ABI_RESULT" -gt 0 ]; then + if [ "$STRICT" -eq 1 ]; then + echo "ERROR: ABI changed — a minor version bump is required." >&2 + else + echo "ERROR: ABI changed with backwards-incompatible problems — a major version bump is required." >&2 + fi + fi + if [ "$((API_RESULT + ABI_RESULT))" -gt 0 ]; then + exit 1 + fi +fi diff --git a/ggml/scripts/check-release-apiabi.sh b/ggml/scripts/check-release-apiabi.sh new file mode 100755 index 000000000..1bcdfee26 --- /dev/null +++ b/ggml/scripts/check-release-apiabi.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# Check API/ABI compatibility between the previous release tag and current HEAD. +# +# Finds the most recent vX.Y.Z tag, checks it out in a temporary git worktree, +# builds both versions with shared libs enabled, and uses check-apiabi-compat.sh +# to compare the results. +# +# Exit codes: +# 0: compatible, or check was skipped +# 1: backwards-incompatible changes found, or build failed +# +# Options: +# --tag <version>: compare against this tag instead of the latest release +# +# Environment: +# SKIP_APIABI_CHECK: set to 1 or true to skip +# APIABI_COMPARE_TAG: equivalent to --tag (used by CI) +# APIABI_REPORT_DIR: preserve reports and dumps in this directory, even if the comparison fails + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +LIBS=(libggml libggml-base) + +usage() { + echo "Usage: $0 [--tag <version>]" >&2 + echo " --tag <version> Compare against this release tag (default: latest)" >&2 +} + +COMPARE_TAG="${APIABI_COMPARE_TAG:-}" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --tag) + if [[ -z "${2:-}" ]]; then usage; exit 1; fi + COMPARE_TAG="$2" + shift 2 + ;; + --tag=*) + COMPARE_TAG="${1#*=}" + shift + ;; + -h | --help) + usage; exit 0 + ;; + *) + usage; exit 1 + ;; + esac +done + +if [[ "${SKIP_APIABI_CHECK:-}" == "1" || "${SKIP_APIABI_CHECK:-}" == "true" ]]; then + echo "SKIP_APIABI_CHECK is set - skipping API/ABI compatibility check" + exit 0 +fi + +if ! command -v abi-compliance-checker >/dev/null 2>&1 || ! command -v abidw >/dev/null 2>&1; then + echo "Warning: abi-compliance-checker or abigail-tools not installed - skipping API/ABI check" + exit 0 +fi + +if [[ -n "${COMPARE_TAG}" ]]; then + PREV_TAG="${COMPARE_TAG}" + if ! git -C "$REPO_ROOT" rev-parse --verify "${PREV_TAG}^{}" >/dev/null 2>&1; then + echo "Error: tag '${PREV_TAG}' not found in repository." >&2 + exit 1 + fi +else + PREV_TAG=$(git -C "$REPO_ROOT" tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1 || true) + if [[ -z "${PREV_TAG}" ]]; then + echo "Warning: no previous release tag found - skipping API/ABI check" + exit 0 + fi +fi +OLD_VERSION="${PREV_TAG#v}" +OLD_MAJOR="${OLD_VERSION%%.*}" +OLD_MINOR="${OLD_VERSION#*.}"; OLD_MINOR="${OLD_MINOR%%.*}" + +NEW_MAJOR=$(grep "set(GGML_VERSION_MAJOR" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*MAJOR \([0-9]*\).*/\1/') +NEW_MINOR=$(grep "set(GGML_VERSION_MINOR" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*MINOR \([0-9]*\).*/\1/') + +if [[ "$NEW_MAJOR" -gt "$OLD_MAJOR" ]]; then + echo "Major version increment ($OLD_MAJOR -> $NEW_MAJOR): API/ABI breaking changes are expected, skipping compatibility check." + exit 0 +fi + +CHECK_FLAGS=() +if [[ "$NEW_MINOR" -eq "$OLD_MINOR" ]]; then + echo "Patch version bump detected: checking for any API/ABI changes (a minor bump is required if any are found)..." + CHECK_FLAGS+=(--strict) +else + echo "Minor version bump detected: checking for backwards-incompatible API/ABI changes..." +fi + +echo "Checking API/ABI compatibility against ${PREV_TAG}..." + +WORKTREE_DIR=$(mktemp -d) +BUILD_OLD=$(mktemp -d) +BUILD_NEW=$(mktemp -d) + +cleanup() { + git -C "$REPO_ROOT" worktree remove --force "$WORKTREE_DIR" 2>/dev/null || true + rm -rf "$WORKTREE_DIR" "$BUILD_OLD" "$BUILD_NEW" +} +trap cleanup EXIT + +git -C "$REPO_ROOT" worktree add "$WORKTREE_DIR" "$PREV_TAG" + +cmake -S "$WORKTREE_DIR" -B "$BUILD_OLD" -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +cmake --build "$BUILD_OLD" --parallel "$(nproc)" + +cmake -S "$REPO_ROOT" -B "$BUILD_NEW" -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +cmake --build "$BUILD_NEW" --parallel "$(nproc)" + +(cd "$WORKTREE_DIR" && "$SCRIPT_DIR/check-apiabi-compat.sh" --generate "$BUILD_OLD" "${LIBS[@]}") +(cd "$REPO_ROOT" && "$SCRIPT_DIR/check-apiabi-compat.sh" --generate "$BUILD_NEW" "${LIBS[@]}") + +CHECK_RESULT=0 +(cd "$REPO_ROOT" && "$SCRIPT_DIR/check-apiabi-compat.sh" "${CHECK_FLAGS[@]}" --check "$BUILD_OLD" "$BUILD_NEW") || CHECK_RESULT=$? + +if [[ -n "${APIABI_REPORT_DIR:-}" ]]; then + mkdir -p "$APIABI_REPORT_DIR/old" "$APIABI_REPORT_DIR/new" + cp -R "$BUILD_OLD/apiabi/." "$APIABI_REPORT_DIR/old/" + cp -R "$BUILD_NEW/apiabi/." "$APIABI_REPORT_DIR/new/" + echo "API/ABI reports saved to $APIABI_REPORT_DIR" +fi + +exit "$CHECK_RESULT" diff --git a/ggml/scripts/ci-status.sh b/ggml/scripts/ci-status.sh new file mode 100755 index 000000000..65d8eb5bb --- /dev/null +++ b/ggml/scripts/ci-status.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# List GitHub Actions workflow runs that are not finished yet, grouped by status. + +set -euo pipefail + +usage() { + echo "usage: $(basename "$0") [-r owner/repo] [-s status]... [-b branch] [-w regex] [-t regex] [-f regex] [--kill [-y]]" + echo " -r, --repo repository (default: current)" + echo " -s, --status run status to include, repeatable (default: the not-finished ones)" + echo " -b, --branch branch to fetch runs for; exact match, handled by the API" + echo " -w, --workflow case-insensitive regex matched against the workflow name (client-side)" + echo " -t, --title case-insensitive regex matched against the run title (client-side)" + echo " -f, --filter case-insensitive regex matched against the branch name (client-side)" + echo " --kill cancel the selected runs (requires -b, -w, -t or -f)" + echo " -y, --yes skip the confirmation prompt of --kill" +} + +repo="" +branch="" +workflow="" +title="" +filter="" +kill=false +assume_yes=false +statuses=() +need_value() { echo "missing value for $1" >&2; exit 1; } +while [[ $# -gt 0 ]]; do + case "$1" in + -r|--repo) [[ $# -ge 2 ]] || need_value "$1"; repo="$2"; shift 2 ;; + -s|--status) [[ $# -ge 2 ]] || need_value "$1"; statuses+=("$2"); shift 2 ;; + -b|--branch) [[ $# -ge 2 ]] || need_value "$1"; branch="$2"; shift 2 ;; + -w|--workflow) [[ $# -ge 2 ]] || need_value "$1"; workflow="$2"; shift 2 ;; + -t|--title) [[ $# -ge 2 ]] || need_value "$1"; title="$2"; shift 2 ;; + -f|--filter) [[ $# -ge 2 ]] || need_value "$1"; filter="$2"; shift 2 ;; + --kill) kill=true; shift ;; + -y|--yes) assume_yes=true; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 1 ;; + esac +done + +# one request per status: the API does not accept multiple statuses at once +[[ ${#statuses[@]} -gt 0 ]] || statuses=(in_progress queued pending waiting requested) + +[[ -n "$repo" ]] || repo=$(gh repo view --json nameWithOwner -q .nameWithOwner) + +query=("repos/$repo/actions/runs" -f per_page=100) +[[ -z "$branch" ]] || query+=(-f branch="$branch") + +selected=$( + for status in "${statuses[@]}"; do + gh api --paginate --method GET "${query[@]}" -f status="$status" -q '.workflow_runs[]' + done | jq -sr --arg branch "$branch" --arg workflow "$workflow" --arg title "$title" --arg filter "$filter" ' + unique_by(.id) + | map( + .age = ((now - (.run_started_at // .created_at | fromdate)) / 60 | floor) + | select( + ($branch == "" or .head_branch == $branch) and + ($workflow == "" or (.name | test($workflow; "i"))) and + ($title == "" or (.display_title | test($title; "i"))) and + ($filter == "" or (.head_branch | test($filter; "i"))) + ) + ) + | sort_by(.status, -.age)' +) + +tsv=$(jq -r '.[] | [ .status, (.age | tostring) + "m", .name, .head_branch, .display_title, .html_url ] | @tsv' <<< "$selected") + +if [[ -z "$tsv" ]]; then + echo "no active runs in $repo" + exit 0 +fi + +printf 'STATUS\tAGE\tWORKFLOW\tBRANCH\tTITLE\tURL\n%s\n' "$tsv" | column -t -s $'\t' +echo +printf '%s:' "$repo" +for status in "${statuses[@]}"; do + printf ' %s=%s' "$status" "$(awk -F'\t' -v s="$status" '$1 == s' <<< "$tsv" | wc -l | tr -d ' ')" +done +echo + +if [[ "$kill" == false ]]; then + exit 0 +fi + +if [[ -z "$branch$workflow$title$filter" ]]; then + echo "refusing to cancel everything: pass -b, -w, -t or -f to select the runs to cancel" >&2 + exit 1 +fi + +count=$(wc -l <<< "$tsv" | tr -d ' ') +if [[ "$assume_yes" == false ]]; then + printf 'cancel %s run(s)? [y/N] ' "$count" + read -r reply </dev/tty || { echo; echo "aborted: no terminal to confirm on" >&2; exit 1; } + [[ "$reply" == [yY]* ]] || { echo "aborted"; exit 1; } +fi + +jq -r '.[] | [ .id, .status ] | @tsv' <<< "$selected" | +while IFS=$'\t' read -r id status; do + gh run cancel "$id" --repo "$repo" || echo "failed to cancel run $id (status: $status)" +done diff --git a/ggml/scripts/make-release-checks.sh b/ggml/scripts/make-release-checks.sh index c18cd998b..6313adca7 100755 --- a/ggml/scripts/make-release-checks.sh +++ b/ggml/scripts/make-release-checks.sh @@ -91,6 +91,23 @@ else fi fi +echo "Checking API/ABI compatibility..." +set +e +bash "$SCRIPT_DIR/check-release-apiabi.sh" +APIABI_RESULT=$? +set -e +if [[ $APIABI_RESULT -ne 0 ]]; then + if [[ "$DRY_RUN" == "true" ]]; then + echo "Warning: API/ABI check found backwards-incompatible changes (dry run, continuing)." + CHECKS_PASSED=false + else + echo "Error: API/ABI check found backwards-incompatible changes." + exit 1 + fi +else + echo "API/ABI compatibility check passed - OK" +fi + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then echo "checks_passed=${CHECKS_PASSED}" >> "$GITHUB_OUTPUT" fi diff --git a/ggml/scripts/make-release-summary.txt b/ggml/scripts/make-release-summary.txt index 8db8545e4..a0721e3ed 100644 --- a/ggml/scripts/make-release-summary.txt +++ b/ggml/scripts/make-release-summary.txt @@ -26,10 +26,12 @@ For all changes, you can refer to the `llama.cpp` and `whisper.cpp` repositories Guidelines: - All bullet point in the summary should be concise and rarely exceed a single line of 120 characters (excluding PR links) +- Do not wrap or break lines within a bullet point or paragraph - each bullet point must fit on a single line (PR links allowed to exceed the line limit) - Avoid mentioning that the changes are synchronized - this is not relevant - Avoid mentioning specific models - focus on generality of the ggml framework - Replace `(llama/XXXX)` and `(whisper/XXXX)` with a link to the respective PR -- Combine related topics (a full list of commits will be appended independently at end of the summary) +- Group similar/related items into a single bullet with multiple trailing PR links, one bullet per topic + - A full list of commits will be appended independently at end of the summary, so it is not a problem to skip some less important changes - Skip minor-impact notes (e.g. "fix compile warnings", "refactored code", ...) Output just the summary in a markdown block, without any extra text. Save it to a local text file called `release-notes-vX.Y.Z.txt`. diff --git a/ggml/scripts/sync-llama.last b/ggml/scripts/sync-llama.last index c772be82e..16fd1e979 100644 --- a/ggml/scripts/sync-llama.last +++ b/ggml/scripts/sync-llama.last @@ -1 +1 @@ -e117148a41d8e9bedb72e4c6c3f003ab0fe7f857 +5676dfbe01e7ebe0387913680eee0ed39bf4a363 diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 947732000..5ff78f240 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -131,15 +131,9 @@ endif() # POSIX conformance # -# clock_gettime came in POSIX.1b (1993) -# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional -# posix_memalign came in POSIX.1-2001 / SUSv3 -# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985) - -# Somehow in OpenBSD whenever POSIX conformance is specified -# some string functions rely on locale_t availability, -# which was introduced in POSIX.1-2008, forcing us to go higher -if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD") +# POSIX.1-2001 with XSI extensions provides clock_gettime, CLOCK_MONOTONIC, posix_memalign and M_PI. +# OpenBSD needs POSIX.1-2008 for locale_t. FreeBSD needs it for F_DUPFD_CLOEXEC and strnlen. +if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD|FreeBSD") add_compile_definitions(_XOPEN_SOURCE=700) elseif (CMAKE_SYSTEM_NAME MATCHES "AIX") # Don't define _XOPEN_SOURCE. We need _ALL_SOURCE, which is the default, @@ -216,6 +210,12 @@ set_target_properties(ggml-base PROPERTIES configure_file(ggml-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.h @ONLY) target_include_directories(ggml-base PRIVATE . ${CMAKE_CURRENT_BINARY_DIR}) + +# icx/icpx default to -fp-model=fast, which contracts and reassociates the reference quantizers, so their output +# depends on the compiler; keep ggml-base IEEE-exact so the CPU reference matches gcc/clang builds +if (CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + target_compile_options(ggml-base PRIVATE -fp-model=precise) +endif() if (GGML_BACKEND_DL) target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL) endif() @@ -462,6 +462,8 @@ function(ggml_add_cpu_backend_variant tag_name) set(GGML_INTERNAL_${feat} ON) endforeach() elseif (GGML_SYSTEM_ARCH STREQUAL "s390x") + set(GGML_NATIVE OFF) + foreach (feat VXE2 NNPA) set(GGML_INTERNAL_${feat} OFF) endforeach() @@ -569,6 +571,14 @@ if (GGML_CPU_ALL_VARIANTS) if (CMAKE_SYSTEM_NAME MATCHES "Linux") ggml_add_cpu_backend_variant(z15 Z15 VXE2) ggml_add_cpu_backend_variant(z16 Z16 VXE2 NNPA) + + # check if compiler supports "-march=z17" codename + check_cxx_compiler_flag("-march=arch15" GGML_CXX_SUPPORTS_Z17) + if (GGML_CXX_SUPPORTS_Z17) + ggml_add_cpu_backend_variant(arch15 Z17 VXE2 NNPA) + else() + message(WARNING "Skipping z17 target: compiler must be GCC 15.1 and later") + endif() else() message(FATAL_ERROR "Unsupported s390x target OS: ${CMAKE_SYSTEM_NAME}") endif() @@ -627,7 +637,11 @@ endif() if (BUILD_SHARED_LIBS) foreach (target ggml-base ggml) - set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON) + set_target_properties(${target} PROPERTIES + POSITION_INDEPENDENT_CODE ON + C_VISIBILITY_PRESET hidden + CXX_VISIBILITY_PRESET hidden + ) target_compile_definitions(${target} PRIVATE GGML_BUILD) target_compile_definitions(${target} PUBLIC GGML_SHARED) endforeach() diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c index b0c067b49..65d2bea27 100644 --- a/ggml/src/ggml-alloc.c +++ b/ggml/src/ggml-alloc.c @@ -23,11 +23,8 @@ bool ggml_op_can_inplace(enum ggml_op op) { switch (op) { case GGML_OP_FILL: case GGML_OP_SCALE: - case GGML_OP_DIAG_MASK_ZERO: - case GGML_OP_DIAG_MASK_INF: case GGML_OP_ADD: case GGML_OP_ADD_ID: - case GGML_OP_ADD1: case GGML_OP_SUB: case GGML_OP_MUL: case GGML_OP_DIV: diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index dc7367dec..545880b06 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -47,6 +47,9 @@ extern "C" { // Backend buffer // + // TODO: remove `set_tensor`, `get_tensor`, `set_tensor_2d`, `get_tensor_2d` and `cpy_tensor` from the buffer interface + // they are superseded by the `set_tensor_n`, `get_tensor_n`, `cpy_tensor_n` interface + // ref: https://github.com/ggml-org/llama.cpp/pull/30881#discussion_r4400299490 struct ggml_backend_buffer_i { // (optional) free the buffer void (*free_buffer) (ggml_backend_buffer_t buffer); @@ -68,6 +71,11 @@ extern "C" { void (*clear) (ggml_backend_buffer_t buffer, uint8_t value); // (optional) reset any internal state due to tensor initialization, such as tensor extras void (*reset) (ggml_backend_buffer_t buffer); + // (optional) batched tensor data copies + void (*set_tensor_n) (ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); + void (*get_tensor_n) (ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); + // (optional) batched tensor copies: returns true if all copies were performed, false if unsupported and no copies were performed + bool (*cpy_tensor_n) (ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors); }; struct ggml_backend_buffer { diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 0394433c0..595a5824a 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -489,6 +489,32 @@ static struct ggml_tensor * ggml_backend_meta_buffer_simple_tensor(const struct return it->second[index]; } +// bytes spanned by all dims except skip_dim +static size_t ggml_backend_meta_inner_extent(const struct ggml_tensor * tensor, int skip_dim) { + size_t extent = 0; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (i != skip_dim && tensor->ne[i] > 1) { + extent += (size_t) (tensor->ne[i] - 1) * tensor->nb[i]; + } + } + return extent; +} + +// the view dim that holds the split axis of src, or -1 +// all other dims of src and view must stay inside one slab of that axis +static int ggml_backend_meta_view_slab_dim(const struct ggml_tensor * view, const struct ggml_tensor * src, int axis) { + const size_t slab = src->nb[axis]; + if (ggml_backend_meta_inner_extent(src, axis) >= slab) { + return -1; + } + for (int dim = 0; dim < GGML_MAX_DIMS; dim++) { + if (view->ne[dim] == src->ne[axis] && view->nb[dim] == slab) { + return ggml_backend_meta_inner_extent(view, dim) < slab ? dim : -1; + } + } + return -1; +} + static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync); static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( @@ -700,9 +726,39 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( } GGML_ABORT("fatal error"); } + // permuted view: split the view dim that spans the src split axis, other dims must be inside one element of it or outside the span + if (ggml_is_permuted(tensor) && !ggml_is_permuted(tensor->src[0]) && axis >= 0 && axis < GGML_MAX_DIMS-1) { + const size_t nb_axis = tensor->src[0]->nb[axis]; + const size_t nb_next = tensor->src[0]->nb[axis+1]; + for (int dim = 0; dim < GGML_MAX_DIMS; dim++) { + if (tensor->ne[dim] == 1 || tensor->nb[dim] < nb_axis || tensor->nb[dim]*tensor->ne[dim] != nb_next) { + continue; + } + bool others_ok = true; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (i == dim || tensor->ne[i] == 1) { + continue; + } + others_ok = others_ok && (tensor->nb[i]*tensor->ne[i] <= tensor->nb[dim] || tensor->nb[i] % nb_next == 0); + } + if (others_ok) { + return {ggml_backend_meta_split_axis(dim), {0}, {1}, 1}; + } + } + } if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED || src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) { return src_ss[0]; } + // a view that only regroups dims inside one slab of the split axis keeps the split, + // the per-device strides are only rescaled above nb[axis] (e.g. the KV streams of a permuted q) + if (axis >= 0 && axis < GGML_MAX_DIMS && tensor->view_offs == tensor->src[0]->view_offs) { + const int dim = ggml_backend_meta_view_slab_dim(tensor, tensor->src[0], axis); + if (dim >= 0) { + ggml_backend_meta_split_state ret = src_ss[0]; + ret.axis = ggml_backend_meta_split_axis(dim); + return ret; + } + } GGML_ABORT("view of permuted tensor not implemented"); //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; }; @@ -869,7 +925,12 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( ggml_backend_meta_split_state split_state; switch (tensor->op) { case GGML_OP_NONE: { - split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; + if (tensor->view_src != nullptr) { + // full-tensor view created with ggml_view_tensor, transparent for the split state + split_state = ggml_backend_meta_get_split_state(stc, tensor->view_src, assume_sync); + } else { + split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; + } } break; case GGML_OP_DUP: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); @@ -878,7 +939,6 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( case GGML_OP_ADD_ID: { split_state = handle_bin_bcast(src_ss); } break; - case GGML_OP_ADD1: case GGML_OP_ACC: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; @@ -959,9 +1019,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( case GGML_OP_SET_ROWS: { split_state = handle_set_rows(src_ss); } break; - case GGML_OP_DIAG: - case GGML_OP_DIAG_MASK_INF: - case GGML_OP_DIAG_MASK_ZERO: { + case GGML_OP_DIAG: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; case GGML_OP_SOFT_MAX: @@ -995,8 +1053,13 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( split_state = handle_pad(src_ss); } break; case GGML_OP_PAD_REFLECT_1D: - case GGML_OP_ROLL: - case GGML_OP_ARANGE: + case GGML_OP_ROLL: { + split_state = handle_generic(src_ss, /*scalar_only =*/ true); + } break; + case GGML_OP_ARANGE: { + // no sources, every device computes the full range + split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; + } break; case GGML_OP_TIMESTEP_EMBEDDING: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; @@ -1041,7 +1104,8 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( } break; case GGML_OP_DSV4_HC_COMB: case GGML_OP_DSV4_HC_PRE: - case GGML_OP_DSV4_HC_POST: { + case GGML_OP_DSV4_HC_POST: + case GGML_OP_TOP_K_MASK: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; case GGML_OP_UNARY: { @@ -1171,7 +1235,22 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( return ret; } +static bool ggml_backend_meta_is_host_view(const struct ggml_tensor * tensor) { + return ggml_is_view(tensor) && ggml_backend_buffer_is_host(tensor->view_src->buffer); +} + static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) { + // [TAG_META_HOST_VIEWS] + // TODO: technically, this check should not be needed if the backend scheduler correctly prevents assigning + // such host-buffer views to the meta backend. figure out how to update the scheduler logic to achieve that + // ref: https://github.com/ggml-org/llama.cpp/pull/30217 + if (!ggml_backend_buffer_is_meta(tensor->buffer)) { + GGML_ASSERT(ggml_backend_meta_is_host_view(tensor)); + + // the view is not allocated in the meta buffer, it is not split across the sub-devices + return { GGML_BACKEND_SPLIT_AXIS_NONE, {0}, {1}, 1 }; + } + ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync); } @@ -1678,6 +1757,9 @@ static const ggml_backend_buffer_i ggml_backend_meta_buffer_iface = { /* .cpy_tensor = */ nullptr, /* .clear = */ ggml_backend_meta_buffer_clear, /* .reset = */ ggml_backend_meta_buffer_reset, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) { @@ -1723,13 +1805,14 @@ static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_b const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); constexpr size_t compute_headroom = 16; // Maximum number of views per statically allocated tensor that can be created between evals. + constexpr size_t compute_headroom_min = 256; // A buffer with few tensors can still get many views, e.g. a KV cache shared by many layers. const ggml_init_params params_static = { /*.mem_size =*/ n_tensors * ggml_tensor_overhead(), /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; const ggml_init_params params_compute = { - /*.mem_size =*/ compute_headroom * n_tensors * ggml_tensor_overhead(), + /*.mem_size =*/ std::max(compute_headroom * n_tensors, compute_headroom_min) * ggml_tensor_overhead(), /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; @@ -2021,9 +2104,11 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (node->view_src != nullptr && node->view_src->op == GGML_OP_NONE && ggml_backend_buffer_is_host(node->view_src->buffer)) { - // FIXME s_copy_main is on the CPU and its view seems to be incorrectly added to the graph nodes. - // For regular usage this doesn't matter since it's a noop but trying to call ggml_backend_meta_buffer_simple_tensor results in a crash. + if (!ggml_backend_buffer_is_meta(node->buffer)) { + // [TAG_META_HOST_VIEWS] + GGML_ASSERT(ggml_backend_meta_is_host_view(node)); + + // keep the node as is, mapping it to a simple tensor is not possible bcj.nodes[i] = node; continue; } @@ -2262,8 +2347,8 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, backend_ctx->ctx.reset(ggml_init(params)); for (size_t j = 0; j < n_backends; j++) { auto & bcj = backend_ctx->backend_configs[j]; - for (size_t i = 0; i < n_subgraphs; i++) { - bcj.cgraphs[i].cgraph_main = ggml_new_graph_custom(backend_ctx->ctx.get(), cgraph->n_nodes, /*grads =*/ false); + for (size_t i = 0; i < backend_ctx->max_subgraphs; i++) { + bcj.cgraphs[i].cgraph_main = ggml_new_graph_custom(backend_ctx->ctx.get(), backend_ctx->max_nnodes, /*grads =*/ false); } } backend_ctx->cgraphs_aux.resize(n_backends*n_cgraphs_per_device*backend_ctx->max_subgraphs); diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp index 7c076862c..dc801ba6f 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -546,7 +546,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, std::error_code cwd_ec; const fs::path cwd = fs::current_path(cwd_ec); if (cwd_ec) { - GGML_LOG_DEBUG("%s: current_path() failure, error-message: %s\n", __func__, cwd_ec.message().c_str()); + GGML_LOG_DEBUG("%s: failed to get current directory: %s\n", __func__, cwd_ec.message().c_str()); } else { search_paths.push_back(cwd); } @@ -559,20 +559,9 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, std::error_code ec; for (const auto & search_path : search_paths) { - if (!fs::exists(search_path, ec)) { - if (ec) { - GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str()); - } else { - GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str()); - } - continue; - } std::error_code dir_ec; - fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied, dir_ec); - if (dir_ec) { - GGML_LOG_DEBUG("%s: failed to enumerate %s: %s\n", __func__, path_str(search_path).c_str(), dir_ec.message().c_str()); - continue; - } + fs::directory_iterator dir_it(search_path, dir_ec); + for (const fs::directory_iterator end; dir_it != end; dir_it.increment(dir_ec)) { const auto & entry = *dir_it; if (entry.is_regular_file(ec)) { @@ -603,6 +592,10 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, } } } + + if (dir_ec) { + GGML_LOG_DEBUG("%s: failed to enumerate %s: %s\n", __func__, path_str(search_path).c_str(), dir_ec.message().c_str()); + } } if (best_score == 0) { @@ -635,21 +628,22 @@ void ggml_backend_load_all_from_path(const char * dir_path) { bool silent = false; #endif - ggml_backend_load_best("blas", silent, dir_path); - ggml_backend_load_best("zendnn", silent, dir_path); - ggml_backend_load_best("cann", silent, dir_path); - ggml_backend_load_best("cuda", silent, dir_path); - ggml_backend_load_best("hip", silent, dir_path); - ggml_backend_load_best("metal", silent, dir_path); - ggml_backend_load_best("rpc", silent, dir_path); - ggml_backend_load_best("sycl", silent, dir_path); - ggml_backend_load_best("vulkan", silent, dir_path); - ggml_backend_load_best("virtgpu", silent, dir_path); - ggml_backend_load_best("opencl", silent, dir_path); - ggml_backend_load_best("hexagon", silent, dir_path); - ggml_backend_load_best("musa", silent, dir_path); + ggml_backend_load_best("blas", silent, dir_path); + ggml_backend_load_best("zendnn", silent, dir_path); // AMD ZenDNN backend + ggml_backend_load_best("zdnn", silent, dir_path); // IBM zDNN backend + ggml_backend_load_best("cann", silent, dir_path); + ggml_backend_load_best("cuda", silent, dir_path); + ggml_backend_load_best("hip", silent, dir_path); + ggml_backend_load_best("metal", silent, dir_path); + ggml_backend_load_best("rpc", silent, dir_path); + ggml_backend_load_best("sycl", silent, dir_path); + ggml_backend_load_best("vulkan", silent, dir_path); + ggml_backend_load_best("virtgpu", silent, dir_path); + ggml_backend_load_best("opencl", silent, dir_path); + ggml_backend_load_best("hexagon", silent, dir_path); + ggml_backend_load_best("musa", silent, dir_path); ggml_backend_load_best("openvino", silent, dir_path); - ggml_backend_load_best("cpu", silent, dir_path); + ggml_backend_load_best("cpu", silent, dir_path); // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend const char * backend_path = std::getenv("GGML_BACKEND_PATH"); if (backend_path && reg_allowed("external")) { diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 5bdade6de..b70d0d874 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -402,6 +402,62 @@ size_t ggml_backend_get_max_size(ggml_backend_t backend) { return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend)); } +template <typename cb_get_buffer> +static void ggml_backend_tensor_buft_groups(size_t n_tensors, cb_get_buffer && get_buffer, std::vector<std::vector<size_t>> & groups) { + std::unordered_map<ggml_backend_buffer_type_t, size_t> buft_idx; + groups.clear(); + for (size_t i = 0; i < n_tensors; i++) { + ggml_backend_buffer_t buf = get_buffer(i); + auto it = buft_idx.find(buf->buft); + if (it == buft_idx.end()) { + buft_idx[buf->buft] = groups.size(); + groups.emplace_back(); + } + groups[buft_idx[buf->buft]].push_back(i); + } +} + +static bool ggml_backend_tensor_batch_is_2d( + const struct ggml_tensor * const * tensors, + const size_t * offsets, + const size_t * sizes, + const size_t * data_offsets, + size_t n_tensors, + size_t * base_offset, + size_t * size, + size_t * stride_tensor, + size_t * stride_data) { + if (n_tensors <= 1) { + return false; + } + + const struct ggml_tensor * tensor = tensors[0]; + const size_t size0 = sizes[0]; + + if (size0 == 0 || offsets[1] < offsets[0] || data_offsets[1] < data_offsets[0]) { + return false; + } + + const size_t stride_tensor0 = offsets[1] - offsets[0]; + const size_t stride_data0 = data_offsets[1] - data_offsets[0]; + + for (size_t i = 1; i < n_tensors; i++) { + if (tensors[i] != tensor || sizes[i] != size0) { + return false; + } + if (offsets[i] != offsets[0] + i*stride_tensor0 || data_offsets[i] != data_offsets[0] + i*stride_data0) { + return false; + } + } + + *base_offset = offsets[0]; + *size = size0; + *stride_tensor = stride_tensor0; + *stride_data = stride_data0; + + return true; +} + void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { GGML_ASSERT(backend); GGML_ASSERT(tensor); @@ -546,6 +602,154 @@ void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, buf->iface.get_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data); } +void ggml_backend_tensor_set_n(struct ggml_tensor * const * tensors, const void * data, + const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors) { + if (n_tensors == 0) { + return; + } + + std::vector<const void *> data_ptrs(n_tensors); + for (size_t i = 0; i < n_tensors; i++) { + data_ptrs[i] = (const char *) data + data_offsets[i]; + } + + ggml_backend_buffer_t buf = NULL; + bool same_buft = true; + + for (size_t i = 0; i < n_tensors; i++) { + GGML_ASSERT(tensors[i] != NULL); + ggml_backend_buffer_t tensor_buf = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer; + GGML_ASSERT(tensor_buf != NULL && "tensor buffer not set"); + GGML_ASSERT(tensors[i]->data != NULL && "tensor not allocated"); + GGML_ASSERT(offsets[i] + sizes[i] <= ggml_nbytes(tensors[i]) && "tensor write out of bounds"); + + if (buf == NULL) { + buf = tensor_buf; + } else if (buf->buft != tensor_buf->buft) { + same_buft = false; + } + } + + if (same_buft && buf->iface.set_tensor_n != NULL) { + buf->iface.set_tensor_n(buf, tensors, data_ptrs.data(), offsets, sizes, n_tensors); + return; + } + + if (!same_buft) { + std::vector<std::vector<size_t>> groups; + ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) { + return tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer; + }, groups); + + for (size_t g = 0; g < groups.size(); g++) { + std::vector<ggml_tensor *> g_tensors; + std::vector<size_t> g_offsets; + std::vector<size_t> g_sizes; + std::vector<size_t> g_data_offsets; + g_tensors.reserve(groups[g].size()); + g_offsets.reserve(groups[g].size()); + g_sizes.reserve(groups[g].size()); + g_data_offsets.reserve(groups[g].size()); + + for (size_t i : groups[g]) { + g_tensors.push_back(tensors[i]); + g_offsets.push_back(offsets[i]); + g_sizes.push_back(sizes[i]); + g_data_offsets.push_back(data_offsets[i]); + } + + ggml_backend_tensor_set_n(g_tensors.data(), data, g_offsets.data(), g_sizes.data(), g_data_offsets.data(), g_tensors.size()); + } + return; + } + + size_t base_offset = 0, size = 0, stride_tensor = 0, stride_data = 0; + if (ggml_backend_tensor_batch_is_2d(tensors, offsets, sizes, data_offsets, n_tensors, + &base_offset, &size, &stride_tensor, &stride_data)) { + ggml_backend_tensor_set_2d(tensors[0], data_ptrs[0], base_offset, size, + n_tensors, stride_tensor, stride_data); + return; + } + + for (size_t i = 0; i < n_tensors; i++) { + ggml_backend_tensor_set(tensors[i], data_ptrs[i], offsets[i], sizes[i]); + } +} + +void ggml_backend_tensor_get_n(const struct ggml_tensor * const * tensors, void * data, + const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors) { + if (n_tensors == 0) { + return; + } + + std::vector<void *> data_ptrs(n_tensors); + for (size_t i = 0; i < n_tensors; i++) { + data_ptrs[i] = (char *) data + data_offsets[i]; + } + + ggml_backend_buffer_t buf = NULL; + bool same_buft = true; + + for (size_t i = 0; i < n_tensors; i++) { + GGML_ASSERT(tensors[i] != NULL); + ggml_backend_buffer_t tensor_buf = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer; + GGML_ASSERT(tensor_buf != NULL && "tensor buffer not set"); + GGML_ASSERT(tensors[i]->data != NULL && "tensor not allocated"); + GGML_ASSERT(offsets[i] + sizes[i] <= ggml_nbytes(tensors[i]) && "tensor read out of bounds"); + + if (buf == NULL) { + buf = tensor_buf; + } else if (buf->buft != tensor_buf->buft) { + same_buft = false; + } + } + + if (same_buft && buf->iface.get_tensor_n != NULL) { + buf->iface.get_tensor_n(buf, tensors, data_ptrs.data(), offsets, sizes, n_tensors); + return; + } + + if (!same_buft) { + std::vector<std::vector<size_t>> groups; + ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) { + return tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer; + }, groups); + + for (size_t g = 0; g < groups.size(); g++) { + std::vector<const ggml_tensor *> g_tensors; + std::vector<size_t> g_offsets; + std::vector<size_t> g_sizes; + std::vector<size_t> g_data_offsets; + g_tensors.reserve(groups[g].size()); + g_offsets.reserve(groups[g].size()); + g_sizes.reserve(groups[g].size()); + g_data_offsets.reserve(groups[g].size()); + + for (size_t i : groups[g]) { + g_tensors.push_back(tensors[i]); + g_offsets.push_back(offsets[i]); + g_sizes.push_back(sizes[i]); + g_data_offsets.push_back(data_offsets[i]); + } + + ggml_backend_tensor_get_n(g_tensors.data(), data, g_offsets.data(), g_sizes.data(), g_data_offsets.data(), g_tensors.size()); + } + return; + } + + size_t base_offset = 0, size = 0, stride_tensor = 0, stride_data = 0; + if (ggml_backend_tensor_batch_is_2d(tensors, offsets, sizes, data_offsets, n_tensors, + &base_offset, &size, &stride_tensor, &stride_data)) { + ggml_backend_tensor_get_2d(tensors[0], data_ptrs[0], base_offset, size, + n_tensors, stride_tensor, stride_data); + return; + } + + for (size_t i = 0; i < n_tensors; i++) { + ggml_backend_tensor_get(tensors[i], data_ptrs[i], offsets[i], sizes[i]); + } +} + void ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { GGML_ASSERT(tensor); ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; @@ -648,6 +852,59 @@ void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor } } +void ggml_backend_tensor_copy_n(const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) { + if (n_tensors == 0) { + return; + } + + for (size_t i = 0; i < n_tensors; ++i) { + GGML_ASSERT(ggml_are_same_layout(srcs[i], dsts[i]) && "cannot copy tensors with different layouts"); + } + + ggml_backend_buffer_t buf = dsts[0]->view_src ? dsts[0]->view_src->buffer : dsts[0]->buffer; + bool same_buft = true; + for (size_t i = 0; i < n_tensors; ++i) { + ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer; + if (dst_buf->buft != buf->buft) { + same_buft = false; + break; + } + } + + // if cpy_tensor_n returns false, it must not have performed any copies + if (same_buft && buf->iface.cpy_tensor_n != NULL) { + if (buf->iface.cpy_tensor_n(buf, srcs, dsts, n_tensors)) { + return; + } + } + + if (!same_buft) { + std::vector<std::vector<size_t>> groups; + ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) { + return dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer; + }, groups); + + for (size_t g = 0; g < groups.size(); g++) { + std::vector<const ggml_tensor *> g_srcs; + std::vector<ggml_tensor *> g_dsts; + g_srcs.reserve(groups[g].size()); + g_dsts.reserve(groups[g].size()); + + for (size_t i : groups[g]) { + g_srcs.push_back(srcs[i]); + g_dsts.push_back(dsts[i]); + } + + ggml_backend_tensor_copy_n(g_srcs.data(), g_dsts.data(), g_srcs.size()); + } + return; + } + + for (size_t i = 0; i < n_tensors; ++i) { + ggml_backend_tensor_copy(srcs[i], dsts[i]); + } +} + void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) { GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts"); @@ -853,6 +1110,9 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = { /* .cpy_tensor = */ NULL, /* .clear = */ ggml_backend_multi_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) { @@ -935,8 +1195,9 @@ struct ggml_backend_sched { // hash map of the nodes in the graph struct ggml_hash_set hash_set; - int * hv_tensor_backend_ids; // [hash_set.size] - struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies] + int * hv_tensor_backend_ids; // [hash_set.size] + struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies] + bool * hv_tensor_requires_copy; // [hash_set.size] decides if input needs to be copied to each split int * node_backend_ids; // [graph_size] int * leaf_backend_ids; // [graph_size] @@ -966,6 +1227,12 @@ struct ggml_backend_sched { ggml_backend_sched_eval_callback callback_eval; void * callback_eval_user_data; + ggml_backend_sched_copy_callback callback_copy; + void * callback_copy_user_data; + + ggml_backend_sched_realloc_callback callback_realloc; + void * callback_realloc_user_data; + char * context_buffer; size_t context_buffer_size; @@ -973,11 +1240,21 @@ struct ggml_backend_sched { int debug; - // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC] + // graph size of the current and the previous scheduling, used by the realloc callback and for debugging + // graph reallocations [GGML_SCHED_DEBUG_REALLOC] // ref: https://github.com/ggml-org/llama.cpp/pull/17617 + int graph_size; + int prev_graph_size; + int debug_realloc; - int debug_graph_size; - int debug_prev_graph_size; + + // inputs of the next split read from the backend of the current split before it computes, see ggml_backend_sched_prefetch_inputs + ggml_backend_event_t prefetch_events[GGML_SCHED_MAX_BACKENDS]; // created on first use, NULL if the backend has no events + bool prefetch_tried [GGML_SCHED_MAX_BACKENDS]; + struct ggml_tensor * prefetched[GGML_SCHED_MAX_SPLIT_INPUTS]; + int n_prefetched; + int prefetch_split; // the split that reads the prefetched inputs + int prefetch_backend_id; // the backend that copies them }; #define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor) @@ -1532,6 +1809,11 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra } split->inputs[n_inputs] = src; } + // inputs of nodes that are not computed (e.g. the unselected branches of ggml_build_forward_select) + // stay in the split so that the allocation does not change, but they do not need to be copied + if (node->flags & GGML_TENSOR_FLAG_COMPUTE) { + sched->hv_tensor_requires_copy[src_id] = true; + } node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy); } } @@ -1630,9 +1912,9 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra } int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies + n_dep_nodes; - // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC] - sched->debug_prev_graph_size = sched->debug_graph_size; - sched->debug_graph_size = graph_size; + // remember the actual graph_size for performing reallocation checks later + sched->prev_graph_size = sched->graph_size; + sched->graph_size = graph_size; if (sched->graph.size < graph_size) { sched->graph.size = graph_size; @@ -1769,14 +2051,21 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { GGML_LOG_DEBUG("%s: failed to allocate graph, reserving (backend_ids_changed = %d)\n", __func__, backend_ids_changed); #endif + // a reallocation with an unchanged graph size means that the previous reservation was not sufficient + const bool graph_size_changed = sched->prev_graph_size != sched->graph_size; + + if (sched->callback_realloc != NULL) { + sched->callback_realloc(sched, &sched->graph, backend_ids_changed, graph_size_changed, sched->callback_realloc_user_data); + } + if (sched->debug_realloc > 0) { // we are interested only in situations where the graph was reallocated even though its size remained the same [GGML_SCHED_DEBUG_REALLOC] // example: https://github.com/ggml-org/llama.cpp/pull/17143 - const bool unexpected = !backend_ids_changed && sched->debug_prev_graph_size == sched->debug_graph_size; + const bool unexpected = !backend_ids_changed && !graph_size_changed; if (unexpected || sched->debug_realloc > 1) { GGML_ABORT("%s: unexpected graph reallocation (graph size = %d, nodes = %d, leafs = %d), debug_realloc = %d\n", __func__, - sched->debug_graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc); + sched->graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc); } } @@ -1799,14 +2088,129 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { return true; } +static bool ggml_backend_sched_is_host_weight(const struct ggml_tensor * t) { + return t->buffer != NULL && + ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + ggml_backend_buffer_is_host(t->buffer); +} + +static void ggml_backend_sched_copy_input(ggml_backend_sched_t sched, struct ggml_backend_sched_split * split, struct ggml_tensor * input) { + const int split_backend_id = split->backend_id; + ggml_backend_t split_backend = sched->backends[split_backend_id]; + ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, input); + struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy); + + // read before the previous split, wait for the copy only + if (split == &sched->splits[sched->prefetch_split]) { + for (int i = 0; i < sched->n_prefetched; i++) { + if (sched->prefetched[i] == input) { + ggml_backend_event_synchronize(sched->prefetch_events[sched->prefetch_backend_id]); + return; + } + } + } + + if (input->flags & GGML_TENSOR_FLAG_INPUT) { + // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + } + ggml_backend_tensor_copy(input, input_cpy); + return; + } + + // wait for the split backend to finish using the input before overwriting it + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + } + + if (sched->callback_copy != NULL && ggml_backend_sched_is_host_weight(input) && + sched->callback_copy(split_backend, input, input_cpy, &split->graph, sched->callback_copy_user_data)) { + return; + } + + // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events + // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface + if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) { + ggml_backend_synchronize(input_backend); + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + } + ggml_backend_tensor_copy(input, input_cpy); + } +} + +// the inputs of the next split that are computed on the backend of this split before it, and copied to host memory, +// are copied before this split computes, so that the next split waits for these copies instead of this split +// e.g. a CPU split runs while a GPU split computes, if it reads only the results of the earlier splits +static void ggml_backend_sched_prefetch_inputs(ggml_backend_sched_t sched, int split_id) { + sched->n_prefetched = 0; + if (split_id + 1 >= sched->n_splits) { + return; + } + struct ggml_backend_sched_split * split = &sched->splits[split_id]; + struct ggml_backend_sched_split * next = &sched->splits[split_id + 1]; + const int backend_id = split->backend_id; + ggml_backend_t backend = sched->backends[backend_id]; + if (backend->iface.get_tensor_async == NULL) { + return; + } + + for (int i = 0; i < next->n_inputs && sched->n_prefetched < GGML_SCHED_MAX_SPLIT_INPUTS; i++) { + struct ggml_tensor * input = next->inputs[i]; + if ((input->flags & GGML_TENSOR_FLAG_INPUT) || tensor_backend_id(input) != backend_id) { + continue; + } + struct ggml_tensor * input_cpy = tensor_copy(input, next->backend_id, sched->cur_copy); + if (input_cpy->buffer == NULL || !ggml_backend_buffer_is_host(input_cpy->buffer)) { + continue; + } + + // only results of earlier splits, the copy callback of this split runs later and can write other tensors (e.g. the MoE cache) + const struct ggml_tensor * base = input->view_src ? input->view_src : input; + if (base->buffer == NULL || ggml_backend_buffer_get_usage(base->buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + continue; + } + + // not if this split computes or writes the input + bool written = false; + for (int j = 0; j < split->graph.n_nodes && !written; j++) { + const struct ggml_tensor * node = split->graph.nodes[j]; + written = !ggml_is_view_op(node->op) && (node == base || node->view_src == base); + } + if (written) { + continue; + } + + if (!sched->prefetch_tried[backend_id]) { + sched->prefetch_tried[backend_id] = true; + sched->prefetch_events[backend_id] = ggml_backend_event_new(backend->device); + } + if (sched->prefetch_events[backend_id] == NULL) { + return; + } + + ggml_backend_tensor_get_async(backend, input, input_cpy->data, 0, ggml_nbytes(input)); + sched->prefetched[sched->n_prefetched++] = input; + } + + if (sched->n_prefetched > 0) { + sched->prefetch_split = split_id + 1; + sched->prefetch_backend_id = backend_id; + ggml_backend_event_record(sched->prefetch_events[backend_id], backend); + } +} + static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { GGML_ASSERT(sched); struct ggml_backend_sched_split * splits = sched->splits; - ggml_tensor * prev_ids_tensor = nullptr; - std::vector<int32_t> ids; - std::vector<ggml_bitset_t> used_ids; - int prev_backend_id = -1; for (int split_id = 0; split_id < sched->n_splits; split_id++) { @@ -1825,129 +2229,20 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } // copy the input tensors to the split backend + // the weights in host memory are copied last, so that the copy callback can read the other inputs of the split for (int input_id = 0; input_id < split->n_inputs; input_id++) { - ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); struct ggml_tensor * input = split->inputs[input_id]; - struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy); - - if (input->flags & GGML_TENSOR_FLAG_INPUT) { - // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); - } else { - ggml_backend_synchronize(split_backend); - } - ggml_backend_tensor_copy(input, input_cpy); - } else { - // wait for the split backend to finish using the input before overwriting it - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]); - } else { - ggml_backend_synchronize(split_backend); - } - - // when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used - ggml_tensor * node = split->graph.nodes[0]; - if (split->graph.n_nodes > 0 && - ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && - ggml_backend_buffer_is_host(input->buffer) && ( - (node->src[0] == input_cpy && node->op == GGML_OP_MUL_MAT_ID) - //|| (node->src[1] == input_cpy && node->op == GGML_OP_ADD_ID) /* GGML_OP_ADD_ID weights are small and not worth splitting */ - )) { - - const int64_t n_expert = node->op == GGML_OP_MUL_MAT_ID ? input->ne[2] : input->ne[1]; - const size_t expert_size = node->op == GGML_OP_MUL_MAT_ID ? input->nb[2] : input->nb[1]; - - ggml_backend_synchronize(input_backend); - - // get the ids - ggml_tensor * ids_tensor = node->src[2]; - ggml_backend_t ids_backend = split_backend; - - if (ggml_nelements(ids_tensor) == 0) { - continue; - } - - // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend - // in that case, we use the original ids tensor - for (int i = input_id + 1; i < split->n_inputs; i++) { - if (ids_tensor == tensor_copy(split->inputs[i], split_backend_id, sched->cur_copy)) { - ids_tensor = split->inputs[i]; - ids_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[i]); - break; - } - } - - if (ids_tensor != prev_ids_tensor) { - ids.resize(ggml_nbytes(ids_tensor) / sizeof(int32_t)); - ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor)); - ggml_backend_synchronize(ids_backend); - - // find the used experts - used_ids.clear(); - used_ids.resize(ggml_bitset_size(n_expert)); - for (int64_t i1 = 0; i1 < ids_tensor->ne[1]; i1++) { - for (int64_t i0 = 0; i0 < ids_tensor->ne[0]; i0++) { - int32_t id = ids[i1 * ids_tensor->nb[1]/sizeof(int32_t) + i0 * ids_tensor->nb[0]/sizeof(int32_t)]; - GGML_ASSERT(id >= 0 && id < n_expert); - ggml_bitset_set(used_ids.data(), id); - } - } - - prev_ids_tensor = ids_tensor; - } - - // group consecutive experts and copy them together - auto copy_experts = [&](int32_t first_id, int32_t last_id) { - const size_t expert_offset = first_id * expert_size; - const size_t expert_size_copy = (last_id - first_id + 1) * expert_size; - const size_t padding = std::min<size_t>(expert_size, 512); - const size_t padding_end = last_id < n_expert - 1 ? padding : 0; - - ggml_backend_tensor_set_async(split_backend, - input_cpy, - (const uint8_t *)input->data + expert_offset, expert_offset, - // copy a bit extra at the to ensure there are no NaNs in the padding of the last expert - // this is necessary for MMQ in the CUDA backend - expert_size_copy + padding_end); - }; - - int id = 0; - while (!ggml_bitset_get(used_ids.data(), id)) { - id++; - } - int32_t first_id = id; - int32_t last_id = first_id; - - for (++id; id < n_expert; ++id) { - if (!ggml_bitset_get(used_ids.data(), id)) { - continue; - } - - if (id == last_id + 1) { - last_id = id; - continue; - } - - copy_experts(first_id, last_id); - - first_id = id; - last_id = id; - } - copy_experts(first_id, last_id); - } else { - // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events - // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface - if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) { - ggml_backend_synchronize(input_backend); - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); - } else { - ggml_backend_synchronize(split_backend); - } - ggml_backend_tensor_copy(input, input_cpy); - } - } + if (!ggml_backend_sched_is_host_weight(input) && sched->hv_tensor_requires_copy[hash_id(input)]) { + ggml_backend_sched_copy_input(sched, split, input); + } + } + // after the copies above, which may read host memory reused by the copies of the next split + // before the weights, whose copy callback may queue large uploads + ggml_backend_sched_prefetch_inputs(sched, split_id); + for (int input_id = 0; input_id < split->n_inputs; input_id++) { + struct ggml_tensor * input = split->inputs[input_id]; + if (ggml_backend_sched_is_host_weight(input) && sched->hv_tensor_requires_copy[hash_id(input)]) { + ggml_backend_sched_copy_input(sched, split, input); } } @@ -2030,8 +2325,9 @@ ggml_backend_sched_t ggml_backend_sched_new( // initialize hash table // FIXME: needs to be size*2 to account for leafs (do it in graph_split instead) sched->hash_set = ggml_hash_set_new(graph_size); - sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0])); - sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); + sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0])); + sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); + sched->hv_tensor_requires_copy = (bool *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_requires_copy[0])); const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2; @@ -2040,8 +2336,8 @@ ggml_backend_sched_t ggml_backend_sched_new( sched->prev_node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0])); sched->prev_leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0])); - sched->debug_graph_size = 0; - sched->debug_prev_graph_size = 0; + sched->graph_size = 0; + sched->prev_graph_size = 0; sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false); sched->context_buffer = (char *) malloc(sched->context_buffer_size); @@ -2081,6 +2377,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { for (int c = 0; c < sched->n_copies; c++) { ggml_backend_event_free(sched->events[b][c]); } + ggml_backend_event_free(sched->prefetch_events[b]); } ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); @@ -2092,6 +2389,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { free(sched->graph_inputs); free(sched->hv_tensor_backend_ids); free(sched->hv_tensor_copies); + free(sched->hv_tensor_requires_copy); free(sched->node_backend_ids); free(sched->leaf_backend_ids); free(sched->prev_node_backend_ids); @@ -2107,8 +2405,9 @@ void ggml_backend_sched_reset(ggml_backend_sched_t sched) { // reset state for the next run if (!sched->is_reset) { ggml_hash_set_reset(&sched->hash_set); - memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0])); - memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); + memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0])); + memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); + memset(sched->hv_tensor_requires_copy, 0, sched->hash_set.size * sizeof(sched->hv_tensor_requires_copy[0])); sched->is_reset = true; } sched->is_alloc = false; @@ -2204,6 +2503,18 @@ void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backe sched->callback_eval_user_data = user_data; } +void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data) { + GGML_ASSERT(sched); + sched->callback_copy = callback; + sched->callback_copy_user_data = user_data; +} + +void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data) { + GGML_ASSERT(sched); + sched->callback_realloc = callback; + sched->callback_realloc_user_data = user_data; +} + int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) { GGML_ASSERT(sched); return sched->n_splits; @@ -2269,6 +2580,7 @@ bool ggml_op_alloc_size_may_expand(enum ggml_op op) { case GGML_OP_CUMSUM: case GGML_OP_ARGSORT: case GGML_OP_TOP_K: + case GGML_OP_ARGMAX: return true; default: return false; @@ -2574,6 +2886,9 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = { /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor, /* .clear = */ ggml_backend_cpu_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = { @@ -2588,6 +2903,9 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = { /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor, /* .clear = */ ggml_backend_cpu_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; // CPU backend buffer type diff --git a/ggml/src/ggml-cann/aclnn_ops.cpp b/ggml/src/ggml-cann/aclnn_ops.cpp index 902d2eda6..e602e4bf3 100644 --- a/ggml/src/ggml-cann/aclnn_ops.cpp +++ b/ggml/src/ggml-cann/aclnn_ops.cpp @@ -1347,31 +1347,6 @@ void ggml_cann_rms_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) { GGML_CANN_CALL_ACLNN_OP(ctx, RmsNorm, acl_src.get(), acl_gamma.get(), eps, acl_dst.get(), acl_rstd.get()); } -// TODO: performace is low. -void ggml_cann_diag_mask(ggml_backend_cann_context & ctx, ggml_tensor * dst, float value) { - ggml_tensor * src = dst->src[0]; - - acl_tensor_ptr acl_src = ggml_cann_create_tensor(src); - acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); - - const int n_past = ((int32_t *) dst->op_params)[0]; - - ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), ggml_nbytes(src)); - void * buffer = one_tensor_allocator.get(); - - acl_tensor_ptr mask_tensor = ggml_cann_create_tensor(buffer, ggml_cann_type_mapping(src->type), - ggml_type_size(src->type), src->ne, src->nb, GGML_MAX_DIMS); - - aclnn_fill_scalar(ctx, value, mask_tensor.get()); - - float alphaValue = 1.0f; - acl_scalar_ptr alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT); - - GGML_CANN_CALL_ACLNN_OP(ctx, InplaceTriu, mask_tensor.get(), n_past + 1); - GGML_CANN_CALL_ACLNN_OP(ctx, Tril, acl_src.get(), n_past + 1, acl_dst.get()); - GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst.get(), mask_tensor.get(), alpha.get()); -} - /** * @brief Permutes the dimensions of a tensor according to a specified order. * @@ -2934,7 +2909,7 @@ void ggml_cann_rope(ggml_backend_cann_context & ctx, ggml_tensor * dst) { const float theta_scale = powf(freq_base, -2.0f / n_dims); float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, true, corr_dims); bool is_neox = mode & GGML_ROPE_TYPE_NEOX; const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope @@ -3289,7 +3264,7 @@ void ggml_cann_rope_cache_preload(ggml_backend_cann_context & ctx, ggml_tensor * const float theta_scale = powf(freq_base, -2.0f / n_dims); float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, true, corr_dims); bool is_neox = mode & GGML_ROPE_TYPE_NEOX; const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; diff --git a/ggml/src/ggml-cann/aclnn_ops.h b/ggml/src/ggml-cann/aclnn_ops.h index 678f4d654..f470042cf 100644 --- a/ggml/src/ggml-cann/aclnn_ops.h +++ b/ggml/src/ggml-cann/aclnn_ops.h @@ -466,21 +466,6 @@ void ggml_cann_dup(ggml_backend_cann_context & ctx, ggml_tensor * dst); */ void ggml_cann_rms_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst); -/** - * @brief Applies a diagonal mask to the tensor with a specified value. - * - * @details This function creates a mask tensor filled with ones, then applies - * an upper triangular and lower triangular operation to it based on - * the number of past elements specified. Afterward, it adds the masked - * tensor to the destination tensor in-place. - * - * @param ctx The backend CANN context used for operations. - * @param dst The destination tensor where the result will be stored. dst->op is - * `GGML_OP_DIAG_MASK` - * @param value The value to use for masking. - */ -void ggml_cann_diag_mask(ggml_backend_cann_context & ctx, ggml_tensor * dst, float value); - /** * @brief Performs an image-to-column transformation on the input tensor. * diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index 28e17f3f7..0bfb96009 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -920,7 +920,7 @@ static void * ggml_backend_cann_buffer_get_base(ggml_backend_buffer_t buffer) { * @param dst Pointer to the destination buffer where transformed data will be * stored. */ -static void ggml_backend_cann_transform_q4_0(ggml_tensor * tensor, const void * src, void * dst) { +static void ggml_backend_cann_transform_q4_0(const ggml_tensor * tensor, const void * src, void * dst) { int64_t n_elems = ggml_nelements(tensor); int64_t groups = n_elems / QK4_0; size_t quant_bytes = n_elems * sizeof(uint8_t) / 2; @@ -1014,7 +1014,7 @@ static void ggml_backend_cann_transform_back_q4_0(const ggml_tensor * tensor, vo * @param dst Pointer to the destination buffer where transformed data will be * stored. */ -static void ggml_backend_cann_transform_q8_0(ggml_tensor * tensor, const void * src, void * dst) { +static void ggml_backend_cann_transform_q8_0(const ggml_tensor * tensor, const void * src, void * dst) { int64_t n_elems = ggml_nelements(tensor); int64_t groups = n_elems / QK8_0; size_t quant_bytes = n_elems * sizeof(uint8_t); @@ -1075,7 +1075,7 @@ static void ggml_backend_cann_transform_back_q8_0(const ggml_tensor * tensor, co * @param dst Pointer to the destination buffer where transformed data will be * stored. */ -static void ggml_backend_cann_transform(ggml_tensor * tensor, const void * src, void * dst) { +static void ggml_backend_cann_transform(const ggml_tensor * tensor, const void * src, void * dst) { switch (tensor->type) { case GGML_TYPE_Q4_0: ggml_backend_cann_transform_q4_0(tensor, src, dst); @@ -1398,34 +1398,35 @@ static void ggml_backend_cann_buffer_get_tensor(ggml_backend_buffer_t buffer, static bool ggml_backend_cann_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { - if (ggml_backend_buft_is_cann(src->buffer->buft)) { - ggml_backend_cann_buffer_context * src_ctx = (ggml_backend_cann_buffer_context *) src->buffer->context; - ggml_backend_cann_buffer_context * dst_ctx = (ggml_backend_cann_buffer_context *) buffer->context; + if (!ggml_backend_buft_is_cann(src->buffer->buft)) { + return false; + } - size_t memcpy_size = ggml_nbytes(src); - // Same device. - if (src_ctx->device == dst_ctx->device) { - ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size, - ACL_MEMCPY_DEVICE_TO_DEVICE)); - return true; - } else { + ggml_backend_cann_buffer_context * src_ctx = (ggml_backend_cann_buffer_context *) src->buffer->context; + ggml_backend_cann_buffer_context * dst_ctx = (ggml_backend_cann_buffer_context *) buffer->context; + + size_t memcpy_size = ggml_nbytes(src); + // Same device. + if (src_ctx->device == dst_ctx->device) { + ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size, + ACL_MEMCPY_DEVICE_TO_DEVICE)); + return true; + } else { #ifdef ASCEND_310P - // TODO: Support 310p P2P copy - return false; + // TODO: Support 310p P2P copy + return false; #endif - // Different device but can access by peer. - int32_t canAccessPeer = 0; - ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, src_ctx->device, dst_ctx->device)); - if (canAccessPeer) { - ggml_cann_set_device(src_ctx->device); - ACL_CHECK(aclrtDeviceEnablePeerAccess(dst_ctx->device, 0)); - ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size, - ACL_MEMCPY_DEVICE_TO_DEVICE)); - return true; - } + // Different device but can access by peer. + int32_t canAccessPeer = 0; + ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, src_ctx->device, dst_ctx->device)); + if (canAccessPeer) { + ggml_cann_set_device(src_ctx->device); + ACL_CHECK(aclrtDeviceEnablePeerAccess(dst_ctx->device, 0)); + ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size, + ACL_MEMCPY_DEVICE_TO_DEVICE)); + return true; } } - return false; } /** @@ -1478,6 +1479,9 @@ static const ggml_backend_buffer_i ggml_backend_cann_buffer_interface = { /* .cpy_tensor = */ ggml_backend_cann_buffer_cpy_tensor, /* .clear = */ ggml_backend_cann_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; /** @@ -1789,7 +1793,6 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg ggml_cann_dup(ctx, dst); break; case GGML_OP_ADD: - case GGML_OP_ADD1: ggml_cann_binary_op<aclnn_add>(ctx, dst); break; case GGML_OP_SUB: @@ -1954,9 +1957,6 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: break; - case GGML_OP_DIAG_MASK_INF: - ggml_cann_diag_mask(ctx, dst, -INFINITY); - break; case GGML_OP_SOFT_MAX: ggml_cann_softmax(ctx, dst); break; @@ -2443,6 +2443,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten break; case GGML_OP_MUL_MAT: { + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; + } switch (op->src[0]->type) { #ifndef ASCEND_310P case GGML_TYPE_BF16: @@ -2455,9 +2458,10 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten #ifdef ASCEND_310P // Q4 && Q8 per group is not support on 310p device return false; -#endif +#else // only support contiguous for quantized types. return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); +#endif default: return false; } @@ -2472,9 +2476,10 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten #ifdef ASCEND_310P // Q4 && Q8 per group is not support on 310p device return false; -#endif +#else // only support contiguous for quantized types. return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); +#endif default: return false; } @@ -2542,6 +2547,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten } case GGML_OP_ROPE: { + if (!ggml_rope_get_truncate(op)) { + return false; + } if (((const int32_t *) op->op_params)[15] != 0) { return false; // FIXME: support ggml_rope_set_offset } @@ -2606,7 +2614,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten case GGML_OP_TRANSPOSE: case GGML_OP_NORM: case GGML_OP_ADD: - case GGML_OP_ADD1: case GGML_OP_SUB: case GGML_OP_MUL: case GGML_OP_DIV: @@ -2614,7 +2621,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten case GGML_OP_SQR: case GGML_OP_SQRT: case GGML_OP_CLAMP: - case GGML_OP_DIAG_MASK_INF: case GGML_OP_SUM_ROWS: case GGML_OP_ARGSORT: case GGML_OP_ACC: @@ -2641,7 +2647,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten #ifdef ASCEND_310P // Ger is not supported on 310p device return false; -#endif +#else switch (op->src[0]->type) { case GGML_TYPE_F16: case GGML_TYPE_F32: @@ -2649,6 +2655,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten default: return false; } +#endif } case GGML_OP_CONV_TRANSPOSE_1D: return true; @@ -2664,10 +2671,14 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten return true; case GGML_OP_FLASH_ATTN_EXT: { + // K/V row lists (ggml_flash_attn_ext_rows) are not supported + if (op->src[5] != nullptr) { + return false; + } #ifdef ASCEND_310P // FA not support on 310p device return false; -#endif +#else // derived from [ggml-cuda.cu] if (op->src[1]->type != GGML_TYPE_F16 || op->src[2]->type != GGML_TYPE_F16) { return false; @@ -2693,6 +2704,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten return false; } return true; +#endif } case GGML_OP_SSM_CONV: return true; @@ -2850,12 +2862,13 @@ static ggml_backend_t ggml_backend_cann_device_init(ggml_backend_dev_t dev, cons * otherwise false. */ static bool ggml_backend_cann_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - if (ggml_backend_buft_is_cann(buft)) { - ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *) dev->context; - ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context; - return buft_ctx->device == dev_ctx->device; + if (!ggml_backend_buft_is_cann(buft)) { + return false; } - return false; + + ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *) dev->context; + ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context; + return buft_ctx->device == dev_ctx->device; } static ggml_backend_buffer_type_t ggml_backend_cann_device_get_buffer_type(ggml_backend_dev_t dev) { diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 71f55b490..8327c52e6 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -593,7 +593,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name) foreach (ZHW RANGE 15 17) if(DEFINED GGML_INTERNAL_Z${ZHW}) message(STATUS "z${ZHW} cross-compile target") - list(APPEND ARCH_FLAGS -march=z${ZHW}) + if (ZHW EQUAL 17) + # z17 is an alias of arch15, use the arch level for wider toolchain support + list(APPEND ARCH_FLAGS -march=arch15) + else() + list(APPEND ARCH_FLAGS -march=z${ZHW}) + endif() endif() endforeach() endif() @@ -645,9 +650,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) # Fetch KleidiAI sources: include(FetchContent) - set(KLEIDIAI_COMMIT_TAG "v1.24.0") + set(KLEIDIAI_COMMIT_TAG "v1.32.0") set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz") - set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "2f02ebe29573d45813e671eb304f2a00") + set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "05f1953428ffe8e8c7f585dc6d508887") set(KLEIDIAI_FETCH_ARGS URL ${KLEIDIAI_DOWNLOAD_URL} diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp index e90c2797f..554a2066a 100644 --- a/ggml/src/ggml-cpu/amx/amx.cpp +++ b/ggml/src/ggml-cpu/amx/amx.cpp @@ -116,6 +116,9 @@ static ggml_backend_buffer_i ggml_backend_amx_buffer_interface = { /* .cpy_tensor = */ nullptr, /* .clear = */ ggml_backend_amx_buffer_clear, /* .reset = */ nullptr, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; static const char * ggml_backend_amx_buffer_type_get_name(ggml_backend_buffer_type_t buft) { diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index e4882984a..4e59d0ae0 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -86,8 +86,6 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) -// quants.c -#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index cfbbf6617..f16505b2c 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -287,6 +287,9 @@ void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi sumf = vaddvq_f32(sumv); #else + UNUSED(nb); + UNUSED(x); + UNUSED(y); ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); return; #endif diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index ad0e5ccaf..b60eff9d5 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -535,6 +535,60 @@ void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(ncols_interleaved); UNUSED(blocklen); +#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) + if (svcntb() * 8 == 256) { + const svbool_t pg16 = svptrue_pat_b8(SV_VL16); + const svbool_t pg4 = svptrue_pat_b32(SV_VL4); + const svbool_t pgb = svptrue_b8(); + const svbool_t pgw = svptrue_b32(); + const svbool_t pg_hi4 = svnot_b_z(pgw, pg4); + const svuint8_t m4b = svdup_n_u8(0x0f); + const svuint8_t values = svld1_u8(pg16, (const uint8_t *) kvalues_mxfp4); + + // Reorder Q8 four-byte groups as [0,2,4,6,1,3,5,7] + const svuint32_t q8_group_index = svsub_n_u32_m(pg_hi4, svindex_u32(0, 2), 7); + + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + float * res_ptr = s; + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb); + svfloat32_t sumf = svdup_n_f32(0.0f); + + for (int l = 0; l < nb; l++) { + const svint8_t q8 = svld1_s8(pgb, a_ptr[l].qs); + const svint8_t q8_even_odd = svreinterpret_s8_u32(svtbl_u32(svreinterpret_u32_s8(q8), q8_group_index)); + const svuint8_t qmx01 = svld1_u8(pgb, b_ptr[l].qs + 0); + const svuint8_t qmx_lo01 = svtbl_u8(values, svand_u8_x(pgb, qmx01, m4b)); + const svuint8_t qmx_hi01 = svtbl_u8(values, svlsr_n_u8_x(pgb, qmx01, 4)); + const svuint8_t qmx23 = svld1_u8(pgb, b_ptr[l].qs + 32); + const svuint8_t qmx_lo23 = svtbl_u8(values, svand_u8_x(pgb, qmx23, m4b)); + const svuint8_t qmx_hi23 = svtbl_u8(values, svlsr_n_u8_x(pgb, qmx23, 4)); + + svint32_t sumi = svdup_n_s32(0); + sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_lo01), q8_even_odd, 0); + sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_lo23), q8_even_odd, 1); + sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_hi01), q8_even_odd, 2); + sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_hi23), q8_even_odd, 3); + const svint32_t sumi_cols = svadd_s32_z(pg4, sumi, svext_s32(sumi, sumi, 4)); + + // Vector form of GGML_CPU_E8M0_TO_FP32_HALF + const svuint32_t e_u32 = svld1ub_u32(pg4, b_ptr[l].e); + const svbool_t e_is_denormal = svcmplt_n_u32(pg4, e_u32, 2); + const svuint32_t e_denormal_bits = svlsl_u32_x(pg4, svdup_n_u32(0x00200000), e_u32); + const svuint32_t e_normal_bits = svlsl_n_u32_x(pg4, svsub_n_u32_x(pg4, e_u32, 1), 23); + const svuint32_t e_bits = svsel_u32(e_is_denormal, e_denormal_bits, e_normal_bits); + const svfloat16_t d_f16 = svreinterpret_f16_u16(svdup_n_u16(a_ptr[l].d)); + const svfloat32_t d_f32 = svcvt_f32_f16_x(pg4, d_f16); + const svfloat32_t scale = svmul_f32_x(pg4, svreinterpret_f32_u32(e_bits), d_f32); + sumf = svmla_f32_m(pg4, sumf,svcvt_f32_s32_m(svdup_n_f32(0.0f), pg4, sumi_cols), scale); + } + svst1_f32(pg4, res_ptr + x * 4, sumf); + } + return; + } +#endif // defined(__aarch64__) && defined(__ARM_FEATURE_SVE) + #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4); const block_q8_0 * a_ptr = (const block_q8_0 *) vy; @@ -1791,6 +1845,71 @@ void ggml_gemv_q8_0_4x8_q8_0(int n, UNUSED(ncols_interleaved); UNUSED(blocklen); +#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) + // 256-bit SVE: 32 x int8 -> 8 x int32 + if (ggml_cpu_get_sve_cnt() == QK8_0) { + const block_q8_0x4 * GGML_RESTRICT b_ptr = (const block_q8_0x4 *) vx; + + const svbool_t pg = svptrue_b8(); + const svbool_t pg4w = svptrue_pat_b32(SV_VL4); + + // after svtbl + svcvt the scale lanes are [d0,d0,d1,d1,d2,d2,d3,d3]: lane pair j = row j + const svuint16_t d_idx = svreinterpret_u16_u32(svlsr_n_u32_x(pg, svindex_u32(0, 1), 1)); + + for (int c = 0; c < nc; c += ncols_interleaved) { + const block_q8_0 * GGML_RESTRICT a_ptr = (const block_q8_0 *) vy; + + svfloat32_t acc = svdup_n_f32(0.0f); + + for (int b = 0; b < nb; ++b) { + const int8_t * GGML_RESTRICT bq = (const int8_t *) b_ptr->qs; + + uint64_t a_chunk0; + uint64_t a_chunk1; + uint64_t a_chunk2; + uint64_t a_chunk3; + uint64_t b_d; + + memcpy(&a_chunk0, a_ptr->qs + 0, sizeof(a_chunk0)); + memcpy(&a_chunk1, a_ptr->qs + 8, sizeof(a_chunk1)); + memcpy(&a_chunk2, a_ptr->qs + 16, sizeof(a_chunk2)); + memcpy(&a_chunk3, a_ptr->qs + 24, sizeof(a_chunk3)); + memcpy(&b_d, b_ptr->d, sizeof(b_d)); + + const svint8_t aq0 = svreinterpret_s8_u64(svdup_n_u64(a_chunk0)); + const svint8_t aq1 = svreinterpret_s8_u64(svdup_n_u64(a_chunk1)); + const svint8_t aq2 = svreinterpret_s8_u64(svdup_n_u64(a_chunk2)); + const svint8_t aq3 = svreinterpret_s8_u64(svdup_n_u64(a_chunk3)); + + const svint8_t bq0 = svld1_s8(pg, bq + 0); + const svint8_t bq1 = svld1_s8(pg, bq + 32); + const svint8_t bq2 = svld1_s8(pg, bq + 64); + const svint8_t bq3 = svld1_s8(pg, bq + 96); + + const svint32_t iacc0 = svdot_s32(svdup_n_s32(0), bq0, aq0); + const svint32_t iacc1 = svdot_s32(svdup_n_s32(0), bq1, aq1); + const svint32_t iacc2 = svdot_s32(svdup_n_s32(0), bq2, aq2); + const svint32_t iacc3 = svdot_s32(svdup_n_s32(0), bq3, aq3); + + const svint32_t isum = svadd_s32_x(pg, svadd_s32_x(pg, iacc0, iacc1), svadd_s32_x(pg, iacc2, iacc3)); + + const svfloat16_t bd_h = svtbl_f16(svreinterpret_f16_u64(svdup_n_u64(b_d)), d_idx); + const svfloat32_t scale = svmul_n_f32_x(pg, svcvt_f32_f16_x(pg, bd_h), GGML_CPU_FP16_TO_FP32(a_ptr->d)); + + acc = svmla_f32_x(pg, acc, svcvt_f32_s32_x(pg, isum), scale); + + ++a_ptr; + ++b_ptr; + } + + svst1_f32(pg4w, s, svadd_f32_x(pg, svuzp1_f32(acc, acc), svuzp2_f32(acc, acc))); + s += ncols_interleaved; + } + + return; + } +#endif // defined(__aarch64__) && defined(__ARM_FEATURE_SVE) + #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx; @@ -4874,6 +4993,7 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, constexpr int ncols_interleaved = 8; constexpr int blocklen = 8; + constexpr int q8_k_blocklen = 4; assert(n % qk == 0); assert(nr % 4 == 0); @@ -4882,9 +5002,195 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, UNUSED(nb); UNUSED(ncols_interleaved); UNUSED(blocklen); + UNUSED(q8_k_blocklen); + +#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) + if (svcntb() * 8 == 256) { + const svbool_t pg8 = svptrue_b8(); + const svbool_t pg32 = svptrue_b32(); + + const svuint8_t low_nibble = svdup_n_u8(0x0f); + const svuint8_t high_lo2 = svdup_n_u8(0x03); + const svuint8_t high_hi2 = svdup_n_u8(0x30); + const svint8_t q6_bias = svdup_n_s8(32); + const svint32_t zero_s32 = svdup_n_s32(0); + + for (int y = 0; y < nr / q8_k_blocklen; ++y) { + const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + y * nb; + + for (int x = 0; x < nc / ncols_interleaved; ++x) { + const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + x * nb; + + // one accumlator per output row + svfloat32_t acc_f32_r0 = svdup_n_f32(0); + svfloat32_t acc_f32_r1 = svdup_n_f32(0); + svfloat32_t acc_f32_r2 = svdup_n_f32(0); + svfloat32_t acc_f32_r3 = svdup_n_f32(0); + + for (int b = 0; b < nb; ++b) { + svint32_t acc_c03_r01 = zero_s32; + svint32_t acc_c03_r23 = zero_s32; + svint32_t acc_c47_r01 = zero_s32; + svint32_t acc_c47_r23 = zero_s32; + + for (int half = 0; half < 2; ++half) { + const uint8_t * ql_base = q6_ptr[b].ql + half * 512; + const uint8_t * qh_base = q6_ptr[b].qh + half * 256; + + for (int sb = 0; sb < QK_K / 64; ++sb) { + const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64; + const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64; + + // load 16 bytes (8B of R0, 8B of R1) and repeat in upper half + // low lanes will multiply with 16 bytes of C0 and high lanes will multiply with 16 bytes of C1 + const svint8_t q8_l_r01_k0 = svld1rq_s8(pg8, q8_base_l + 0); + const svint8_t q8_l_r01_k1 = svld1rq_s8(pg8, q8_base_l + 32); + const svint8_t q8_l_r23_k0 = svld1rq_s8(pg8, q8_base_l + 16); + const svint8_t q8_l_r23_k1 = svld1rq_s8(pg8, q8_base_l + 48); + + const svint8_t q8_h_r01_k0 = svld1rq_s8(pg8, q8_base_h + 0); + const svint8_t q8_h_r01_k1 = svld1rq_s8(pg8, q8_base_h + 32); + const svint8_t q8_h_r23_k0 = svld1rq_s8(pg8, q8_base_h + 16); + const svint8_t q8_h_r23_k1 = svld1rq_s8(pg8, q8_base_h + 48); + + const int ql_off = sb * QK_K / 2; + const int qh_off = ql_off & 255; + + const int scale_idx_l = half * 8 + sb; + const int scale_idx_h = scale_idx_l + 4; + + const svint32_t scales_l_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_l * 8); + const svint32_t scales_h_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_h * 8); + + const svint32_t scales_l_c03 = svzip1_s32(scales_l_all, scales_l_all); + const svint32_t scales_h_c03 = svzip1_s32(scales_h_all, scales_h_all); + + { + svuint8_t q6_ql0_01 = svld1_u8(pg8, ql_base + ql_off); + svuint8_t q6_ql1_01 = svld1_u8(pg8, ql_base + ql_off + 64); + svuint8_t q6_qh0_01 = svld1_u8(pg8, qh_base + qh_off); + svuint8_t q6_qh1_01 = svld1_u8(pg8, qh_base + qh_off + 64); + + if (sb > 1) { + q6_qh0_01 = svlsr_n_u8_x(pg8, q6_qh0_01, 2); + q6_qh1_01 = svlsr_n_u8_x(pg8, q6_qh1_01, 2); + } + + const svuint8_t q6_qh0_01_hi = svand_u8_x(pg8, q6_qh0_01, high_hi2); + const svuint8_t q6_qh1_01_hi = svand_u8_x(pg8, q6_qh1_01, high_hi2); + + const svuint8_t q6_l0_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_01, high_lo2), 4)); + const svuint8_t q6_l1_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_01, high_lo2), 4)); + const svuint8_t q6_h0_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_01, 4), q6_qh0_01_hi); + const svuint8_t q6_h1_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_01, 4), q6_qh1_01_hi); + + const svint8_t q6_l0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_01_bits), q6_bias); + const svint8_t q6_l1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_01_bits), q6_bias); + const svint8_t q6_h0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_01_bits), q6_bias); + const svint8_t q6_h1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_01_bits), q6_bias); + + svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r01_k0); + dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_01, q8_l_r01_k1); + + svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r01_k0); + dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_01, q8_h_r01_k1); + + svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r23_k0); + dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_01, q8_l_r23_k1); + + svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r23_k0); + dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_01, q8_h_r23_k1); + + acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_l_r01, scales_l_c03); + acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_h_r01, scales_h_c03); + acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_l_r23, scales_l_c03); + acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_h_r23, scales_h_c03); + } + + const svint32_t scales_l_c47 = svzip2_s32(scales_l_all, scales_l_all); + const svint32_t scales_h_c47 = svzip2_s32(scales_h_all, scales_h_all); + + { + svuint8_t q6_ql0_23 = svld1_u8(pg8, ql_base + ql_off + 32); + svuint8_t q6_ql1_23 = svld1_u8(pg8, ql_base + ql_off + 96); + svuint8_t q6_qh0_23 = svld1_u8(pg8, qh_base + qh_off + 32); + svuint8_t q6_qh1_23 = svld1_u8(pg8, qh_base + qh_off + 96); + + if (sb > 1) { + q6_qh0_23 = svlsr_n_u8_x(pg8, q6_qh0_23, 2); + q6_qh1_23 = svlsr_n_u8_x(pg8, q6_qh1_23, 2); + } + + const svuint8_t q6_qh0_23_hi = svand_u8_x(pg8, q6_qh0_23, high_hi2); + const svuint8_t q6_qh1_23_hi = svand_u8_x(pg8, q6_qh1_23, high_hi2); + + const svuint8_t q6_l0_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_23, high_lo2), 4)); + const svuint8_t q6_l1_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_23, high_lo2), 4)); + const svuint8_t q6_h0_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_23, 4), q6_qh0_23_hi); + const svuint8_t q6_h1_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_23, 4), q6_qh1_23_hi); + + const svint8_t q6_l0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_23_bits), q6_bias); + const svint8_t q6_l1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_23_bits), q6_bias); + const svint8_t q6_h0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_23_bits), q6_bias); + const svint8_t q6_h1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_23_bits), q6_bias); + + svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r01_k0); + dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_23, q8_l_r01_k1); + + svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r01_k0); + dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_23, q8_h_r01_k1); + + svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r23_k0); + dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_23, q8_l_r23_k1); + + svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r23_k0); + dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_23, q8_h_r23_k1); + + acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_l_r01, scales_l_c47); + acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_h_r01, scales_h_c47); + acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_l_r23, scales_l_c47); + acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_h_r23, scales_h_c47); + } + } + } + + // acc_c03_r01 -> c0-r0,c0-r1,c1-r0,c1-r1,c2-r0,c2-r1,c3-r0,c3-r1 + // acc_c47_r01 -> c4-r0,c4-r1,c5-r0,c5-r1,c6-r0,c6-r1,c7-r0,c7-r1 + // svuzp1_s32 picks even lanes -> c0-r0,c1-r0,c2-r0,c3-r0,c4-r0,c5-r0,c6-r0,c7-r0 + // similarly svuzp2_s32 picks remaining values from odd lanes + const svint32_t dot_r0 = svuzp1_s32(acc_c03_r01, acc_c47_r01); + const svint32_t dot_r1 = svuzp2_s32(acc_c03_r01, acc_c47_r01); + const svint32_t dot_r2 = svuzp1_s32(acc_c03_r23, acc_c47_r23); + const svint32_t dot_r3 = svuzp2_s32(acc_c03_r23, acc_c47_r23); + + const svuint32_t q6_d_bits = svld1uh_u32(pg32, (const uint16_t *) q6_ptr[b].d); + const svfloat32_t q6_d_f32 = svcvt_f32_f16_x(pg32, svreinterpret_f16_u32(q6_d_bits)); + + const svfloat32_t scale_r0 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[0]); + const svfloat32_t scale_r1 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[1]); + const svfloat32_t scale_r2 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[2]); + const svfloat32_t scale_r3 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[3]); + + acc_f32_r0 = svmla_f32_x(pg32, acc_f32_r0, svcvt_f32_s32_x(pg32, dot_r0), scale_r0); + acc_f32_r1 = svmla_f32_x(pg32, acc_f32_r1, svcvt_f32_s32_x(pg32, dot_r1), scale_r1); + acc_f32_r2 = svmla_f32_x(pg32, acc_f32_r2, svcvt_f32_s32_x(pg32, dot_r2), scale_r2); + acc_f32_r3 = svmla_f32_x(pg32, acc_f32_r3, svcvt_f32_s32_x(pg32, dot_r3), scale_r3); + } + + const int row = y * q8_k_blocklen; + const int col = x * ncols_interleaved; + + svst1_f32(pg32, s + (row + 0) * bs + col, acc_f32_r0); + svst1_f32(pg32, s + (row + 1) * bs + col, acc_f32_r1); + svst1_f32(pg32, s + (row + 2) * bs + col, acc_f32_r2); + svst1_f32(pg32, s + (row + 3) * bs + col, acc_f32_r3); + } + } + return; + } +#endif // SVE compile-time end #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) - constexpr int q8_k_blocklen = 4; const uint8x16_t m4b = vdupq_n_u8(0x0f); const uint8x16_t mask_lo = vdupq_n_u8(0x03); const uint8x16_t mask_hi = vdupq_n_u8(0x30); diff --git a/ggml/src/ggml-cpu/arch/riscv/repack.cpp b/ggml/src/ggml-cpu/arch/riscv/repack.cpp index c37488cae..49797df85 100644 --- a/ggml/src/ggml-cpu/arch/riscv/repack.cpp +++ b/ggml/src/ggml-cpu/arch/riscv/repack.cpp @@ -29,7 +29,7 @@ void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTR assert(k % QK8_0 == 0); const int nb = k / QK8_0; -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; const size_t vl_calc = __riscv_vsetvl_e32m8(QK8_0); const size_t vl_save = __riscv_vsetvl_e64m2(4); diff --git a/ggml/src/ggml-cpu/arch/s390/quants.c b/ggml/src/ggml-cpu/arch/s390/quants.c index 52344828e..22b8b8310 100644 --- a/ggml/src/ggml-cpu/arch/s390/quants.c +++ b/ggml/src/ggml-cpu/arch/s390/quants.c @@ -310,7 +310,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_yl = vec_xl(0 , y[ib].qs); const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs); - const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const int32x4_t v_xy_ = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh); const float32x4_t v_xy = vec_float(v_xy_); const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)); @@ -379,8 +379,8 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const int8x16_t v_y1l = vec_xl(0, y1->qs); const int8x16_t v_y1h = vec_xl(QK8_0/2, y1->qs); - const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h); - const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h); + const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h); + const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h); const float32x4_t v_xy0f = vec_float(v_xy0); const float32x4_t v_xy1f = vec_float(v_xy1); @@ -407,7 +407,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const int8x16_t v_yl = vec_xl(0, y0->qs); const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs); - const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh); const float32x4_t v_xyf = vec_float(v_xy); const float32x4_t v_d = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d)); @@ -502,8 +502,8 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_y1l = vec_xl(0, (const int8_t *)y1->qs); const int8x16_t v_y1h = vec_xl(QK8_0/2, (const int8_t *)y1->qs); - const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h); - const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h); + const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h); + const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h); const float32x4_t v_xy0f = vec_float(v_xy0); const float32x4_t v_xy1f = vec_float(v_xy1); @@ -548,7 +548,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_yl = vec_xl(0, (const int8_t *)y0->qs); const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)y0->qs); - const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh); + const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh); const float32x4_t v_xyf = vec_float(v_xy); const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d)); @@ -653,8 +653,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_y1l = vec_xl(0 , y1->qs); const int8x16_t v_y1h = vec_xl(QK8_1/2, y1->qs); - const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h); - const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h); + const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h); + const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h); const float32x4_t v_xy0f = vec_float(v_xy0); const float32x4_t v_xy1f = vec_float(v_xy1); @@ -701,7 +701,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_yl = vec_xl(0 , y0->qs); const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs); - const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh); + const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh); const float32x4_t v_xyf = vec_float(v_xy); const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d)); @@ -751,7 +751,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_yl = vec_xl(0 , y[ib].qs); const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs); - const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const int32x4_t v_xy_ = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh); const float32x4_t v_xy = vec_float(v_xy_); const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)); @@ -855,10 +855,10 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 2), v_3m), (int8x16_t)q3h[2]); q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 2), v_3m), (int8x16_t)q3h[3]); - isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[0]); - isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[1]); - isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[2]); - isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[3]); + isum0 = ggml_vec_dot_s32(v_z, q3bytes[0], q8bytes[0]); + isum1 = ggml_vec_dot_s32(v_z, q3bytes[1], q8bytes[1]); + isum2 = ggml_vec_dot_s32(v_z, q3bytes[2], q8bytes[2]); + isum3 = ggml_vec_dot_s32(v_z, q3bytes[3], q8bytes[3]); isum += (isum0[0] + isum0[1] + isum0[2] + isum0[3]) * scale[0]; isum += (isum1[0] + isum1[1] + isum1[2] + isum1[3]) * scale[1]; @@ -877,10 +877,10 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 6), v_3m), (int8x16_t)q3h[2]); q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 6), v_3m), (int8x16_t)q3h[3]); - isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[4]); - isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[5]); - isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[6]); - isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[7]); + isum0 = ggml_vec_dot_s32(v_z, q3bytes[0], q8bytes[4]); + isum1 = ggml_vec_dot_s32(v_z, q3bytes[1], q8bytes[5]); + isum2 = ggml_vec_dot_s32(v_z, q3bytes[2], q8bytes[6]); + isum3 = ggml_vec_dot_s32(v_z, q3bytes[3], q8bytes[7]); isum += vec_hsum_i32x4(isum0) * scale[0]; isum += vec_hsum_i32x4(isum1) * scale[1]; @@ -981,7 +981,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi v_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm); v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm); - const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); + const int32x4_t p1 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); sumi1 += vec_hsum_i32x4(p1) * scales[2*j+0]; v_y[0] = vec_xl(0 , y0); @@ -991,7 +991,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi v_xl[0] = (int8x16_t)vec_sr(v_x[0], 4); v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4); - const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); + const int32x4_t p2 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); sumi2 += vec_hsum_i32x4(p2) * scales[2*j+1]; } @@ -1107,8 +1107,8 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]); q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]); - int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]); - int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]); + int32x4_t sumi0 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]); + int32x4_t sumi1 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]); sumi += vec_hsum_i32x4(sumi0) * *scales++; sumi += vec_hsum_i32x4(sumi1) * *scales++; @@ -1217,10 +1217,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2])); q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3])); - int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]); - int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]); - int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]); - int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]); + int32x4_t summs0 = ggml_vec_dot_s32(v_z, q6b[0], v_y[0]); + int32x4_t summs1 = ggml_vec_dot_s32(v_z, q6b[1], v_y[1]); + int32x4_t summs2 = ggml_vec_dot_s32(v_z, q6b[2], v_y[2]); + int32x4_t summs3 = ggml_vec_dot_s32(v_z, q6b[3], v_y[3]); isum += vec_hsum_i32x4(summs0) * scale[0] + vec_hsum_i32x4(summs1) * scale[1] + @@ -1251,10 +1251,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2])); q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3])); - summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]); - summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]); - summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]); - summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]); + summs0 = ggml_vec_dot_s32(v_z, q6b[0], v_y[0]); + summs1 = ggml_vec_dot_s32(v_z, q6b[1], v_y[1]); + summs2 = ggml_vec_dot_s32(v_z, q6b[2], v_y[2]); + summs3 = ggml_vec_dot_s32(v_z, q6b[3], v_y[3]); isum += vec_hsum_i32x4(summs0) * scale[0] + vec_hsum_i32x4(summs1) * scale[1] + @@ -1367,8 +1367,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi // q2u2 = vec_mul(q2u2, q2s2); // q2u3 = vec_mul(q2u3, q2s3); -// const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1); -// const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3); +// const int32x4_t p1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1); +// const int32x4_t p2 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3); // sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28)); // sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28)); @@ -1445,7 +1445,7 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const int8x16_t v_yl = vec_xl(0 , y0->qs); const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs); - const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh); sumf += GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d) * vec_hsum_i32x4(v_xy); } @@ -1508,8 +1508,8 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v const int8x16_t v_y3 = vec_xl(48, q8); q8 += 64; - int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1); - int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3); + int32x4_t vsumi0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1); + int32x4_t vsumi1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3); int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32; int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32; diff --git a/ggml/src/ggml-cpu/arch/s390/repack.cpp b/ggml/src/ggml-cpu/arch/s390/repack.cpp index abf3433ad..c12fe215e 100644 --- a/ggml/src/ggml-cpu/arch/s390/repack.cpp +++ b/ggml/src/ggml-cpu/arch/s390/repack.cpp @@ -70,23 +70,6 @@ void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTR #endif } -#if defined(__VXE__) || defined(__VXE2__) -static inline int16x8_t vxe_dot_acc(const int8x16_t v_x, const int8x16_t v_y, const int16x8_t v_acc) { - return vec_meadd(v_x, v_y, vec_moadd(v_x, v_y, v_acc)); -} - -static inline int8x16_t vxe_splat_granule(const int8_t * qs) { - uint32_t g; - memcpy(&g, qs, sizeof(g)); - return (int8x16_t)vec_splats(g); -} - -static inline int32x4_t vxe_fold(const int16x8_t v_sumi) { - const int16x8_t v_ones = vec_splats((int16_t)1); - return vec_add(vec_mule(v_sumi, v_ones), vec_mulo(v_sumi, v_ones)); -} -#endif - void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -131,21 +114,21 @@ void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo int16x8_t v_sumi = vec_splats((int16_t)0); - v_sumi = vxe_dot_acc(v_x0l, vxe_splat_granule(y_lo + 0), v_sumi); - v_sumi = vxe_dot_acc(v_x1l, vxe_splat_granule(y_lo + 4), v_sumi); - v_sumi = vxe_dot_acc(v_x2l, vxe_splat_granule(y_lo + 8), v_sumi); - v_sumi = vxe_dot_acc(v_x3l, vxe_splat_granule(y_lo + 12), v_sumi); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x0l, ggml_vec_xl_s8x4_splat(y_lo + 0)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x1l, ggml_vec_xl_s8x4_splat(y_lo + 4)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x2l, ggml_vec_xl_s8x4_splat(y_lo + 8)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x3l, ggml_vec_xl_s8x4_splat(y_lo + 12)); - v_sumi = vxe_dot_acc(v_x0h, vxe_splat_granule(y_hi + 0), v_sumi); - v_sumi = vxe_dot_acc(v_x1h, vxe_splat_granule(y_hi + 4), v_sumi); - v_sumi = vxe_dot_acc(v_x2h, vxe_splat_granule(y_hi + 8), v_sumi); - v_sumi = vxe_dot_acc(v_x3h, vxe_splat_granule(y_hi + 12), v_sumi); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x0h, ggml_vec_xl_s8x4_splat(y_hi + 0)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x1h, ggml_vec_xl_s8x4_splat(y_hi + 4)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x2h, ggml_vec_xl_s8x4_splat(y_hi + 8)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x3h, ggml_vec_xl_s8x4_splat(y_hi + 12)); const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(a_ptr[l].d)); - const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d); + const float32x4_t v_xd = ggml_vec_xl_f16x4_finite(b_ptr[l].d); const float32x4_t v_d = vec_mul(v_yd, v_xd); - v_sumf = vec_madd(vec_float(vxe_fold(v_sumi)), v_d, v_sumf); + v_sumf = vec_madd(vec_float(vec_paddl_s16(v_sumi)), v_d, v_sumf); } vec_xst(v_sumf, 0, res_ptr + x * ncols_interleaved); @@ -192,24 +175,24 @@ void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const int8_t * y_lo = a_ptr[l].qs + 16 * k; const int8_t * y_hi = y_lo + qk / 2 * 4; - v_sumi0 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 0), v_sumi0); - v_sumi1 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 4), v_sumi1); - v_sumi2 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 8), v_sumi2); - v_sumi3 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 12), v_sumi3); + v_sumi0 = ggml_vec_dot_s16(v_sumi0, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 0)); + v_sumi1 = ggml_vec_dot_s16(v_sumi1, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 4)); + v_sumi2 = ggml_vec_dot_s16(v_sumi2, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 8)); + v_sumi3 = ggml_vec_dot_s16(v_sumi3, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 12)); - v_sumi0 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 0), v_sumi0); - v_sumi1 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 4), v_sumi1); - v_sumi2 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 8), v_sumi2); - v_sumi3 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 12), v_sumi3); + v_sumi0 = ggml_vec_dot_s16(v_sumi0, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 0)); + v_sumi1 = ggml_vec_dot_s16(v_sumi1, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 4)); + v_sumi2 = ggml_vec_dot_s16(v_sumi2, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 8)); + v_sumi3 = ggml_vec_dot_s16(v_sumi3, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 12)); } const float32x4_t v_yd = __lzs_f16cx4_load(a_ptr[l].d); - const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d); + const float32x4_t v_xd = ggml_vec_xl_f16x4_finite(b_ptr[l].d); - v_sumf[0] = vec_madd(vec_float(vxe_fold(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]); - v_sumf[1] = vec_madd(vec_float(vxe_fold(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]); - v_sumf[2] = vec_madd(vec_float(vxe_fold(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]); - v_sumf[3] = vec_madd(vec_float(vxe_fold(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]); + v_sumf[0] = vec_madd(vec_float(vec_paddl_s16(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]); + v_sumf[1] = vec_madd(vec_float(vec_paddl_s16(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]); + v_sumf[2] = vec_madd(vec_float(vec_paddl_s16(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]); + v_sumf[3] = vec_madd(vec_float(vec_paddl_s16(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]); } for (int m = 0; m < 4; m++) { diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index ea54cfe44..21cd6312a 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -698,6 +698,70 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } +#if defined(__AVX2__) +// four consecutive Q2_0 weights per byte: byte b becomes b & 3, (b >> 2) & 3, (b >> 4) & 3, (b >> 6) & 3 (OR, never add: the overlapping copies only touch masked-off bits) +static inline __m256i q2_0_unpack_32_avx2(const uint8_t * qs) { + __m256i v = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *) qs)); + v = _mm256_or_si256(v, _mm256_slli_epi32(v, 12)); + v = _mm256_or_si256(v, _mm256_slli_epi32(v, 6)); + return _mm256_and_si256(v, _mm256_set1_epi32(0x03030303)); +} + +// 32 weights against one Q8_0 block: sum((v - 1) * y) = sum(v * y) - sum(y), exact for every int8 y +static inline __m256 q2_0_dot_32_avx2(const uint8_t * qs, const int8_t * qy) { + const __m256i v = q2_0_unpack_32_avx2(qs); + const __m256i y = _mm256_loadu_si256((const __m256i *) qy); + const __m256i p = _mm256_sub_epi16(_mm256_maddubs_epi16(v, y), _mm256_maddubs_epi16(_mm256_set1_epi8(1), y)); + return _mm256_cvtepi32_ps(_mm256_madd_epi16(p, _mm256_set1_epi16(1))); +} + +static inline __m256 q2_0_acc_block_avx2(const block_q2_0 * x, const block_q8_0 * y, __m256 acc) { + const float dx = GGML_CPU_FP16_TO_FP32(x->d); + const __m256 d0 = _mm256_set1_ps(dx * GGML_CPU_FP16_TO_FP32(y[0].d)); + const __m256 d1 = _mm256_set1_ps(dx * GGML_CPU_FP16_TO_FP32(y[1].d)); + const __m256 p0 = q2_0_dot_32_avx2(x->qs + 0, y[0].qs); + const __m256 p1 = q2_0_dot_32_avx2(x->qs + 8, y[1].qs); +#if defined(__FMA__) + acc = _mm256_fmadd_ps(d0, p0, acc); + return _mm256_fmadd_ps(d1, p1, acc); +#else + acc = _mm256_add_ps(acc, _mm256_mul_ps(d0, p0)); + return _mm256_add_ps(acc, _mm256_mul_ps(d1, p1)); +#endif +} +#endif + +void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined(__AVX2__) + assert(n % QK2_0 == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + const int nb = n / QK2_0; + + // two Q2_0 blocks per step, each into its own accumulator (two FMA chains) + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + int i = 0; + for (; i + 1 < nb; i += 2) { + acc0 = q2_0_acc_block_avx2(x + i + 0, y + 2*i + 0, acc0); + acc1 = q2_0_acc_block_avx2(x + i + 1, y + 2*i + 2, acc1); + } + if (i < nb) { + acc0 = q2_0_acc_block_avx2(x + i, y + 2*i, acc0); + } + + *s = hsum_float_8(_mm256_add_ps(acc0, acc1)); +#else + ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/common.h b/ggml/src/ggml-cpu/common.h index abbadc359..2c62e6976 100644 --- a/ggml/src/ggml-cpu/common.h +++ b/ggml/src/ggml-cpu/common.h @@ -43,6 +43,14 @@ static inline float f32_to_f32(float x) { return x; } +static inline float f8_e4m3_to_f32(ggml_fp8_e4m3_t x) { + return ggml_f8_e4m3_to_fp32(x.bits); +} + +static inline ggml_fp8_e4m3_t f32_to_f8_e4m3(float x) { + return { ggml_fp32_to_f8_e4m3(x) }; +} + // TODO - merge this into the traits table, after using row-based conversions template <class T> struct type_conversion_table; @@ -65,6 +73,12 @@ struct type_conversion_table<ggml_bf16_t> { static constexpr ggml_bf16_t (*from_f32)(float) = f32_to_bf16; }; +template <> +struct type_conversion_table<ggml_fp8_e4m3_t> { + static constexpr float (*to_f32)(ggml_fp8_e4m3_t) = f8_e4m3_to_f32; + static constexpr ggml_fp8_e4m3_t (*from_f32)(float) = f32_to_f8_e4m3; +}; + template <> struct type_conversion_table<int32_t> { static constexpr float (*to_f32)(int32_t) = i32_to_f32; diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 5dd9ec8e6..f8b363728 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -30,7 +30,7 @@ struct ggml_compute_params { }; -#if defined(_MSC_VER) +#if defined(_MSC_VER) && !defined(__clang__) #define m512bh(p) p #define m512i(p) p @@ -345,7 +345,7 @@ static inline int32x4_t ggml_nvfp4_dot8(const int8x8_t q4_lo, const int8x8_t q8_ #include <immintrin.h> #endif -#ifdef __riscv_v_intrinsic +#if defined(__riscv_v) #include <riscv_vector.h> #endif @@ -390,17 +390,16 @@ typedef unsigned char uchar8x16_t __attribute__((vector_size(16))); typedef int8_t int8x16_t __attribute__((vector_size(16))); typedef int16_t int16x8_t __attribute__((vector_size(16))); typedef int32_t int32x4_t __attribute__((vector_size(16))); +typedef int64_t int64x2_t __attribute__((vector_size(16))); typedef uint8_t uint8x16_t __attribute__((vector_size(16))); typedef uint16_t uint16x8_t __attribute__((vector_size(16))); typedef uint32_t uint32x4_t __attribute__((vector_size(16))); +typedef uint64_t uint64x2_t __attribute__((vector_size(16))); typedef float float32x4_t __attribute__((vector_size(16))); typedef double double64x2_t __attribute__((vector_size(16))); -typedef signed long long long64x2_t __attribute__((vector_size(16))); -typedef unsigned long long ulong64x2_t __attribute__((vector_size(16))); - typedef struct ggml_uint8x16x2_t { uint8x16_t val[2]; } ggml_uint8x16x2_t; @@ -444,6 +443,13 @@ inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) { return res; } +// load 4 bytes and replicate them into all 4 word lanes +inline static int8x16_t ggml_vec_xl_s8x4_splat(const int8_t * ptr) { + uint32_t tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + return (int8x16_t)vec_splats(tmp); +} + typedef struct ggml_int16x8x2_t { int16x8_t val[2]; } ggml_int16x8x2_t; @@ -457,31 +463,17 @@ inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) { return res; } -/* - ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs - ! or iq4_nl for example implementation. -*/ -inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { - int8x16_t res; - - res[ 0] = a[b[ 0]]; - res[ 1] = a[b[ 1]]; - res[ 2] = a[b[ 2]]; - res[ 3] = a[b[ 3]]; - res[ 4] = a[b[ 4]]; - res[ 5] = a[b[ 5]]; - res[ 6] = a[b[ 6]]; - res[ 7] = a[b[ 7]]; - res[ 8] = a[b[ 8]]; - res[ 9] = a[b[ 9]]; - res[10] = a[b[10]]; - res[11] = a[b[11]]; - res[12] = a[b[12]]; - res[13] = a[b[13]]; - res[14] = a[b[14]]; - res[15] = a[b[15]]; - - return res; +// faster fp16 -> fp32 conversion using vector intrinsics +// WARNING: fp16 inf converts into 65536.0f. use this only for finite values. +inline static float32x4_t ggml_vec_xl_f16x4_finite(const ggml_fp16_t * x) { + uint64_t tmp; + memcpy(&tmp, x, sizeof(tmp)); + + const uint64x2_t v_x = { tmp, tmp }; + const uint32x4_t v_xh = vec_unpackh((uint16x8_t)v_x); + const uint32x4_t v_bits = vec_sl(vec_and(v_xh, vec_splats((uint32_t)0x7FFF)), 13); + const float32x4_t v_abs = vec_mul((float32x4_t)v_bits, vec_splats(0x1.0p+112f)); + return (float32x4_t)vec_or((uint32x4_t)v_abs, vec_sl(vec_and(v_xh, vec_splats((uint32_t)0x8000)), 16)); } inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { @@ -493,6 +485,11 @@ inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { return v_abo + v_abe; } +inline static int32x4_t vec_paddl_s16(int16x8_t a) { + const int16x8_t v_ones = vec_splats((int16_t)1); + return vec_mule(a, v_ones) + vec_mulo(a, v_ones); +} + /** * @see https://github.com/ggml-org/llama.cpp/pull/14037 */ @@ -506,7 +503,11 @@ inline static int32_t vec_hsum_i32x4(int32x4_t v) { return v_temp[0] + v_temp[1]; } -inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { +inline static int16x8_t ggml_vec_dot_s16(int16x8_t acc, int8x16_t a, int8x16_t b) { + return vec_meadd(a, b, vec_moadd(a, b, acc)); +} + +inline static int32x4_t ggml_vec_dot_s32(int32x4_t acc, int8x16_t a, int8x16_t b) { const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b); return acc + (vec_unpackh(p) + vec_unpackl(p)); } diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index cbd703d3e..612291871 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -86,6 +86,9 @@ float ggml_table_f32_e8m0_half[1 << 8]; // precomputed f32 table for ue4m3 (1 KB) (simd-mappings.h) float ggml_table_f32_ue4m3[1 << 8]; +// Signed FP8 needs its own table: UE4M3 ignores the sign, halves values for NVFP4, and maps NaN to zero. +float ggml_table_f32_f8_e4m3[1 << 8]; + #if defined(__ARM_ARCH) struct ggml_arm_arch_features_type { int sve_cnt; @@ -221,8 +224,8 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { }, [GGML_TYPE_F16] = { .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp16, - .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16, - .vec_dot_type = GGML_TYPE_F16, + .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16_f32, + .vec_dot_type = GGML_TYPE_F32, .nrows = 1, }, [GGML_TYPE_Q1_0] = { @@ -296,6 +299,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_F8_E4M3] = { + .from_float = quantize_row_f8_e4m3, + .vec_dot = ggml_vec_dot_f8_e4m3_f32, + .vec_dot_type = GGML_TYPE_F32, + .nrows = 1, + }, [GGML_TYPE_Q2_K] = { .from_float = quantize_row_q2_K, .vec_dot = ggml_vec_dot_q2_K_q8_K, @@ -772,7 +781,7 @@ static void ggml_init_arm_arch_features(void) {} #endif #endif // __ARM_ARCH -#if defined(__riscv) && defined(__riscv_v_intrinsic) +#if defined(__riscv) && defined(__riscv_v) #include <riscv_vector.h> static void ggml_init_riscv_arch_features(void) { ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb(); @@ -1310,9 +1319,12 @@ void ggml_compute_forward_mul_mat( const int ith = params->ith; const int nth = params->nth; - enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type; - ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; - int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows; + enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type; + ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; + int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows; + + // TODO: a more efficient F16 x F16 path can be used here if ggml_prec_get_src(dst, 1) allows converting src1 to F16 + // the code needs to be refactored to enable this (src1 is currently converted based on vec_dot_type alone) GGML_ASSERT(ne0 == ne01); GGML_ASSERT(ne1 == ne11); @@ -1535,8 +1547,8 @@ static void ggml_compute_forward_mul_mat_id_one_chunk( const enum ggml_type type = src0->type; - ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; - enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; + enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; const int64_t blck_0 = 16; const int64_t blck_1 = 16; @@ -1603,8 +1615,11 @@ static void ggml_compute_forward_mul_mat_id( const bool src1_cont = ggml_is_contiguous(src1); - enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; - ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; + enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; + + // TODO: a more efficient F16 x F16 path can be used here if ggml_prec_get_src(dst, 1) allows converting src1 to F16 + // the code needs to be refactored to enable this (src1 is currently converted based on vec_dot_type alone) // we don't support permuted src0 or src1 GGML_ASSERT(nb00 == ggml_type_size(type)); @@ -1687,7 +1702,12 @@ static void ggml_compute_forward_mul_mat_id( for (int id = 0; id < n_ids; ++id) { const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]); - assert(i02 >= 0 && i02 < n_as); + assert(i02 == -1 || (i02 >= 0 && i02 < n_as)); + + if (i02 == -1) { + memset((char *) dst->data + id*nb1 + iid1*nb2, 0, ne0*sizeof(float)); + continue; + } MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) {id, iid1}; matrix_row_counts[i02] += 1; @@ -1797,10 +1817,6 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_add_id(params, tensor); } break; - case GGML_OP_ADD1: - { - ggml_compute_forward_add1(params, tensor); - } break; case GGML_OP_ACC: { ggml_compute_forward_acc(params, tensor); @@ -1941,14 +1957,6 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_diag(params, tensor); } break; - case GGML_OP_DIAG_MASK_INF: - { - ggml_compute_forward_diag_mask_inf(params, tensor); - } break; - case GGML_OP_DIAG_MASK_ZERO: - { - ggml_compute_forward_diag_mask_zero(params, tensor); - } break; case GGML_OP_SOFT_MAX: { ggml_compute_forward_soft_max(params, tensor); @@ -2140,6 +2148,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_dsv4_hc_post(params, tensor); } break; + case GGML_OP_TOP_K_MASK: + { + ggml_compute_forward_top_k_mask(params, tensor); + } break; case GGML_OP_MAP_CUSTOM1: { ggml_compute_forward_map_custom1(params, tensor); @@ -2296,7 +2308,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_CONT: case GGML_OP_ADD: case GGML_OP_ADD_ID: - case GGML_OP_ADD1: case GGML_OP_ACC: case GGML_OP_CUMSUM: case GGML_OP_TRI: @@ -2323,6 +2334,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_DSV4_HC_COMB: case GGML_OP_DSV4_HC_PRE: case GGML_OP_DSV4_HC_POST: + case GGML_OP_TOP_K_MASK: { n_tasks = n_threads; } break; @@ -2395,11 +2407,11 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: case GGML_OP_OUT_PROD: + case GGML_OP_SET_ROWS: { n_tasks = n_threads; } break; case GGML_OP_GET_ROWS: - case GGML_OP_SET_ROWS: { // FIXME: get_rows can use additional threads, but the cost of launching additional threads // decreases performance with GPU offloading @@ -2417,8 +2429,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { { n_tasks = 1; } break; - case GGML_OP_DIAG_MASK_ZERO: - case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX_BACK: case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: @@ -2899,9 +2909,8 @@ struct ggml_cplan ggml_graph_plan( } break; case GGML_OP_ADD: case GGML_OP_ADD_ID: - case GGML_OP_ADD1: { - if (ggml_is_quantized(node->src[0]->type)) { + if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3) { cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; } } break; @@ -2949,7 +2958,7 @@ struct ggml_cplan ggml_graph_plan( } break; case GGML_OP_OUT_PROD: { - if (ggml_is_quantized(node->src[0]->type) || + if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3 || node->src[0]->type == GGML_TYPE_F16) { cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; } @@ -2981,8 +2990,8 @@ struct ggml_cplan ggml_graph_plan( if ((node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_BF16) && node->src[1]->type == GGML_TYPE_F32) { + cur += sizeof(float)*ne10*ne11; cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02; - cur += sizeof(ggml_fp16_t)*ne10*ne11; } else if (node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32) { cur += sizeof(float)*ne00*ne01*ne02; @@ -3011,8 +3020,8 @@ struct ggml_cplan ggml_graph_plan( GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32); + cur += sizeof(float) * ne10 * ne11 * ne12 * ne13; cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03; - cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13; } break; case GGML_OP_TOP_K: @@ -3539,6 +3548,12 @@ void ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) { vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl); __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl); } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 7 < n; i += 8) { + const uint32x4_t v_yl = __lzs_f32cx4_to_f16(vec_xl(0, x + i + 0)); + const uint32x4_t v_yh = __lzs_f32cx4_to_f16(vec_xl(0, x + i + 4)); + vec_xst(vec_pack(v_yl, v_yh), 0, (uint16_t *)(y + i)); + } #endif for (; i < n; ++i) { y[i] = GGML_CPU_FP32_TO_FP16(x[i]); @@ -3566,7 +3581,7 @@ void ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) { _mm_storeu_ps(y + i, y_vec); } -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin) +#elif defined(__riscv_v) && defined(__riscv_zvfhmin) // calculate step size const int epr = __riscv_vsetvlmax_e16m2(); const int step = epr * 2; @@ -3636,7 +3651,7 @@ void ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) { (const __m128i *)(x + i))), 16))); } -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin) +#elif defined(__riscv_v) && defined(__riscv_zvfbfmin) // calculate step size const int epr = __riscv_vsetvlmax_e16m2(); const int step = epr * 2; @@ -3756,7 +3771,7 @@ int ggml_cpu_has_arm_fma(void) { } int ggml_cpu_has_riscv_v(void) { -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) return 1; #else return 0; @@ -3764,7 +3779,7 @@ int ggml_cpu_has_riscv_v(void) { } int ggml_cpu_get_rvv_vlen(void) { -#if defined(__riscv) && defined(__riscv_v_intrinsic) +#if defined(__riscv) && defined(__riscv_v) return ggml_riscv_arch_features.rvv_vlen; #else return 0; @@ -3927,6 +3942,7 @@ void ggml_cpu_init(void) { // initialize UE4M3 table (256 entries) for (int i = 0; i < (1 << 8); ++i) { ggml_table_f32_ue4m3[i] = ggml_ue4m3_to_fp32(i); + ggml_table_f32_f8_e4m3[i] = ggml_f8_e4m3_to_fp32(i); } const uint64_t t_end = ggml_time_us(); UNUSED(t_end); diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 81ff5d79f..297834001 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -455,10 +455,11 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) { return src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16; } - // BF16 in src1 is widened into the F32 work buffer + // F16 or BF16 in src1 is widened into the F32 work buffer return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type || - (src1->type == GGML_TYPE_BF16 && ggml_get_type_traits_cpu(src0->type)->vec_dot_type == GGML_TYPE_F32); + ((src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16) && + ggml_get_type_traits_cpu(src0->type)->vec_dot_type == GGML_TYPE_F32); case GGML_OP_SOFT_MAX_BACK: { if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) { return false; diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cpp index d4551298f..24aa28244 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp @@ -3,6 +3,8 @@ // // KleidiAI micro-kernels +#include <cstddef> + #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h" #include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h" #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h" @@ -33,6 +35,10 @@ #include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.h" #include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.h" +#include "kai/ukernels/matmul/kai_matmul.h" +#include "kai/ukernels/matmul/kai_matmul_pack_lhs.h" +#include "kai/ukernels/matmul/kai_matmul_pack_rhs.h" + #include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h" #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.h" #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h" @@ -42,8 +48,6 @@ #include "kai/kai_common.h" -#include "simd-mappings.h" - #define GGML_COMMON_DECL_CPP #include "ggml-common.h" @@ -154,16 +158,6 @@ static inline size_t rhs_ps_fn2(size_t n, size_t k, size_t /*nr*/, size_t /*kr*/ return Fn(n, k); } -template<size_t(*Fn)(size_t,size_t,size_t,size_t)> -static inline size_t rhs_stride_fn4(size_t k, size_t nr, size_t kr, size_t bl) { - return Fn(k, nr, kr, bl); -} - -template<size_t(*Fn)(size_t)> -static inline size_t rhs_stride_fn1(size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) { - return Fn(k); -} - template<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,size_t,const uint8_t*,const float*,void*,size_t,const struct kai_rhs_pack_qs4cxs1s0_param*)> static inline void rhs_pack_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl, size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* /*scale*/, @@ -194,137 +188,237 @@ static inline void rhs_pack_fn13(size_t num_groups, size_t n, size_t k, size_t n Fn(num_groups, n, k, nr, kr, sr, rhs_stride, rhs, bias, scale, rhs_packed, extra_bytes, params); } -static const size_t INT4_PER_BYTE = 2; -static const size_t INT4_BITS = 4; -static const int Q4_0_ZERO_POINT = 8; -const size_t INT4_PER_UINT16 = 4; - -static void dequantize_row_qsi4c32pscalef16( - const void *packed_data, - int32_t row_idx, - int64_t nc, - float *out, - size_t nr_pack, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - size_t group_idx = row_idx / nr_pack; - size_t row_in_group = row_idx % nr_pack; - const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride; - size_t num_blocks = nc / bl; - const uint8_t *block_ptr = packed_group; - - for (size_t b = 0; b < num_blocks; ++b) { - uint16_t scale_f16 = *((const uint16_t *)(block_ptr + row_in_group * num_bytes_multiplier)); - float scale = GGML_CPU_FP16_TO_FP32(scale_f16); - - const uint8_t *segment_ptr = block_ptr + nr_pack * num_bytes_multiplier; - size_t num_segments = bl / kr; - size_t num_bytes_per_segment = kr / INT4_PER_BYTE; - - for (size_t s = 0; s < num_segments; ++s) { - const uint8_t *seg_base = segment_ptr + s * nr_pack * num_bytes_per_segment; - const uint8_t *qbytes = seg_base + row_in_group * num_bytes_per_segment; - for (size_t k = 0; k < num_bytes_per_segment; ++k) { - uint8_t byte = qbytes[k] ^ 0x88; - int x0 = (byte & 0x0F) - Q4_0_ZERO_POINT; - int x1 = (byte >> INT4_BITS) - Q4_0_ZERO_POINT; - out[b * bl + s * num_bytes_per_segment + k] = x0 * scale; - out[b * bl + s * num_bytes_per_segment + k + bl/2] = x1 * scale; - } - } - block_ptr += nr_pack * num_bytes_multiplier + num_segments * nr_pack * num_bytes_per_segment; - } +static constexpr size_t KLEIDIAI_Q4_K_BLOCK_LENGTH = QK_K / 8; + +static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks"); +static_assert(sizeof(block_q4_K) == 144, "Unexpected ggml Q4_K block layout"); +static_assert(offsetof(block_q4_K, scales) == 4, "Unexpected ggml Q4_K metadata offset"); +static_assert(offsetof(block_q4_K, qs) == 16, "Unexpected ggml Q4_K data offset"); + +static struct kai_matmul_uker_config q4_k_matmul_config() { + struct kai_matmul_uker_config config = {}; + config.format.bl = KLEIDIAI_Q4_K_BLOCK_LENGTH; + return config; } -static void dequantize_row_qsi4c32ps1s0scalef16( - const void *packed_data, - int32_t row_idx, - int64_t k, - float *out, - size_t nr, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - const size_t num_blocks = k / bl; - const size_t bl4 = bl / INT4_PER_UINT16; - - size_t group_idx = row_idx / nr; - size_t row_in_group = row_idx % nr; - - const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride; - const uint16_t *qdata = (const uint16_t *)packed_group; - const uint16_t *scales = (const uint16_t *)(packed_group + packed_row_stride - (nr * num_blocks * num_bytes_multiplier)); - - for (size_t block_idx = 0; block_idx < num_blocks; ++block_idx) { - uint16_t scale_f16 = scales[row_in_group + block_idx * nr]; - float scale = GGML_CPU_FP16_TO_FP32(scale_f16); - - for (size_t bl4_idx = 0; bl4_idx < bl4; ++bl4_idx) { - uint16_t q = qdata[(block_idx * bl4 + bl4_idx) * nr + row_in_group]; - - for (size_t qidx = 0; qidx < INT4_PER_UINT16; ++qidx) { - int v = ((q >> (qidx * 4)) & 0xF) - Q4_0_ZERO_POINT; - out[block_idx * bl + bl4_idx * INT4_BITS + qidx] = v * scale; - } +static struct kai_matmul_uker_config default_matmul_config() { + return {}; +} + +template< + struct kai_matmul_uker_api (*GetApi)(void), + struct kai_matmul_uker_config (*GetConfig)(void), + size_t MR, size_t NR, size_t KR, size_t SR> +struct matmul_uker_adapter { + static size_t get_m_step() { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + return api.get_step(&config).m; + } + + static size_t get_n_step() { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + return api.get_step(&config).n; + } + + static size_t get_mr() { + if constexpr (MR == 0) { + return get_m_step(); } + return MR; } - GGML_UNUSED(kr); -} -static void dequantize_row_qsi8cxp( - const void *packed_data, - int32_t row_idx, - int64_t k, - float *out, - size_t nr, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - GGML_UNUSED(bl); - GGML_UNUSED(num_bytes_multiplier); - - const size_t k_internal = ((size_t) k + QK8_0 - 1) / QK8_0 * QK8_0; - const size_t group_idx = row_idx / nr; - const size_t row_in_group = row_idx % nr; - - const uint8_t * group_ptr = static_cast<const uint8_t *>(packed_data) + group_idx * packed_row_stride; - const int8_t * data_base = reinterpret_cast<const int8_t *>(group_ptr); - - const size_t num_blocks = k_internal / kr; - - for (size_t block = 0; block < num_blocks; ++block) { - const int8_t * block_ptr = data_base + (block * nr + row_in_group) * kr; - for (size_t i = 0; i < kr; ++i) { - const size_t k_idx = block * kr + i; - if (k_idx < (size_t) k) { - out[k_idx] = static_cast<float>(block_ptr[i]); - } + static size_t get_nr() { + if constexpr (NR == 0) { + return get_n_step(); } + return NR; } - const uint8_t * sums_ptr = group_ptr + nr * k_internal; - GGML_UNUSED(sums_ptr); + static size_t get_kr() { + return KR; + } - const float * scale_ptr = reinterpret_cast<const float *>(sums_ptr + nr * sizeof(int32_t)); - const float scale = scale_ptr[row_in_group]; + static size_t get_sr() { + return SR; + } - if (scale == 0.0f) { - for (size_t i = 0; i < (size_t) k; ++i) { - out[i] = 0.0f; - } - return; + static size_t get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx }; + const struct kai_matmul_uker_dst_stride_args stride = { dst_stride }; + return api.get_dst_offset(&config, &index, &stride); + } + + static size_t get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k }; + const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 }; + const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape); + return api.get_lhs_offset(&config, &index, &stride); + } + + static size_t get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k }; + const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 }; + const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape); + return api.get_rhs_offset(&config, &index, &stride); } - for (size_t i = 0; i < (size_t) k; ++i) { - out[i] *= scale; + static void run( + size_t m, size_t n, size_t k, size_t bl, + const void * lhs_packed, const void * rhs_packed, + void * dst, size_t dst_stride_row, size_t dst_stride_col, + float clamp_min, float clamp_max) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + GGML_ASSERT(dst_stride_col == sizeof(float)); + const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k }; + const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k }; + + struct kai_matmul_uker_args args = {}; + args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP; + args.shape = { m, n, k }; + args.operand.lhs.ptr = lhs_packed; + args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape); + args.operand.rhs.ptr = rhs_packed; + args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape); + args.operand.dst.ptr = dst; + args.operand.dst.stride.m = dst_stride_row; + args.activation.clamp.min_ptr = &clamp_min; + args.activation.clamp.max_ptr = &clamp_max; + api.run(&config, &args); } +}; + +using q4_k_gemm_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa, + q4_k_matmul_config, 0, 0, 4, 2>; +using q4_k_gemv_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot, + q4_k_matmul_config, 0, 0, 4, 2>; +// GEMM and GEMV share the RHS layout only at SVE256 (NR=8, KR=8, SR=1). +using sve_q8_gemm_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qai8dxp4x8sf32_qsi8cxp4vsx8sf32bf32_16x4vs_sve_i8mm, + default_matmul_config, 4, 0, 8, 1>; +using sve_q8_gemv_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp8x8_1x8_sve_dot, + default_matmul_config, 1, 8, 8, 1>; + +static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() { + struct kai_matmul_pack_lhs_uker_config config = {}; + return config; +} + +static size_t q4_k_lhs_get_offset(size_t m_idx, size_t lhs_stride) { + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_dim_args index = { m_idx, 0 }; + const struct kai_matmul_pack_lhs_uker_lhs_stride_args stride = { lhs_stride }; + return api.get_lhs_offset(&config, &index, &stride); +} + +static size_t q4_k_lhs_get_packed_offset( + size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m_idx, k }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args index = { m_idx, 0 }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape); + return api.get_lhs_packed_offset(&config, &index, &stride); +} + +static size_t q4_k_lhs_get_packed_size( + size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m, k }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape); + return api.get_lhs_packed_size(&config, &shape, &stride); +} + +static void q4_k_lhs_pack( + size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr, + size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + GGML_ASSERT(m_idx_start == 0); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args packed_shape = { m, k }; + + struct kai_matmul_pack_lhs_uker_args args = {}; + args.flags = 0; + args.shape = { m, k }; + args.operand.lhs.ptr = lhs; + args.operand.lhs.stride.m = lhs_stride; + args.operand.lhs_packed.ptr = lhs_packed; + args.operand.lhs_packed.stride = api.get_lhs_packed_stride(&config, &packed_shape); + api.run(&config, &args); +} + +static struct kai_matmul_pack_rhs_uker_config q4_k_rhs_pack_config() { + struct kai_matmul_pack_rhs_uker_config config = {}; + return config; +} + +static size_t q4_k_rhs_get_packed_size(size_t n, size_t k, size_t nr, size_t /*kr*/, size_t bl) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + + const struct kai_matmul_pack_rhs_uker_api api = + kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme(); + const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config(); + GGML_ASSERT(api.get_step(&config).n == nr); + const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args shape = { n, k }; + const struct kai_matmul_pack_rhs_uker_rhs_packed_stride_args stride = api.get_rhs_packed_stride(&config, &shape); + return api.get_rhs_packed_size(&config, &shape, &stride); +} + +static void q4_k_rhs_pack( + size_t num_groups, size_t n, size_t k, size_t nr, size_t /*kr*/, size_t /*sr*/, size_t bl, + size_t rhs_stride, const void * rhs, const void * bias, const void * scale, + void * rhs_packed, size_t extra_bytes, const void * params) { + GGML_ASSERT(num_groups == 1); + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(bias == nullptr); + GGML_ASSERT(scale == nullptr); + GGML_ASSERT(extra_bytes == 0); + GGML_ASSERT(params == nullptr); + + const struct kai_matmul_pack_rhs_uker_api api = + kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme(); + const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config(); + GGML_ASSERT(api.get_step(&config).n == nr); + const struct kai_matmul_pack_rhs_uker_rhs_dim_args rhs_shape = { n, k }; + const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args packed_shape = { n, k }; + + struct kai_matmul_pack_rhs_uker_args args = {}; + args.flags = 0; + args.shape = { n, k }; + args.operand.rhs.ptr = rhs; + args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape); + if (rhs_stride != 0) { + GGML_ASSERT(rhs_stride >= args.operand.rhs.stride.n); + args.operand.rhs.stride.n = rhs_stride; + } + args.operand.rhs_packed.ptr = rhs_packed; + args.operand.rhs_packed.stride = api.get_rhs_packed_stride(&config, &packed_shape); + api.run(&config, &args); } static ggml_kleidiai_kernels gemm_gemv_kernels[] = { @@ -338,7 +432,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>, @@ -359,7 +452,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>, @@ -371,10 +463,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32_neon>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon, - /* .to_float = */ dequantize_row_qsi4c32ps1s0scalef16, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>, /* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>, }, /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16, @@ -392,7 +481,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>, /* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>, @@ -412,7 +500,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_lhs_offset_ex = */ nullptr, /* .get_rhs_packed_offset_ex = */ nullptr, /* .run_kernel_ex = */ nullptr, @@ -424,10 +511,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_bf16p2vlx2_f32_sme>, }, /* .rhs_info = */ { - /* .packed_stride = */ nullptr, - /* .to_float = */ nullptr, /* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>, - /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>, /* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>, }, /* .required_cpu = */ CPU_FEATURE_SME2, @@ -446,7 +530,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>, @@ -466,7 +549,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>, @@ -478,10 +560,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, /* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -499,7 +578,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>, @@ -519,7 +597,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>, @@ -531,10 +608,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, /* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, }, /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, @@ -553,7 +627,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>, @@ -573,7 +646,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>, @@ -585,13 +657,10 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, /* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, }, - /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, + /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD, /* .lhs_type = */ GGML_TYPE_F32, /* .rhs_type = */ GGML_TYPE_Q4_0, /* .op_type = */ GGML_TYPE_F32, @@ -606,7 +675,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>, @@ -626,7 +694,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>, @@ -638,10 +705,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, /* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, }, /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, @@ -659,7 +723,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>, @@ -679,7 +742,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>, @@ -691,10 +753,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, /* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -706,6 +765,59 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { { /* Sentinel */ } }; +static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = { + { + /* SME2 GEMM */ + { + /* .get_m_step = */ q4_k_gemm_adapter::get_m_step, + /* .get_n_step = */ q4_k_gemm_adapter::get_n_step, + /* .get_mr = */ q4_k_gemm_adapter::get_mr, + /* .get_nr = */ q4_k_gemm_adapter::get_nr, + /* .get_kr = */ q4_k_gemm_adapter::get_kr, + /* .get_sr = */ q4_k_gemm_adapter::get_sr, + /* .get_dst_offset = */ q4_k_gemm_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ q4_k_gemm_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ q4_k_gemm_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ q4_k_gemm_adapter::run, + }, + { + /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon, + /* .get_packed_offset_ex = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_pack_f16pmrx2_f32_neon>, + /* .packed_size_ex = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_pack_f16pmrx2_f32_neon>, + /* .pack_func_ex = */ &lhs_pack_void_fn10<kai_run_lhs_pack_f16pmrx2_f32_neon>, + }, + /* SME2 GEMV */ + { + /* .get_m_step = */ q4_k_gemv_adapter::get_m_step, + /* .get_n_step = */ q4_k_gemv_adapter::get_n_step, + /* .get_mr = */ q4_k_gemv_adapter::get_mr, + /* .get_nr = */ q4_k_gemv_adapter::get_nr, + /* .get_kr = */ q4_k_gemv_adapter::get_kr, + /* .get_sr = */ q4_k_gemv_adapter::get_sr, + /* .get_dst_offset = */ q4_k_gemv_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ q4_k_gemv_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ q4_k_gemv_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ q4_k_gemv_adapter::run, + }, + { + /* .get_offset = */ q4_k_lhs_get_offset, + /* .get_packed_offset_ex = */ q4_k_lhs_get_packed_offset, + /* .packed_size_ex = */ q4_k_lhs_get_packed_size, + /* .pack_func_ex = */ q4_k_lhs_pack, + }, + { + /* .packed_size_ex = */ q4_k_rhs_get_packed_size, + /* .pack_func_ex = */ q4_k_rhs_pack, + /* .repack_mode = */ RHS_REPACK_SINGLE_ONLY, + }, + /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_Q4_K, + /* .op_type = */ GGML_TYPE_F32, + }, + { /* Sentinel */ } +}; + static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { { /* SME2 GEMM */ @@ -717,7 +829,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>, /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>, @@ -737,7 +848,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>, /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>, @@ -749,10 +859,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, /* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, }, /* .required_cpu = */ CPU_FEATURE_SME2, @@ -770,7 +877,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>, /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>, @@ -790,7 +896,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>, /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>, @@ -802,10 +907,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, /* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, }, /* .required_cpu = */ CPU_FEATURE_SME, @@ -813,6 +915,54 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .rhs_type = */ GGML_TYPE_Q8_0, /* .op_type = */ GGML_TYPE_F32, }, + { + /* SVE I8MM GEMM */ + { + /* .get_m_step = */ sve_q8_gemm_adapter::get_m_step, + /* .get_n_step = */ sve_q8_gemm_adapter::get_n_step, + /* .get_mr = */ sve_q8_gemm_adapter::get_mr, + /* .get_nr = */ sve_q8_gemm_adapter::get_nr, + /* .get_kr = */ sve_q8_gemm_adapter::get_kr, + /* .get_sr = */ sve_q8_gemm_adapter::get_sr, + /* .get_dst_offset = */ sve_q8_gemm_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ sve_q8_gemm_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ sve_q8_gemm_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ sve_q8_gemm_adapter::run, + }, + /* .gemm_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32, + /* .get_packed_offset_ex = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>, + /* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>, + /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>, + }, + /* SVE DOT GEMV */ + { + /* .get_m_step = */ sve_q8_gemv_adapter::get_m_step, + /* .get_n_step = */ sve_q8_gemv_adapter::get_n_step, + /* .get_mr = */ sve_q8_gemv_adapter::get_mr, + /* .get_nr = */ sve_q8_gemv_adapter::get_nr, + /* .get_kr = */ sve_q8_gemv_adapter::get_kr, + /* .get_sr = */ sve_q8_gemv_adapter::get_sr, + /* .get_dst_offset = */ sve_q8_gemv_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ sve_q8_gemv_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ sve_q8_gemv_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ sve_q8_gemv_adapter::run, + }, + /* .gemv_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32, + /* .get_packed_offset_ex = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>, + /* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>, + /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>, + }, + /* .rhs_info = */ { + /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, + /* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, + }, + /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_Q8_0, + /* .op_type = */ GGML_TYPE_F32, + }, { /* I8MM GEMM */ { @@ -823,7 +973,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>, /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>, @@ -843,7 +992,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>, @@ -855,10 +1003,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, /* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, }, /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, @@ -876,7 +1021,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>, @@ -896,7 +1040,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>, /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>, @@ -908,10 +1051,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, - /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, /* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -933,7 +1073,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>, /* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>, @@ -953,7 +1092,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>, /* .run_kernel_ex = */ &kernel_run_lhs_stride_fn10<kai_run_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>, @@ -965,10 +1103,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .pack_func_ex = */ nullptr, }, /* .rhs_info = */ { - /* .packed_stride = */ nullptr, - /* .to_float = */ nullptr, /* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>, - /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>, /* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>, }, /* .required_cpu = */ CPU_FEATURE_SME2, @@ -986,7 +1121,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>, /* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>, @@ -1006,7 +1140,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_lhs_offset_ex = */ nullptr, /* .get_rhs_packed_offset_ex = */ nullptr, /* .run_kernel_ex = */ nullptr, @@ -1018,10 +1151,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>, }, /* .rhs_info = */ { - /* .packed_stride = */ nullptr, - /* .to_float = */ nullptr, /* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>, - /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>, /* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>, }, /* .required_cpu = */ CPU_FEATURE_SME, @@ -1049,8 +1179,23 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, c return false; }; - if (tensor->src[0]->type == GGML_TYPE_Q8_0) { - try_table(gemm_gemv_kernels_q8); + if (tensor->src[0]->type == GGML_TYPE_Q4_K) { + try_table(gemm_gemv_kernels_q4_k); + } else if (tensor->src[0]->type == GGML_TYPE_Q8_0 || tensor->src[0]->type == GGML_TYPE_Q6_K) { + const ggml_type original_type = tensor->src[0]->type; + if (original_type == GGML_TYPE_Q6_K) { + for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) { + if ((cpu_features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu && + gemm_gemv_kernels_q8[i].lhs_type == tensor->src[1]->type && + gemm_gemv_kernels_q8[i].rhs_type == GGML_TYPE_Q8_0 && + gemm_gemv_kernels_q8[i].op_type == tensor->type) { + kernel = &gemm_gemv_kernels_q8[i]; + break; + } + } + } else { + try_table(gemm_gemv_kernels_q8); + } } else if (tensor->src[0]->type == GGML_TYPE_F32) { try_table(ggml_kleidiai_kernels_f32); } else { @@ -1075,6 +1220,19 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features) return kernels; } +ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features) { + ggml_kleidiai_kernels * kernels = nullptr; + + for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q4_k) - 1; ++i) { + if ((features & gemm_gemv_kernels_q4_k[i].required_cpu) == gemm_gemv_kernels_q4_k[i].required_cpu) { + kernels = &gemm_gemv_kernels_q4_k[i]; + break; + } + } + + return kernels; +} + ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) { ggml_kleidiai_kernels * kernels = nullptr; diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.h index 1da8610ea..ac5d5d302 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.h +++ b/ggml/src/ggml-cpu/kleidiai/kernels.h @@ -13,7 +13,8 @@ enum cpu_feature { CPU_FEATURE_SVE = 4, CPU_FEATURE_SME = 8, CPU_FEATURE_SME2 = 16, - CPU_FEATURE_FP16 = 32 + CPU_FEATURE_FP16 = 32, + CPU_FEATURE_SVEI8MM = 64 }; inline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) { @@ -33,7 +34,6 @@ struct kernel_info { size_t (*get_sr)(void); size_t (*get_dst_offset)(size_t m_idx, size_t n_idx, size_t stride); - size_t (*get_dst_size)(size_t m, size_t n); size_t (*get_lhs_offset_ex)(size_t m_idx, size_t k, size_t bl); @@ -59,21 +59,12 @@ struct lhs_packing_info { enum rhs_repack_mode { RHS_REPACK_PER_KERNEL, - RHS_REPACK_SHARED, RHS_REPACK_SINGLE_ONLY, }; struct rhs_packing_info { - size_t (*packed_stride)(size_t k, size_t nr, size_t kr, size_t bl); - - void (*to_float)(const void *packed_data, int32_t row_idx, int64_t nc, float *out, - size_t nr_pack, size_t packed_row_stride, size_t kr, size_t bl, - size_t num_bytes_multiplier); - size_t (*packed_size_ex)(size_t n, size_t k, size_t nr, size_t kr, size_t bl); - size_t (*packed_stride_ex)(size_t k, size_t nr, size_t kr, size_t bl); - void (*pack_func_ex)(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl, size_t rhs_stride, const void * rhs, const void * bias, const void * scale, void * rhs_packed, size_t extra_bytes, const void * params); @@ -97,5 +88,6 @@ struct ggml_kleidiai_kernels { ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor); ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features); +ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features); ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features); ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_f32(cpu_feature features); diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp index 1384ca927..57a4a5662 100644 --- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp @@ -43,6 +43,7 @@ #include "ggml-impl.h" #include "ggml-feats.h" #include "ggml-backend-impl.h" +#include "ggml-quants.h" #include "ggml-threading.h" #include "traits.h" @@ -57,16 +58,20 @@ static constexpr int GGML_KLEIDIAI_MAX_KERNEL_SLOTS = 2; static constexpr uint32_t GGML_KLEIDIAI_PACK_MAGIC = 0x4b4c4149; // "KLAI" static constexpr uint16_t GGML_KLEIDIAI_PACK_VERSION = 1; static constexpr size_t GGML_KLEIDIAI_PACK_ALIGN = 64; +static constexpr size_t GGML_KLEIDIAI_Q4_K_BL = QK_K / 8; + +static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks"); struct ggml_kleidiai_context { cpu_feature features; ggml_kleidiai_kernels * kernels_q4; + ggml_kleidiai_kernels * kernels_q4k; ggml_kleidiai_kernels * kernels_q8; ggml_kleidiai_kernels * kernels_f32; int sme_thread_cap; // <= 0 means "SME disabled/unknown" int thread_hint; // <= 0 means "no hint" int chunk_multiplier; -} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 }; +} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, nullptr, 0, -1, 4 }; static inline bool is_sme_family(cpu_feature f) { return (f & (CPU_FEATURE_SME | CPU_FEATURE_SME2)) != CPU_FEATURE_NONE; @@ -314,8 +319,10 @@ static void init_kleidiai_context(void) { size_t detected_smcus = 0; + // The fixed-width SVE kernels require 32-byte vectors, including the Q8 shared RHS layout. ctx.features = (runtime_feat.has_dotprod ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) | (runtime_feat.has_i8mm ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) | + (runtime_feat.has_svei8mm ? CPU_FEATURE_SVEI8MM : CPU_FEATURE_NONE) | (runtime_feat.has_fp16 ? CPU_FEATURE_FP16 : CPU_FEATURE_NONE) | (runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE); @@ -387,6 +394,7 @@ static void init_kleidiai_context(void) { // Kernel selection ctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features); + ctx.kernels_q4k = ggml_kleidiai_select_kernels_q4_k(ctx.features); ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features); ctx.kernels_f32 = ggml_kleidiai_select_kernels_f32(ctx.features); @@ -402,6 +410,12 @@ static void init_kleidiai_context(void) { GGML_LOG_INFO("kleidiai: primary q8 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q8->required_cpu)); } + if (!ctx.kernels_q4k) { + GGML_LOG_INFO("kleidiai: no compatible q4_k kernels found for CPU features mask %d\n", (int) ctx.features); + } else { + GGML_LOG_INFO("kleidiai: primary q4_k kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q4k->required_cpu)); + } + if (!ctx.kernels_f32) { GGML_LOG_INFO("kleidiai: no compatible f32 kernels found for CPU features mask %d\n", (int)ctx.features); } else { @@ -410,6 +424,7 @@ static void init_kleidiai_context(void) { const bool has_selected_sme_family_kernel = (ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) || + (ctx.kernels_q4k && is_sme_family(ctx.kernels_q4k->required_cpu)) || (ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) || (ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu)); ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0; @@ -486,6 +501,8 @@ static inline kleidiai_block_args kleidiai_get_block_args(ggml_type rhs_type) { switch (rhs_type) { case GGML_TYPE_Q4_0: return { QK4_0, QK4_0, QK4_0 }; + case GGML_TYPE_Q4_K: + return { GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL }; case GGML_TYPE_Q8_0: return { 0, 0, QK8_0 }; default: @@ -556,6 +573,10 @@ static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4() { return ctx.kernels_q4; } +static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4k() { + return ctx.kernels_q4k; +} + static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q8() { return ctx.kernels_q8; } @@ -610,6 +631,12 @@ static int kleidiai_collect_q4_chain(std::array<ggml_kleidiai_kernels *, GGML_KL [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_0(mask); }); } +static int kleidiai_collect_q4k_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) { + ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q4k(); + return kleidiai_collect_kernel_chain_common(primary, ctx.features, out, + [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_k(mask); }); +} + static int kleidiai_collect_q8_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) { ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q8(); return kleidiai_collect_kernel_chain_common(primary, ctx.features, out, @@ -662,8 +689,10 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t n = op->src[0]->ne[1]; const size_t m = op->src[1]->ne[1]; - if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) { - const size_t qk = (op->src[0]->type == GGML_TYPE_Q4_0) ? QK4_0 : QK8_0; + if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K) { + const kleidiai_block_args block_args = kleidiai_get_block_args(kernel_chain[0]->rhs_type); + const size_t qk = block_args.pack_bl; size_t cursor = 0; bool any_slot = false; @@ -789,17 +818,14 @@ class tensor_traits : public ggml::cpu::tensor_traits { bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * dst) override { if (dst->op == GGML_OP_MUL_MAT) { - if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) { + if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K || + dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K) { return compute_forward_qx(params, dst); } else if (dst->src[0]->type == GGML_TYPE_F32) { return compute_forward_f32(params, dst); } else if (dst->src[0]->type == GGML_TYPE_F16) { return compute_forward_fp16(params, dst); } - } else if (dst->op == GGML_OP_GET_ROWS) { - if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) { - return compute_forward_get_rows(params, dst); - } } return false; } @@ -870,9 +896,12 @@ class tensor_traits : public ggml::cpu::tensor_traits { const bool disable_chunking = ggml_is_numa(); GGML_ASSERT(n <= (size_t) INT_MAX); - for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) { - const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2]; - uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2]; + const int64_t batch_size = ne12 * ne13; + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + const int64_t i12 = batch_idx % ne12; + const int64_t i13 = batch_idx / ne12; + const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + i12 * src1->nb[2] + i13 * src1->nb[3]; + uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + i12 * dst->nb[2] + i13 * dst->nb[3]; if (!use_gemv) { const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr); @@ -947,7 +976,7 @@ class tensor_traits : public ggml::cpu::tensor_traits { current_col = ggml_threadpool_chunk_add(params->threadpool, (int) chunk_cols); } - if (batch_idx != ne12 - 1) { + if (batch_idx != batch_size - 1) { ggml_barrier(params->threadpool); } } @@ -1107,7 +1136,8 @@ class tensor_traits : public ggml::cpu::tensor_traits { } bool compute_forward_qx(struct ggml_compute_params * params, struct ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0); + GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K || + dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K); const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -1265,7 +1295,9 @@ class tensor_traits : public ggml::cpu::tensor_traits { } } - int sme_cap = kleidiai_sme_thread_cap(); + const bool sme_single_only = sme_slot != -1 && + runtime[sme_slot].kernels->rhs_info.repack_mode == RHS_REPACK_SINGLE_ONLY; + int sme_cap = sme_single_only ? nth_total : kleidiai_sme_thread_cap(); if (sme_cap < 0) { sme_cap = nth_total; } @@ -1301,23 +1333,14 @@ class tensor_traits : public ggml::cpu::tensor_traits { threads_remaining -= share; } - if (threads_remaining > 0) { - const int fallback_slot = (sme_slot != -1) ? sme_slot : 0; - runtime[fallback_slot].assigned_threads += threads_remaining; - threads_remaining = 0; - } - int thread_cursor = 0; for (int i = 0; i < runtime_count; ++i) { runtime[i].thread_begin = thread_cursor; thread_cursor += runtime[i].assigned_threads; runtime[i].thread_end = thread_cursor; } - - if (thread_cursor < nth_total && runtime_count > 0) { - runtime[runtime_count - 1].assigned_threads += nth_total - thread_cursor; - runtime[runtime_count - 1].thread_end = nth_total; - } + GGML_ASSERT(thread_cursor > 0); + const int active_threads = thread_cursor; int local_slot = -1; int local_ith = 0; @@ -1328,10 +1351,6 @@ class tensor_traits : public ggml::cpu::tensor_traits { break; } } - if (local_slot == -1) { - return false; - } - const size_t k = ne00; const size_t m = ne11; const size_t n = ne01; @@ -1363,7 +1382,7 @@ class tensor_traits : public ggml::cpu::tensor_traits { const bool disable_chunking = ggml_is_numa(); const size_t chunk_multiplier = std::max(1, ctx.chunk_multiplier); - const size_t chunk_divisor = (nth_total == 1 || disable_chunking) ? (size_t)nth_total : (size_t)nth_total * chunk_multiplier; + const size_t chunk_divisor = (active_threads == 1 || disable_chunking) ? (size_t) active_threads : (size_t) active_threads * chunk_multiplier; size_t chunk_cols = align_up(std::max<size_t>(1, ceil_div_size(n, chunk_divisor)), common_step); if (chunk_cols == 0) { chunk_cols = common_step; @@ -1397,7 +1416,7 @@ class tensor_traits : public ggml::cpu::tensor_traits { const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2]; uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2]; - if (runtime[local_slot].assigned_threads > 0) { + if (local_slot != -1) { runtime_slot & slot = runtime[local_slot]; const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr); int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads; @@ -1438,28 +1457,30 @@ class tensor_traits : public ggml::cpu::tensor_traits { } if (ith_total == 0) { - ggml_threadpool_chunk_set(params->threadpool, nth_total); + ggml_threadpool_chunk_set(params->threadpool, active_threads); } // Publishes both LHS packing and the initialized dynamic chunk queue. ggml_barrier(params->threadpool); - runtime_slot & slot = runtime[local_slot]; - int current_chunk = ith_total; - while (current_chunk < nchunk) { - const size_t global_start = (size_t)current_chunk * chunk_cols; - if (global_start >= n) { - break; - } + if (local_slot != -1) { + runtime_slot & slot = runtime[local_slot]; + int current_chunk = ith_total; + while (current_chunk < nchunk) { + const size_t global_start = (size_t)current_chunk * chunk_cols; + if (global_start >= n) { + break; + } - const size_t cols = std::min(chunk_cols, n - global_start); - if (cols > 0) { - // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths; - // only non-tail chunks are guaranteed to be n_step-aligned. - run_chunk(slot, global_start, cols, dst_batch_base); - } + const size_t cols = std::min(chunk_cols, n - global_start); + if (cols > 0) { + // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths; + // only non-tail chunks are guaranteed to be n_step-aligned. + run_chunk(slot, global_start, cols, dst_batch_base); + } - current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1); + current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1); + } } if (batch_idx != ne12 - 1) { @@ -1470,101 +1491,11 @@ class tensor_traits : public ggml::cpu::tensor_traits { return true; } - bool compute_forward_get_rows(struct ggml_compute_params * params, struct ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_TENSOR_BINARY_OP_LOCALS - - const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data); - const bool has_header = kleidiai_is_weight_header_valid(header); - - std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain; - const bool want_q8 = src0->type == GGML_TYPE_Q8_0; - const int chain_count = want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); - - ggml_kleidiai_kernels * kernels = nullptr; - const uint8_t * packed_base = static_cast<const uint8_t *>(src0->data); - - if (has_header && chain_count > 0) { - int select_slot = 0; - if (select_slot >= header->slot_count) { - select_slot = header->slot_count - 1; - } - if (select_slot >= 0 && select_slot < chain_count) { - kernels = kernel_chain[select_slot]; - const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, select_slot); - if (slot_ptr) { - packed_base = slot_ptr; - } - } - } - - if (!kernels && chain_count > 0) { - kernels = kernel_chain[0]; - if (has_header) { - const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, 0); - if (slot_ptr) { - packed_base = slot_ptr; - } - } - } - - if (!kernels) { - return false; - } - - rhs_packing_info * rhs_info = &kernels->rhs_info; - kernel_info * kernel = &kernels->gemm; - if (!rhs_info->to_float || !kernel->get_nr) { - return false; - } - - const int64_t nc = ne00; - const int64_t nr = ggml_nelements(src1); - - const ggml_type rhs_type = kernels->rhs_type; - size_t block_len = 0; - size_t num_bytes_multiplier = 0; - if (rhs_type == GGML_TYPE_Q4_0) { - block_len = QK4_0; - num_bytes_multiplier = sizeof(uint16_t); - } else if (rhs_type == GGML_TYPE_Q8_0) { - block_len = QK8_0; - num_bytes_multiplier = sizeof(float); - } else { - return false; - } - - const size_t block_rows = kernel->get_nr(); - const size_t kr = kernel->get_kr(); - - const size_t packed_stride = rhs_info->packed_stride(nc, block_rows, kr, block_len); - - const int ith = params->ith; - const int nth = params->nth; - - const int dr = (nr + nth - 1) / nth; - const int ir0 = dr * ith; - const int ir1 = MIN(ir0 + dr, nr); - - for (int64_t i = ir0; i < ir1; ++i) { - GGML_ASSERT(src1->type == GGML_TYPE_I32); - int64_t row_idx = ((const int32_t *)src1->data)[i]; - GGML_ASSERT(row_idx >= 0 && row_idx < src0->ne[1]); - - float *out = (float *)((char *)dst->data + i * nb1); - rhs_info->to_float(packed_base, row_idx, nc, out, block_rows, packed_stride, kr, block_len, num_bytes_multiplier); - } - - return true; - } - public: int repack(struct ggml_tensor * tensor, const void * data, size_t data_size) { - GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_F32); + GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q4_K || + tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K || + tensor->type == GGML_TYPE_F32); const size_t n = tensor->ne[1]; const size_t k = tensor->ne[0]; @@ -1582,16 +1513,19 @@ class tensor_traits : public ggml::cpu::tensor_traits { cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN); std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain; - const bool want_q8 = tensor->type == GGML_TYPE_Q8_0; + const bool want_q4k = tensor->type == GGML_TYPE_Q4_K; + const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K; const bool want_f32 = tensor->type == GGML_TYPE_F32; const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain) : want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); + : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain) + : kleidiai_collect_q4_chain(kernel_chain); const bool allow_fallback = kleidiai_pack_fallback_allowed(); std::vector<int8_t> qdata; std::vector<float> scales; std::vector<float> bias; + std::vector<float> row_f32; if (want_q8 && slot_total > 0) { qdata.resize(n * k, 0); @@ -1600,21 +1534,38 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t row_stride = tensor->nb[1]; const size_t k_blocks = (k + QK8_0 - 1) / QK8_0; + if (tensor->type == GGML_TYPE_Q6_K) { + if (k % QK_K != 0 || k % QK8_0 != 0) { + return -1; + } + row_f32.resize(k); + } + for (size_t row = 0; row < n; ++row) { - const auto * row_blocks = reinterpret_cast<const block_q8_0 *>( - static_cast<const uint8_t *>(data) + row * row_stride); + const uint8_t * row_data = static_cast<const uint8_t *>(data) + row * row_stride; + + if (tensor->type == GGML_TYPE_Q6_K) { + dequantize_row_q6_K(reinterpret_cast<const block_q6_K *>(row_data), row_f32.data(), k); + } float max_abs = 0.0f; - for (size_t block = 0; block < k_blocks; ++block) { - const block_q8_0 & blk = row_blocks[block]; - const float d = GGML_FP16_TO_FP32(blk.d); - for (size_t l = 0; l < QK8_0; ++l) { - const size_t linear_idx = block * QK8_0 + l; - if (linear_idx >= k) { - break; + if (tensor->type == GGML_TYPE_Q6_K) { + for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) { + max_abs = std::max(max_abs, std::fabs(row_f32[linear_idx])); + } + } else { + const block_q8_0 * row_blocks = reinterpret_cast<const block_q8_0 *>(row_data); + for (size_t block = 0; block < k_blocks; ++block) { + const block_q8_0 & blk = row_blocks[block]; + const float d = GGML_FP16_TO_FP32(blk.d); + for (size_t l = 0; l < QK8_0; ++l) { + const size_t linear_idx = block * QK8_0 + l; + if (linear_idx >= k) { + break; + } + const float value = d * static_cast<float>(blk.qs[l]); + max_abs = std::max(max_abs, std::fabs(value)); } - const float value = d * static_cast<float>(blk.qs[l]); - max_abs = std::max(max_abs, std::fabs(value)); } } @@ -1622,19 +1573,28 @@ class tensor_traits : public ggml::cpu::tensor_traits { scales[row] = scale; const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; - for (size_t block = 0; block < k_blocks; ++block) { - const block_q8_0 & blk = row_blocks[block]; - const float d = GGML_FP16_TO_FP32(blk.d); - for (size_t l = 0; l < QK8_0; ++l) { - const size_t linear_idx = block * QK8_0 + l; - if (linear_idx >= k) { - break; - } - const float value = d * static_cast<float>(blk.qs[l]); - int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0; + if (tensor->type == GGML_TYPE_Q6_K) { + for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) { + int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(row_f32[linear_idx] * inv_scale)) : 0; q = std::clamp(q, -127, 127); qdata[row * k + linear_idx] = static_cast<int8_t>(q); } + } else { + const block_q8_0 * row_blocks = reinterpret_cast<const block_q8_0 *>(row_data); + for (size_t block = 0; block < k_blocks; ++block) { + const block_q8_0 & blk = row_blocks[block]; + const float d = GGML_FP16_TO_FP32(blk.d); + for (size_t l = 0; l < QK8_0; ++l) { + const size_t linear_idx = block * QK8_0 + l; + if (linear_idx >= k) { + break; + } + const float value = d * static_cast<float>(blk.qs[l]); + int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0; + q = std::clamp(q, -127, 127); + qdata[row * k + linear_idx] = static_cast<int8_t>(q); + } + } } } } @@ -1658,8 +1618,14 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t kr = kernel->get_kr(); const size_t sr = kernel->get_sr(); const ggml_type rhs_type = kernels->rhs_type; + if (rhs_type == GGML_TYPE_Q8_0 && (kernels->required_cpu & CPU_FEATURE_SVE)) { + GGML_ASSERT(nr == kernels->gemv.get_nr()); + GGML_ASSERT(kr == kernels->gemv.get_kr()); + GGML_ASSERT(sr == kernels->gemv.get_sr()); + } const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : + rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL : rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX; if (block_len == SIZE_MAX) { continue; @@ -1677,6 +1643,9 @@ class tensor_traits : public ggml::cpu::tensor_traits { rhs_info->pack_func_ex(1, n, k, nr, kr, sr, QK4_0, 0, static_cast<const uint8_t *>(data), nullptr, nullptr, dst_ptr, 0, ¶ms); + } else if (rhs_type == GGML_TYPE_Q4_K) { + rhs_info->pack_func_ex(1, n, k, nr, kr, sr, GGML_KLEIDIAI_Q4_K_BL, tensor->nb[1], + data, nullptr, nullptr, dst_ptr, 0, nullptr); } else if (rhs_type == GGML_TYPE_Q8_0) { struct kai_rhs_pack_qsi8cx_params params; params.lhs_zero_point = 1; @@ -1719,7 +1688,6 @@ static enum ggml_status ggml_backend_cpu_kleidiai_buffer_init_tensor(ggml_backen tensor->extra = (void *) ggml::cpu::kleidiai::get_tensor_traits(buffer, tensor); return GGML_STATUS_SUCCESS; - GGML_UNUSED(buffer); } static void ggml_backend_cpu_kleidiai_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, @@ -1762,7 +1730,9 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alignment(ggml_backend_b static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) { GGML_UNUSED(buft); - if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_F32) { + if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q4_K && + tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_Q6_K && + tensor->type != GGML_TYPE_F32) { return ggml_nbytes(tensor); } @@ -1773,11 +1743,13 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN); std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain; - const bool want_q8 = tensor->type == GGML_TYPE_Q8_0; + const bool want_q4k = tensor->type == GGML_TYPE_Q4_K; + const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K; const bool want_f32 = tensor->type == GGML_TYPE_F32; const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain) : want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); + : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain) + : kleidiai_collect_q4_chain(kernel_chain); const bool allow_fallback = kleidiai_pack_fallback_allowed(); size_t slot_count = 0; @@ -1797,6 +1769,7 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ const ggml_type rhs_type = kernels->rhs_type; const size_t block_len = rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : + rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL : rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX; if (block_len == SIZE_MAX) { @@ -1818,6 +1791,11 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ namespace ggml::cpu::kleidiai { class extra_buffer_type : ggml::cpu::extra_buffer_type { bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override { + // GET_ROWS needs the original weights in a native buffer. + if (op->op != GGML_OP_MUL_MAT) { + return false; + } + std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain; const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain); const bool src0_is_kleidiai = @@ -1826,8 +1804,9 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { op->src[0]->buffer->buft->context == this && slot_total > 0; - if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) && - (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_F32) && + if ((op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K || + op->src[0]->type == GGML_TYPE_F32) && src0_is_kleidiai) { if (op->src[0]->type == GGML_TYPE_Q4_0 && ctx.kernels_q4 == nullptr) { return false; @@ -1835,6 +1814,12 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { if (op->src[0]->type == GGML_TYPE_Q8_0 && ctx.kernels_q8 == nullptr) { return false; } + if (op->src[0]->type == GGML_TYPE_Q6_K && ctx.kernels_q8 == nullptr) { + return false; + } + if (op->src[0]->type == GGML_TYPE_Q4_K && ctx.kernels_q4k == nullptr) { + return false; + } if (op->src[0]->type == GGML_TYPE_F32 && ctx.kernels_f32 == nullptr) { return false; } @@ -1842,8 +1827,18 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { return false; } - if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) { - if ((op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_I32) && + if (op->src[0]->type == GGML_TYPE_Q4_K) { + if (op->op != GGML_OP_MUL_MAT || op->src[1]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) { + return false; + } + return ggml_ne(op->src[1], 2) == 1 && ggml_ne(op->src[1], 3) == 1 && + op->src[1]->nb[0] == sizeof(float) && + (op->src[1]->ne[1] == 1 || op->src[1]->nb[1] == op->src[1]->ne[0] * sizeof(float)); + } + + if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || + op->src[0]->type == GGML_TYPE_Q6_K) { + if (op->src[1]->type == GGML_TYPE_F32 && ggml_ne(op->src[1], 3) == 1) { return true; } @@ -1861,21 +1856,22 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { } ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override { - if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) { + if (op->op == GGML_OP_MUL_MAT) { if (op->src[0]->buffer && op->src[0]->buffer->buft->context == this) { return (ggml::cpu::tensor_traits *) op->src[0]->extra; } else { - // KleidiAI only has kernels for Q4_0 and Q8_0. For a quantized weight of any + // KleidiAI only has kernels for Q4_0, Q4_K, Q8_0 and Q6_K. For a quantized weight of any // other type (K-quants, IQ) it declines the op and returns nullptr below, so // KleidiAI does not accelerate it. Another CPU backend may still take the op, // and this can run during graph planning, so the message says what KleidiAI // did rather than what ends up executing. Warn once per process. if (ggml_is_quantized(op->src[0]->type) && - op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q8_0) { + op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q4_K && + op->src[0]->type != GGML_TYPE_Q8_0 && op->src[0]->type != GGML_TYPE_Q6_K) { static std::atomic<bool> warned(false); if (!warned.exchange(true)) { GGML_LOG_WARN("kleidiai: no kernel for tensor type %s, not accelerated by KleidiAI " - "(kernels available for Q4_0 and Q8_0)\n", + "(kernels available for Q4_0, Q4_K, Q8_0 and Q6_K)\n", ggml_type_name(op->src[0]->type)); } } diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index d28840a91..384f7dbe2 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -69,7 +69,7 @@ #define VECTOR_REGISTERS 16 #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) #define LMUL 4 #endif @@ -180,7 +180,7 @@ inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <> inline vfloat32m1_t madd(vfloat32m1_t a, vfloat32m1_t b, vfloat32m1_t c) { return __riscv_vfmacc_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1()); } @@ -277,7 +277,7 @@ inline float hsum(__m512 x) { } #endif // __AVX512F__ -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) inline float hsum(vfloat32m1_t x) { return __riscv_vfmv_f_s_f32m1_f32( __riscv_vfredusum_vs_f32m1_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m1())); @@ -326,6 +326,12 @@ template <> inline float32x4_t load(const ggml_fp16_t * p) { return vec_xl(0, (const float *)(tmp)); } +template <> inline float32x4_t load(const ggml_bf16_t * p) { + uint64_t bits; + memcpy(&bits, p, sizeof(bits)); + const uint64x2_t v_bits = { bits, bits }; + return GGML_BF16_TO_F32_HI((GGML_BF16_VEC) v_bits); +} template <> inline float32x4_t load(const float * p) { return vec_xl(0, p); } @@ -458,7 +464,7 @@ template <> inline __m512bh load_partial(const ggml_bf16_t *p, int n) { #endif #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <> inline vfloat32m1_t load(const float *p) { return __riscv_vle32_v_f32m1(p, __riscv_vsetvlmax_e32m1()); } @@ -503,7 +509,7 @@ template <> inline vbfloat16m4_t load(const ggml_bf16_t *p) { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <typename T> T set_zero(); template <> inline vfloat32m1_t set_zero() { @@ -520,7 +526,7 @@ template <> inline vfloat32m8_t set_zero() { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <typename T> size_t vlmax() { if constexpr (std::is_same_v<T, vfloat32m1_t>) { return __riscv_vsetvlmax_e32m1(); } else if constexpr (std::is_same_v<T, vfloat32m2_t>) { return __riscv_vsetvlmax_e32m2(); } @@ -732,7 +738,7 @@ class tinyBLAS { const int64_t ldc; }; -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <typename D, typename V, typename TA, typename TB, typename TC> class tinyBLAS_RVV { public: @@ -1302,6 +1308,88 @@ class tinyBLAS_RVV { ////////////////////////////////////////////////////////////////////////////////////////// // QUANT ZERO MATRIX MULTIPLICATION +#if defined(__VXE__) || defined(__VXE2__) +template <typename TA> +class tinyBLAS_Q0_VXE { + static_assert(sizeof(TA::qs) == QK8_0, "VXE weight loads require 32 quantized bytes per block"); + + public: + tinyBLAS_Q0_VXE(int64_t k, + const TA * A, int64_t lda, + const block_q8_0 * B, int64_t ldb, + float * C, int64_t ldc, + int32_t ith, int32_t nth) + : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) { + } + + void matmul(int64_t m, int64_t n) { + constexpr int32_t tile_size = 2; + const int64_t row_tiles = m / tile_size; + const int64_t tiles = row_tiles * (n / tile_size); + const int64_t duty = (tiles + nth - 1) / nth; + const int64_t start = duty * ith; + const int64_t end = MIN(start + duty, tiles); + + for (int64_t job = start; job < end; ++job) { + const int64_t ii = tile_size * (job % row_tiles); + const int64_t jj = tile_size * (job / row_tiles); + float32x4_t v_sum[tile_size][tile_size] = {}; + + for (int64_t l = 0; l < k; ++l) { + int8x16_t v_xl[tile_size], v_xh[tile_size], v_yl[tile_size], v_yh[tile_size]; + float x_scale[tile_size], y_scale[tile_size]; + + for (int32_t i = 0; i < tile_size; ++i) { + const TA & x = A[lda * (ii + i) + l]; + const block_q8_0 & y = B[ldb * (jj + i) + l]; + + const int8_t * x_qs = x.qs; + const int8_t * y_qs = y.qs; + v_xl[i] = vec_xl(0, x_qs); + v_xh[i] = vec_xl(0, x_qs + QK8_0 / 2); + v_yl[i] = vec_xl(0, y_qs); + v_yh[i] = vec_xl(0, y_qs + QK8_0 / 2); + x_scale[i] = unhalf(x.d); + y_scale[i] = unhalf(y.d); + } + + for (int32_t j = 0; j < tile_size; ++j) { + for (int32_t i = 0; i < tile_size; ++i) { + const int32x4_t v_dot = dot_s8_wide(v_xl[i], v_yl[j]) + dot_s8_wide(v_xh[i], v_yh[j]); + const float32x4_t v_scale = vec_splats(x_scale[i] * y_scale[j]); + v_sum[j][i] = vec_madd(vec_float(v_dot), v_scale, v_sum[j][i]); + } + } + } + + for (int32_t j = 0; j < tile_size; ++j) { + for (int32_t i = 0; i < tile_size; ++i) { + C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]); + } + } + } + } + + private: + static int32x4_t dot_s8_wide(int8x16_t v_x, int8x16_t v_y) { + const int16x8_t v_even = vec_mule(v_x, v_y); + const int16x8_t v_odd = vec_mulo(v_x, v_y); + // Two (-128) * (-128) products can overflow a signed 16-bit sum. + return vec_unpackh(v_even) + vec_unpackl(v_even) + vec_unpackh(v_odd) + vec_unpackl(v_odd); + } + + const TA * const A; + const block_q8_0 * const B; + float * const C; + const int64_t k; + const int64_t lda; + const int64_t ldb; + const int64_t ldc; + const int32_t ith; + const int32_t nth; +}; +#endif + #if defined(__ARM_FEATURE_DOTPROD) template <typename TA> class tinyBLAS_Q0_ARM { @@ -3958,7 +4046,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) #if LMUL == 1 tinyBLAS_RVV<vfloat32m1_t, vfloat32m1_t, float, float, float> tb{ params, k, (const float *)A, lda, @@ -4006,6 +4094,14 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 (float *)C, ldc}; return tb.matmul(m, n); } +#elif defined(__VXE__) || defined(__VXE2__) + if (Btype == GGML_TYPE_BF16) { + tinyBLAS<4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k, + (const ggml_bf16_t *)A, lda, + (const ggml_bf16_t *)B, ldb, + (float *)C, ldc}; + return tb.matmul(m, n); + } #elif defined(__MMA__) if (k % 8) { return false; @@ -4054,6 +4150,13 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 (float *)C, ldc}; return tb.matmul(m, n); } + if (Btype == GGML_TYPE_F32) { + tinyBLAS<16, __m512, __m512, ggml_fp16_t, float, float> tb{ params, k, + (const ggml_fp16_t *)A, lda, + (const float *)B, ldb, + (float *)C, ldc}; + return tb.matmul(m, n); + } #elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__) if (Btype == GGML_TYPE_F16) { tinyBLAS<8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k, @@ -4062,6 +4165,13 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 (float *)C, ldc}; return tb.matmul(m, n); } + if (Btype == GGML_TYPE_F32) { + tinyBLAS<8, __m256, __m256, ggml_fp16_t, float, float> tb{ params, k, + (const ggml_fp16_t *)A, lda, + (const float *)B, ldb, + (float *)C, ldc}; + return tb.matmul(m, n); + } #elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER) if (n < 8) return false; @@ -4148,6 +4258,17 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; +#elif defined(__VXE__) || defined(__VXE2__) + if (m % 2 || n % 2) { + return false; + } + tinyBLAS_Q0_VXE<block_q8_0> tb{ + k, (const block_q8_0 *)A, lda, + (const block_q8_0 *)B, ldb, + (float *)C, ldc, + params->ith, params->nth}; + tb.matmul(m, n); + return true; #elif defined(__MMA__) //TO-DO: Remove this condition once gemv forwarding is enabled. if (n < 8 && n != 4) diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 16a916d20..eed5d69e3 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -537,26 +537,36 @@ void ggml_compute_forward_dup( switch (src0->type) { case GGML_TYPE_F16: { - /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp16_t>(params, dst); - else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_bf16_t>(params, dst); - else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_fp16_t, float >(params, dst); + /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp16_t>(params, dst); + else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_bf16_t>(params, dst); + else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_fp16_t, float >(params, dst); + else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp8_e4m3_t>(params, dst); else ggml_compute_forward_dup_to_q<ggml_fp16_t>(params, dst); } break; case GGML_TYPE_BF16: { - /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst); - else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_bf16_t>(params, dst); - else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_bf16_t, float >(params, dst); + /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst); + else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_bf16_t>(params, dst); + else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_bf16_t, float >(params, dst); + else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp8_e4m3_t>(params, dst); else ggml_compute_forward_dup_to_q<ggml_bf16_t>(params, dst); } break; case GGML_TYPE_F32: { - /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<float, ggml_fp16_t>(params, dst); - else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<float, ggml_bf16_t>(params, dst); - else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<float, float >(params, dst); - else if (dst->type == GGML_TYPE_I32) ggml_compute_forward_dup_flt<float, int32_t >(params, dst); + /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<float, ggml_fp16_t>(params, dst); + else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<float, ggml_bf16_t>(params, dst); + else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<float, float >(params, dst); + else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<float, ggml_fp8_e4m3_t>(params, dst); + else if (dst->type == GGML_TYPE_I32) ggml_compute_forward_dup_flt<float, int32_t >(params, dst); else ggml_compute_forward_dup_to_q<float>(params, dst); } break; + case GGML_TYPE_F8_E4M3: + { + /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, ggml_fp16_t>(params, dst); + else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, ggml_bf16_t>(params, dst); + else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, float >(params, dst); + else GGML_ABORT("not implemented"); + } break; case GGML_TYPE_I32: { if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<int32_t, float>(params, dst); @@ -605,8 +615,9 @@ static void ggml_compute_forward_add_q_f32( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - GGML_ASSERT(ggml_is_quantized(src0->type)); + GGML_ASSERT(ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3); GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(ne00 % ggml_blck_size(type) == 0); // rows per thread const int dr = (nr + nth - 1)/nth; @@ -636,8 +647,6 @@ static void ggml_compute_forward_add_q_f32( float * src1_row = (float *)((char *) src1->data + (i11*nb11 + i12*nb12 + i13*nb13)); void * dst_row = (void *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3)); - assert(ne00 % 32 == 0); - // unquantize row from src0 to temp buffer dequantize_row_q(src0_row, wdata, ne00); // add src1 @@ -673,6 +682,7 @@ void ggml_compute_forward_add( case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -743,12 +753,17 @@ static void ggml_compute_forward_add_id_f32( // src1 indices const int i11 = *(int32_t *) ((char *) src2->data + i1*nb20 + i2*nb21); - GGML_ASSERT(i11 >= 0 && i11 < ne11); + GGML_ASSERT(i11 == -1 || (i11 >= 0 && i11 < ne11)); - ggml_vec_add_f32(ne0, - (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), - (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), - (float *) ((char *) src1->data + i11*nb11)); + float * dst_row = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ); + const float * src0_row = (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01); + + if (i11 == -1) { + memcpy(dst_row, src0_row, ne0*sizeof(float)); + continue; + } + + ggml_vec_add_f32(ne0, dst_row, src0_row, (float *) ((char *) src1->data + i11*nb11)); } } @@ -950,7 +965,7 @@ static void ggml_compute_forward_add1_q_f32( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - GGML_ASSERT(ggml_is_quantized(src0->type)); + GGML_ASSERT(ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3); GGML_ASSERT(dst->type == src0->type); GGML_ASSERT(src1->type == GGML_TYPE_F32); @@ -1125,6 +1140,7 @@ void ggml_compute_forward_add1( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -1256,6 +1272,7 @@ void ggml_compute_forward_acc( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -1292,24 +1309,37 @@ static void ggml_compute_forward_sum_f32( } assert(ggml_is_scalar(dst)); - assert(src0->nb[0] == sizeof(float)); GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) - ggml_float sum = 0; - ggml_float row_sum = 0; + ggml_float sum = 0; - for (int64_t i03 = 0; i03 < ne03; i03++) { - for (int64_t i02 = 0; i02 < ne02; i02++) { - for (int64_t i01 = 0; i01 < ne01; i01++) { - ggml_vec_sum_f32_ggf(ne00, - &row_sum, - (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03)); - sum += row_sum; + if (nb00 == sizeof(float)) { + ggml_float row_sum = 0; + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = 0; i01 < ne01; i01++) { + ggml_vec_sum_f32_ggf(ne00, + &row_sum, + (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03)); + sum += row_sum; + } + } + } + } else { + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = 0; i01 < ne01; i01++) { + const char * row = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + for (int64_t i00 = 0; i00 < ne00; i00++) { + sum += (ggml_float) *(const float *) (row + i00*nb00); + } + } } } } + ((float *) dst->data)[0] = sum; } @@ -1896,213 +1926,87 @@ void ggml_compute_forward_repeat_back( } // ggml_compute_forward_concat -static void ggml_compute_forward_concat_any( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - GGML_ASSERT(ggml_is_contiguous_rows(src0)); - GGML_ASSERT(ggml_is_contiguous_rows(src1)); - - int64_t o[4] = {0, 0, 0, 0}; - - if (dim == 0) { - GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0); - GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0); - - o[dim] = src0->ne[dim]/ggml_blck_size(src0->type); - } else { - o[dim] = src0->ne[dim]; - } - // Region 1: copy rows from src0 - for (int i3 = 0; i3 < ne03; i3++) { - for (int i2 = ith; i2 < ne02; i2 += nth) { - for (int i1 = 0; i1 < ne01; i1++) { - const char * x = (const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03; - char * y = ( char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3; - memcpy(y, x, ggml_row_size(src0->type, ne00)); - } - } - } +// rows of dst copied together, so that a strided source is read along neighboring rows +static constexpr int64_t GGML_CONCAT_TILE = 32; - // Region 2: copy rows from src1, offset into dst by o[] - for (int i3 = 0; i3 < ne13; i3++) { - for (int i2 = ith; i2 < ne12; i2 += nth) { - for (int i1 = 0; i1 < ne11; i1++) { - const char * x = (const char *) src1->data + i1*nb11 + i2*nb12 + i3*nb13; - char * y = ( char *) dst->data + (i1 + o[1])*nb1 + (i2 + o[2])*nb2 + (i3 + o[3])*nb3 + o[0]*nb0; - memcpy(y, x, ggml_row_size(src1->type, ne10)); - } +// copy n_rows rows of n elements of es bytes from a source with element stride nb0, element by element +template <size_t es> +static void ggml_concat_copy_strided(char * y, size_t nby, const char * x, size_t nbx, size_t nb0, int64_t n, int64_t n_rows) { + for (int64_t i0 = 0; i0 < n; i0++) { + for (int64_t r = 0; r < n_rows; r++) { + memcpy(y + r*nby + i0*es, x + r*nbx + i0*nb0, es); } } } -static void ggml_compute_forward_concat_i8( - const ggml_compute_params * params, - ggml_tensor * dst) { +// copy n_rows rows of n elements of src into dst, rows are nbx bytes apart in src and nby bytes apart in dst +static void ggml_concat_copy_rows(char * y, size_t nby, const ggml_tensor * src, const char * x, size_t nbx, int64_t n, int64_t n_rows) { + const size_t es = ggml_type_size(src->type); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t)); - - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; - - const int8_t * x; - - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } - - int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); - - *y = *x; - } - } + if (src->nb[0] == es) { + const size_t row_size = ggml_row_size(src->type, n); + for (int64_t r = 0; r < n_rows; r++) { + memcpy(y + r*nby, x + r*nbx, row_size); } + return; } -} -static void ggml_compute_forward_concat_f16( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t)); - - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; - - const ggml_fp16_t * x; - - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } - - ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); - - *y = *x; - } - } - } + switch (es) { + case 1: ggml_concat_copy_strided<1>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 2: ggml_concat_copy_strided<2>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 4: ggml_concat_copy_strided<4>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 8: ggml_concat_copy_strided<8>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + default: GGML_ABORT("unsupported element size"); } } -static void ggml_compute_forward_concat_f32( +// each dst row comes from src0 or src1, or for dim 0 from the row of src0 followed by the row of src1 +void ggml_compute_forward_concat( const ggml_compute_params * params, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float)); - - const int ith = params->ith; - const int nth = params->nth; - GGML_TENSOR_BINARY_OP_LOCALS const int32_t dim = ggml_get_op_params_i32(dst, 0); GGML_ASSERT(dim >= 0 && dim < 4); + GGML_ASSERT(nb0 == ggml_type_size(dst->type)); + GGML_ASSERT(!ggml_is_quantized(dst->type) || (nb00 == nb0 && nb10 == nb0)); + GGML_ASSERT(dim != 0 || (ne00 % ggml_blck_size(dst->type) == 0 && ne10 % ggml_blck_size(dst->type) == 0)); - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; + const size_t y1 = dim == 0 ? ggml_row_size(dst->type, ne00) : 0; - const float * x; + const int64_t nr = ne1*ne2*ne3; + const int64_t dr = (nr + params->nth - 1)/params->nth; - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } + const int64_t ir0 = dr*params->ith; + const int64_t ir1 = MIN(ir0 + dr, nr); - float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + for (int64_t ir = ir0; ir < ir1; ) { + int64_t i[4] = { 0, ir % ne1, (ir/ne1) % ne2, ir/(ne1*ne2) }; - *y = *x; - } - } + // a tile of consecutive rows sharing i2, i3 and their source + int64_t nt = MIN(MIN(ir1 - ir, ne1 - i[1]), GGML_CONCAT_TILE); + if (dim == 1 && i[1] < ne01) { + nt = MIN(nt, ne01 - i[1]); } - } -} -void ggml_compute_forward_concat( - const ggml_compute_params * params, - ggml_tensor * dst) { + char * y = (char *) dst->data + i[1]*nb1 + i[2]*nb2 + i[3]*nb3; - const ggml_tensor * src0 = dst->src[0]; + if (dim == 0 || i[dim] < src0->ne[dim]) { + ggml_concat_copy_rows(y, nb1, src0, (const char *) src0->data + i[1]*nb01 + i[2]*nb02 + i[3]*nb03, nb01, ne00, nt); + } - switch (src0->type) { - case GGML_TYPE_F16: - case GGML_TYPE_BF16: - case GGML_TYPE_I16: - { - ggml_compute_forward_concat_f16(params, dst); - } break; - case GGML_TYPE_I8: - { - ggml_compute_forward_concat_i8(params, dst); - } break; - case GGML_TYPE_F32: - case GGML_TYPE_I32: - { - ggml_compute_forward_concat_f32(params, dst); - } break; - default: - { - ggml_compute_forward_concat_any(params, dst); - } + if (dim == 0 || i[dim] >= src0->ne[dim]) { + i[dim] -= src0->ne[dim]; // i[0] does not address rows + ggml_concat_copy_rows(y + y1, nb1, src1, (const char *) src1->data + i[1]*nb11 + i[2]*nb12 + i[3]*nb13, nb11, ne10, nt); + } + + ir += nt; } } @@ -3548,6 +3452,73 @@ static void ggml_compute_forward_swiglu_oai_f32( } } +static void ggml_compute_forward_swiglu_oai_f16( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + char * src0_d = (char *) src0->data; + char * src1_d = (char *) (src1 ? src1->data : src0->data); + const size_t src0_o = src0->nb[1]; + const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(ggml_is_contiguous_1(dst)); + + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src0->type == src1->type); + } + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + const int nr = ggml_nrows(src0); + + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == nr); + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + const float alpha = ggml_get_op_params_f32(dst, 2); + const float limit = ggml_get_op_params_f32(dst, 3); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o); + ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o); + ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])); + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + for (int k = 0; k < nc; k++) { + const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit); + const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit); + const float out_glu = x / (1.f + expf(alpha * (-x))); + dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f)); + } + +#ifndef NDEBUG + for (int k = 0; k < nc; k++) { + const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]); + GGML_UNUSED(x); + assert(!isnan(x)); + assert(!isinf(x)); + } +#endif // NDEBUG + } +} + static void ggml_compute_forward_swiglu_oai( const ggml_compute_params * params, ggml_tensor * dst) { @@ -3559,6 +3530,10 @@ static void ggml_compute_forward_swiglu_oai( { ggml_compute_forward_swiglu_oai_f32(params, dst); } break; + case GGML_TYPE_F16: + { + ggml_compute_forward_swiglu_oai_f16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -4841,6 +4816,7 @@ void ggml_compute_forward_out_prod( case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -4883,8 +4859,8 @@ static void ggml_compute_forward_scale_f32( const ggml_tensor * src0 = dst->src[0]; - GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); float s; // scale factor @@ -4899,6 +4875,8 @@ static void ggml_compute_forward_scale_f32( const int nc = src0->ne[0]; const int nr = ggml_nrows(src0); + GGML_TENSOR_UNARY_OP_LOCALS + // rows per thread const int dr = (nr + nth - 1)/nth; @@ -4906,25 +4884,23 @@ static void ggml_compute_forward_scale_f32( const int ir0 = dr*ith; const int ir1 = MIN(ir0 + dr, nr); - const size_t nb01 = src0->nb[1]; + for (int i = ir0; i < ir1; i++) { + const int64_t i1 = i % ne01; + const int64_t i2 = (i / ne01) % ne02; + const int64_t i3 = i / (ne01*ne02); - const size_t nb1 = dst->nb[1]; + float * y = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); + const float * x = (const float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); - if (b == 0.0f) { - for (int i1 = ir0; i1 < ir1; i1++) { + if (b == 0.0f) { if (dst->data != src0->data) { // src0 is same shape as dst => same indices // TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy - memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float)); + memcpy(y, x, nc * sizeof(float)); } - ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s); - } - } else { - for (int i1 = ir0; i1 < ir1; i1++) { - ggml_vec_mad1_f32(nc, - (float *) ((char *) dst->data + i1*nb1), - (float *) ((char *) src0->data + i1*nb1), - s, b); + ggml_vec_scale_f32(nc, y, s); + } else { + ggml_vec_mad1_f32(nc, y, x, s, b); } } } @@ -4935,8 +4911,8 @@ static void ggml_compute_forward_scale_bf16( const ggml_tensor * src0 = dst->src[0]; - GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); float s; // scale factor @@ -4951,6 +4927,8 @@ static void ggml_compute_forward_scale_bf16( const int nc = src0->ne[0]; const int nr = ggml_nrows(src0); + GGML_TENSOR_UNARY_OP_LOCALS + // rows per thread const int dr = (nr + nth - 1)/nth; @@ -4958,9 +4936,13 @@ static void ggml_compute_forward_scale_bf16( const int ir0 = dr*ith; const int ir1 = MIN(ir0 + dr, nr); - for (int i1 = ir0; i1 < ir1; i1++) { - const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*src0->nb[1]); - ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*dst->nb[1]); + for (int i = ir0; i < ir1; i++) { + const int64_t i1 = i % ne01; + const int64_t i2 = (i / ne01) % ne02; + const int64_t i3 = i / (ne01*ne02); + + const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); + ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); for (int k = 0; k < nc; k++) { y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b); } @@ -5160,6 +5142,7 @@ void ggml_compute_forward_set( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5385,6 +5368,7 @@ void ggml_compute_forward_get_rows( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5720,94 +5704,6 @@ void ggml_compute_forward_diag( } } -// ggml_compute_forward_diag_mask_inf - -static void ggml_compute_forward_diag_mask_f32( - const ggml_compute_params * params, - ggml_tensor * dst, - const float value) { - - const ggml_tensor * src0 = dst->src[0]; - - const int ith = params->ith; - const int nth = params->nth; - - const int n_past = ((int32_t *) dst->op_params)[0]; - const bool inplace = src0->data == dst->data; - - GGML_ASSERT(n_past >= 0); - - if (!inplace) { - if (ith == 0) { - // memcpy needs to be synchronized across threads to avoid race conditions. - // => do it in INIT phase - GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); - GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0)); - memcpy( - ((char *) dst->data), - ((char *) src0->data), - ggml_nbytes(dst)); - } - ggml_barrier(params->threadpool); - } - - // TODO: handle transposed/permuted matrices - - const int n = ggml_nrows(src0); - const int nc = src0->ne[0]; - const int nr = src0->ne[1]; - const int nz = n/nr; - - GGML_ASSERT( dst->nb[0] == sizeof(float)); - GGML_ASSERT(src0->nb[0] == sizeof(float)); - - for (int k = 0; k < nz; k++) { - for (int j = ith; j < nr; j += nth) { - for (int i = n_past; i < nc; i++) { - if (i > n_past + j) { - *(float *)((char *) dst->data + k*dst->nb[2] + j*dst->nb[1] + i*dst->nb[0]) = value; - } - } - } - } -} - -void ggml_compute_forward_diag_mask_inf( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - - switch (src0->type) { - case GGML_TYPE_F32: - { - ggml_compute_forward_diag_mask_f32(params, dst, -INFINITY); - } break; - default: - { - GGML_ABORT("fatal error"); - } - } -} - -void ggml_compute_forward_diag_mask_zero( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - - switch (src0->type) { - case GGML_TYPE_F32: - { - ggml_compute_forward_diag_mask_f32(params, dst, 0); - } break; - default: - { - GGML_ABORT("fatal error"); - } - } -} - // ggml_compute_forward_soft_max static void ggml_compute_forward_soft_max_f32( @@ -6063,18 +5959,18 @@ static void ggml_compute_forward_clamp_f32( const int n = ggml_nrows(src0); const int nc = src0->ne[0]; - const size_t nb00 = src0->nb[0]; - const size_t nb01 = src0->nb[1]; - - const size_t nb0 = dst->nb[0]; - const size_t nb1 = dst->nb[1]; + GGML_TENSOR_UNARY_OP_LOCALS GGML_ASSERT( nb0 == sizeof(float)); GGML_ASSERT(nb00 == sizeof(float)); for (int j = ith; j < n; j += nth) { - float * dst_ptr = (float *) ((char *) dst->data + j*nb1); - float * src0_ptr = (float *) ((char *) src0->data + j*nb01); + const int64_t i1 = j % ne01; + const int64_t i2 = (j / ne01) % ne02; + const int64_t i3 = j / (ne01*ne02); + + float * dst_ptr = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); + float * src0_ptr = (float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); for (int i = 0; i < nc; i++) { dst_ptr[i] = MAX(MIN(src0_ptr[i], max), min); @@ -6099,18 +5995,18 @@ static void ggml_compute_forward_clamp_f16( const int n = ggml_nrows(src0); const int nc = src0->ne[0]; - const size_t nb00 = src0->nb[0]; - const size_t nb01 = src0->nb[1]; - - const size_t nb0 = dst->nb[0]; - const size_t nb1 = dst->nb[1]; + GGML_TENSOR_UNARY_OP_LOCALS GGML_ASSERT( nb0 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); for (int j = ith; j < n; j += nth) { - ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + j*nb1); - ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + j*nb01); + const int64_t i1 = j % ne01; + const int64_t i2 = (j / ne01) % ne02; + const int64_t i3 = j / (ne01*ne02); + + ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); + ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); for (int i = 0; i < nc; i++) { float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]); @@ -6145,6 +6041,7 @@ void ggml_compute_forward_clamp( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -6325,8 +6222,8 @@ static void ggml_compute_forward_rope_flt( //const int n_past = ((int32_t *) dst->op_params)[0]; const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; - //const int n_ctx = ((int32_t *) dst->op_params)[3]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; + const bool truncate = ggml_rope_get_truncate(dst); memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float)); memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float)); @@ -6371,7 +6268,7 @@ static void ggml_compute_forward_rope_flt( const float theta_scale = powf(freq_base, -2.0f/n_dims); float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, truncate, corr_dims); const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope const bool mrope_used = mode & GGML_ROPE_TYPE_MROPE; // ggml_rope_multi, note: also true for vision (24 & 8 == true) and for imrope @@ -6529,17 +6426,28 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( const int ith = params->ith; const int nth = params->nth; - const int nk = ne00*ne01*ne02; - GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb10 == sizeof(float)); if (ith == 0) { memset(params->wdata, 0, params->wsize); + // permute source data (src1) from (L x Cin) to (Cin x L) + { + float * const wdata = (float *) params->wdata + 0; + float * dst_data = wdata; + + for (int64_t i11 = 0; i11 < ne11; i11++) { + const float * const src = (float *)((char *) src1->data + i11*nb11); + for (int64_t i10 = 0; i10 < ne10; i10++) { + dst_data[i10*ne11 + i11] = src[i10]; + } + } + } + // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout) { - ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0; + ggml_fp16_t * const wdata = (ggml_fp16_t *)((char *) params->wdata + ne10*ne11*sizeof(float)); for (int64_t i02 = 0; i02 < ne02; i02++) { for (int64_t i01 = 0; i01 < ne01; i01++) { @@ -6552,19 +6460,6 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( } } - // permute source data (src1) from (L x Cin) to (Cin x L) - { - ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk; - ggml_fp16_t * dst_data = wdata; - - for (int64_t i11 = 0; i11 < ne11; i11++) { - const float * const src = (float *)((char *) src1->data + i11*nb11); - for (int64_t i10 = 0; i10 < ne10; i10++) { - dst_data[i10*ne11 + i11] = GGML_CPU_FP32_TO_FP16(src[i10]); - } - } - } - // need to zero dst since we are accumulating into it memset(dst->data, 0, ggml_nbytes(dst)); } @@ -6582,19 +6477,19 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( const int ir0 = dr*ith; const int ir1 = MIN(ir0 + dr, nr); - ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0; - ggml_fp16_t * const wdata_src = wdata + nk; + float * const wdata_src = (float *) params->wdata + 0; + ggml_fp16_t * const wdata_kernel = (ggml_fp16_t *)((char *) params->wdata + ne10*ne11*sizeof(float)); for (int i1 = ir0; i1 < ir1; i1++) { float * dst_data = (float *)((char *) dst->data + i1*nb1); - ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00; + ggml_fp16_t * wdata_kernel_row = wdata_kernel + i1*ne02*ne00; for (int i10 = 0; i10 < ne10; i10++) { const int i1n = i10*ne11; for (int i00 = 0; i00 < ne00; i00++) { float v = 0; - ggml_vec_dot_f16(ne02, &v, 0, - (ggml_fp16_t *) wdata_src + i1n, 0, - (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1); + ggml_vec_dot_f16_f32(ne02, &v, 0, + wdata_kernel_row + i00*ne02, 0, + wdata_src + i1n, 0, 1); dst_data[i10*s0 + i00] += v; } } @@ -7187,29 +7082,30 @@ void ggml_compute_forward_im2col_3d( } } -static void ggml_call_mul_mat(ggml_type type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k, +static void ggml_call_mul_mat(ggml_type src0_type, ggml_type src1_type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k, void * a, void * b, float * c) { - const ggml_type_traits * traits = ggml_get_type_traits(type); + const ggml_type_traits * traits0 = ggml_get_type_traits(src0_type); + const ggml_type_traits * traits1 = ggml_get_type_traits(src1_type); struct ggml_tensor src1 = {}; - src1.type = type; + src1.type = src1_type; src1.ne[0] = k; src1.ne[1] = m; src1.ne[2] = 1; src1.ne[3] = 1; - src1.nb[0] = traits->type_size; - src1.nb[1] = k * traits->type_size; + src1.nb[0] = traits1->type_size; + src1.nb[1] = k * traits1->type_size; src1.nb[2] = src1.nb[1]; src1.nb[3] = src1.nb[2]; src1.data = a; struct ggml_tensor src0 = {}; - src0.type = type; + src0.type = src0_type; src0.ne[0] = k; src0.ne[1] = n; src0.ne[2] = 1; src0.ne[3] = 1; - src0.nb[0] = traits->type_size; - src0.nb[1] = k * traits->type_size; + src0.nb[0] = traits0->type_size; + src0.nb[1] = k * traits0->type_size; src0.nb[2] = src0.nb[1]; src0.nb[3] = src0.nb[2]; src0.data = b; @@ -7319,8 +7215,6 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32); GGML_ASSERT(kernel->type == kernel_type); - const ggml_type_traits * traits = ggml_get_type_traits(kernel_type); - const int32_t stride_x = dst->op_params[0]; const int32_t stride_y = dst->op_params[1]; const int32_t pad_x = dst->op_params[2]; @@ -7346,7 +7240,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params const int64_t knl_n = knl_w * knl_h * c_in; const int64_t patch_total = dst->ne[3] * dst_w * dst_h; - const int64_t space_per_patch = knl_n * traits->type_size + c_out * sizeof(float); + const int64_t space_per_patch = knl_n * sizeof(float) + c_out * sizeof(float); const int64_t batch_size = params->wsize / space_per_patch; const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size; const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch; @@ -7373,7 +7267,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params const int64_t src_y = p % dst_w; const float * src_base = (const float *)((const char *)src_data + batch_n * src->nb[3]); - char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n * traits->type_size; + float * dst_row = (float *) tmp + (p % patches_per_batch) * knl_n; for (int64_t ic = 0; ic < c_in; ++ic) { for (int64_t ky = 0; ky < knl_h; ++ky) { @@ -7391,12 +7285,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params src_val = *src_ptr; } - char * element_ptr = dst_row + dst_idx * traits->type_size; - if (kernel_type == GGML_TYPE_F32) { - *(float *) element_ptr = src_val; - } else if (kernel_type == GGML_TYPE_F16) { - *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val); - } + dst_row[dst_idx] = src_val; } } } @@ -7404,12 +7293,12 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params ggml_barrier(params->threadpool); - float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * traits->type_size); + float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * sizeof(float)); GGML_ASSERT(gemm_output + patch_n * c_out <= (float*)tmp + params->wsize); // GEMM: patches[patch_n, knl_n] × kernel[knl_n, c_out] = output[patch_n, c_out] - ggml_call_mul_mat(kernel_type, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output); + ggml_call_mul_mat(kernel_type, GGML_TYPE_F32, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output); ggml_barrier(params->threadpool); @@ -7456,8 +7345,6 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32); GGML_ASSERT(kernel->type == kernel_type); - const ggml_type_traits * traits = ggml_get_type_traits(kernel_type); - const int32_t s0 = dst->op_params[0]; const int32_t s1 = dst->op_params[1]; const int32_t s2 = dst->op_params[2]; @@ -7489,7 +7376,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params const int64_t knl_n_total = knl_n_per_channel * c; const int64_t patch_total = n * dst_w * dst_h * dst_d; - const int64_t space_per_patch = knl_n_total * traits->type_size + oc * sizeof(float); + const int64_t space_per_patch = knl_n_total * sizeof(float) + oc * sizeof(float); const int64_t batch_size = params->wsize / space_per_patch; const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size; const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch; @@ -7515,7 +7402,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params const int64_t dst_y = p_in_depth / dst_w; const int64_t dst_x = p_in_depth % dst_w; - char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n_total * traits->type_size; + float * dst_row = (float *) tmp + (p % patches_per_batch) * knl_n_total; for (int64_t ic = 0; ic < c; ++ic) { for (int64_t kz = 0; kz < knl_d; ++kz) { @@ -7536,12 +7423,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params src_val = *src_ptr; } - char * element_ptr = dst_row + dst_idx * traits->type_size; - if (kernel_type == GGML_TYPE_F32) { - *(float *)element_ptr = src_val; - } else if (kernel_type == GGML_TYPE_F16) { - *(ggml_fp16_t *)element_ptr = GGML_CPU_FP32_TO_FP16(src_val); - } + dst_row[dst_idx] = src_val; } } } @@ -7550,8 +7432,8 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params ggml_barrier(params->threadpool); - float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * traits->type_size); - ggml_call_mul_mat(kernel_type, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output); + float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * sizeof(float)); + ggml_call_mul_mat(kernel_type, GGML_TYPE_F32, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output); ggml_barrier(params->threadpool); @@ -7603,7 +7485,7 @@ static void ggml_compute_forward_conv_transpose_2d_impl( const int ith = params->ith; const int nth = params->nth; - const int nk = ne00*ne01*ne02*ne03; + const int nsrc = ne10*ne11*ne12*ne13; GGML_ASSERT(nb00 == ggml_type_size(src0->type)); GGML_ASSERT(nb10 == sizeof(float)); @@ -7611,9 +7493,26 @@ static void ggml_compute_forward_conv_transpose_2d_impl( if (ith == 0) { memset(params->wdata, 0, params->wsize); + // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches + { + float * const wdata = (float *) params->wdata + 0; + for (int i13 = 0; i13 < ne13; i13++) { + float * const wdata_b = wdata + i13*ne10*ne11*ne12; + for (int i12 = 0; i12 < ne12; i12++) { + for (int i11 = 0; i11 < ne11; i11++) { + const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11); + float * dst_data = wdata_b + i11*ne10*ne12; + for (int i10 = 0; i10 < ne10; i10++) { + dst_data[i10*ne12 + i12] = src[i10]; + } + } + } + } + } + // permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout) { - kernel_t * const wdata = (kernel_t *) params->wdata + 0; + kernel_t * const wdata = (kernel_t *)((char *) params->wdata + nsrc*sizeof(float)); for (int64_t i03 = 0; i03 < ne03; i03++) { for (int64_t i02 = 0; i02 < ne02; i02++) { @@ -7628,27 +7527,6 @@ static void ggml_compute_forward_conv_transpose_2d_impl( } } - // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches - { - kernel_t * const wdata = (kernel_t *) params->wdata + nk; - for (int i13 = 0; i13 < ne13; i13++) { - kernel_t * const wdata_b = wdata + i13*ne10*ne11*ne12; - for (int i12 = 0; i12 < ne12; i12++) { - for (int i11 = 0; i11 < ne11; i11++) { - const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11); - kernel_t * dst_data = wdata_b + i11*ne10*ne12; - for (int i10 = 0; i10 < ne10; i10++) { - if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) { - dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]); - } else { - dst_data[i10*ne12 + i12] = src[i10]; - } - } - } - } - } - } - memset(dst->data, 0, ggml_nbytes(dst)); } ggml_barrier(params->threadpool); @@ -7665,14 +7543,14 @@ static void ggml_compute_forward_conv_transpose_2d_impl( const int ip0 = dp*ith; const int ip1 = MIN(ip0 + dp, np); - kernel_t * const wdata = (kernel_t *) params->wdata + 0; - kernel_t * const wdata_src = wdata + nk; + float * const wdata_src = (float *) params->wdata + 0; + kernel_t * const wdata_kernel = (kernel_t *)((char *) params->wdata + nsrc*sizeof(float)); for (int i2 = ip0; i2 < ip1; i2++) { // Cout - kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03; + kernel_t * wdata_kernel_c = wdata_kernel + i2*ne01*ne00*ne03; for (int i3 = 0; i3 < ne3; i3++) { // batch float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2); - kernel_t * wdata_src_b = wdata_src + i3*ne10*ne11*ne12; + float * wdata_src_b = wdata_src + i3*ne10*ne11*ne12; for (int i11 = 0; i11 < ne11; i11++) { for (int i10 = 0; i10 < ne10; i10++) { const int i1n = i11*ne10*ne12 + i10*ne12; @@ -7680,13 +7558,13 @@ static void ggml_compute_forward_conv_transpose_2d_impl( for (int i00 = 0; i00 < ne00; i00++) { float v = 0; if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) { - ggml_vec_dot_f16(ne03, &v, 0, - wdata_src_b + i1n, 0, - wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + ggml_vec_dot_f16_f32(ne03, &v, 0, + wdata_kernel_c + i01*ne00*ne03 + i00*ne03, 0, + wdata_src_b + i1n, 0, 1); } else { ggml_vec_dot_f32(ne03, &v, 0, - wdata_src_b + i1n, 0, - wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + wdata_kernel_c + i01*ne00*ne03 + i00*ne03, 0, + wdata_src_b + i1n, 0, 1); } dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v; } @@ -8444,40 +8322,47 @@ static void ggml_compute_forward_pad_f32( const int32_t lp3 = ggml_get_op_params_i32(dst, 6); const int32_t rp3 = ggml_get_op_params_i32(dst, 7); - // TODO: optimize + const int64_t nrows = ne1 * ne2 * ne3; + const bool src_row_contig = src0->nb[0] == sizeof(float); + + for (int64_t row = ith; row < nrows; row += nth) { + const int64_t i1 = row % ne1; + const int64_t i2 = (row / ne1) % ne2; + const int64_t i3 = row / (ne1 * ne2); + + float * dst_row = dst_ptr + row * ne0; + + if constexpr (circular_t) { + // circular padding wraps around on a torus + const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01); + const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02); + const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03); - for (int64_t i2 = 0; i2 < ne2; ++i2) { - for (int64_t i1 = ith; i1 < ne1; i1 += nth) { for (int64_t i0 = 0; i0 < ne0; ++i0) { - for (int64_t i3 = 0; i3 < ne3; ++i3) { - // circular means wrap around on a torus, so x and y loop around - if constexpr (circular_t) { - const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0; - const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00); - const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01); - const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02); - const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03); - - const int64_t src_idx = - src_i3*nb03 + - src_i2*nb02 + - src_i1*nb01 + - src_i0*nb00; - - const float * src_ptr = (const float *)((char *) src0->data + src_idx); - dst_ptr[dst_idx] = *src_ptr; + const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00); + dst_row[i0] = *(const float *)((const char *) src0->data + src_i3*nb03 + src_i2*nb02 + src_i1*nb01 + src_i0*nb00); + } + } else { + const bool interior = (i1 >= lp1 && i1 < ne1 - rp1) + && (i2 >= lp2 && i2 < ne2 - rp2) + && (i3 >= lp3 && i3 < ne3 - rp3); + if (!interior) { + memset(dst_row, 0, ne0 * sizeof(float)); + continue; + } + + const char * src_row = (const char *) src0->data + (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01; + + if (src_row_contig && lp0 >= 0 && rp0 >= 0) { + memset(dst_row, 0, lp0 * sizeof(float)); + memcpy(dst_row + lp0, src_row, ne00 * sizeof(float)); + memset(dst_row + lp0 + ne00, 0, rp0 * sizeof(float)); + } else { + for (int64_t i0 = 0; i0 < ne0; ++i0) { + if (i0 >= lp0 && i0 < ne0 - rp0) { + dst_row[i0] = *(const float *)(src_row + (i0 - lp0)*nb00); } else { - const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0; - if ((i0 >= lp0 && i0 < ne0 - rp0) \ - && (i1 >= lp1 && i1 < ne1 - rp1) \ - && (i2 >= lp2 && i2 < ne2 - rp2) \ - && (i3 >= lp3 && i3 < ne3 - rp3)) { - const int64_t src_idx = (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01 + (i0 - lp0)*nb00; - const float * src_ptr = (const float *)((char *) src0->data + src_idx); - dst_ptr[dst_idx] = *src_ptr; - } else { - dst_ptr[dst_idx] = 0; - } + dst_row[i0] = 0.0f; } } } @@ -8852,6 +8737,7 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( const ggml_tensor * v = dst->src[2]; const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; + const ggml_tensor * kv_rows = dst->src[5]; GGML_TENSOR_LOCALS(int64_t, neq, q, ne) GGML_TENSOR_LOCALS(size_t, nbq, q, nb) @@ -8937,14 +8823,9 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( float * VKQ32 = (float *) params->wdata + ith*(1*DK + 2*DV + CACHE_LINE_SIZE_F32); // FP32 VKQ accumulator float * V32 = (VKQ32 + 1*DV); // (temporary) FP32 V buffer - ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16 - if (v->type == GGML_TYPE_F16) { - memset(VKQ16, 0, DV*sizeof(ggml_fp16_t)); - } else { - memset(VKQ32, 0, DV*sizeof(float)); - } + memset(VKQ32, 0, DV*sizeof(float)); const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL; @@ -8959,6 +8840,8 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)); q_to_vec_dot(pq, Q_q, DK); + const int32_t * rows = kv_rows ? (const int32_t *) ((const char *) kv_rows->data + iq3*kv_rows->nb[1]) : nullptr; + // online softmax / attention // loop over n_kv and n_head_kv // ref: https://arxiv.org/pdf/2112.05682.pdf @@ -8969,9 +8852,14 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( continue; } + const int64_t ik1 = rows ? rows[ic] : ic; + if (ik1 < 0) { + continue; + } + float s; // KQ value - const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3); + const char * k_data = (const char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3); kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1); s = s*scale; // scale KQ value @@ -8987,55 +8875,34 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value float vs = 1.0f; // post-softmax KQ value, expf(s - M) - const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3)); - - if (v->type == GGML_TYPE_F16) { - if (s > M) { - // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f - M = s; - ms = expf(Mold - M); + const char * v_data = ((const char *) v->data + (ik1*nbv1 + iv2*nbv2 + iv3*nbv3)); - // V = V*expf(Mold - M) - ggml_vec_scale_f16(DV, VKQ16, ms); - } else { - // no new maximum, ms == 1.0f, vs != 1.0f - vs = expf(s - M); - } + if (s > M) { + // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f + M = s; + ms = expf(Mold - M); - // V += v*expf(s - M) - ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs); + // V = V*expf(Mold - M) + ggml_vec_scale_f32(DV, VKQ32, ms); } else { - if (s > M) { - // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f - M = s; - ms = expf(Mold - M); - - // V = V*expf(Mold - M) - ggml_vec_scale_f32(DV, VKQ32, ms); - } else { - // no new maximum, ms == 1.0f, vs != 1.0f - vs = expf(s - M); - } + // no new maximum, ms == 1.0f, vs != 1.0f + vs = expf(s - M); + } - // V += v*expf(s - M) - if (v_to_float) { - v_to_float(v_data, V32, DV); - ggml_vec_mad_f32(DV, VKQ32, V32, vs); - } else { - // V is F32 - ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs); - } + // V += v*expf(s - M) + if (v->type == GGML_TYPE_F16) { + ggml_vec_mad_f16_f32(DV, VKQ32, (const ggml_fp16_t *) v_data, vs); + } else if (v_to_float) { + v_to_float(v_data, V32, DV); + ggml_vec_mad_f32(DV, VKQ32, V32, vs); + } else { + // V is F32 + ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs); } S = S*ms + vs; // scale and increment sum with partial sum } - if (v->type == GGML_TYPE_F16) { - for (int64_t d = 0; d < DV; ++d) { - VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]); - } - } - // sinks - apply only on the first kv-chunk if (sinks && ic_start == 0) { const float s = ((float *)((char *) sinks->data))[h]; @@ -9154,6 +9021,10 @@ static void ggml_compute_forward_flash_attn_ext_tiled( static constexpr int Q_TILE_SZ = ggml_fa_tile_config::Q; static constexpr int KV_TILE_SZ = ggml_fa_tile_config::KV; + // with kv_rows the loop runs over the mask columns and reads K/V rows through kv_rows + const ggml_tensor * kv_rows = dst->src[5]; + const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : nek1; + int ir = ir0; while (ir < ir1) { // q indices for the start of this tile @@ -9161,6 +9032,10 @@ static void ggml_compute_forward_flash_attn_ext_tiled( const int iq2 = (ir - iq3*neq2*neq1)/neq1; const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1); + // padding rows are masked, read row 0 for them + const int32_t * rows = kv_rows ? (const int32_t *) ((const char *) kv_rows->data + iq3*kv_rows->nb[1]) : nullptr; + auto kv_row = [rows](int64_t i) -> int64_t { return rows ? std::max<int32_t>(rows[i], 0) : i; }; + // Number of valid rows in this tile: // - limited by tile size (Q_TILE_SZ) // - limited by chunk boundary (ir1 - ir) @@ -9220,8 +9095,8 @@ static void ggml_compute_forward_flash_attn_ext_tiled( memset(K_f32, 0, DK * KV_TILE_SZ * sizeof(float)); memset(V32, 0, KV_TILE_SZ * DV * sizeof(float)); - for (int64_t ic = 0; ic < nek1; ic += KV_TILE_SZ) { - const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, nek1 - ic); + for (int64_t ic = 0; ic < n_kv_cols; ic += KV_TILE_SZ) { + const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, n_kv_cols - ic); // skip the tile entirely if all the masks are -inf if (mask) { @@ -9248,7 +9123,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled( // Pack K tile transposed: K_f32[dk][kv] so KV_TILE is contiguous (SIMD dim) // Zero-pad the last tile so the GEMM always operates on KV_TILE_SZ columns for (int tk = 0; tk < kv_tile; tk++) { - const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3; + const char * k_data = (const char *)k->data + kv_row(ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3; if (kv_type == GGML_TYPE_F16) { const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data; for (int64_t dk = 0; dk < DK; dk++) { @@ -9313,7 +9188,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled( // V accumulation: VKQ32 += softmax(KQ) * V // Pack V tile to contiguous F32, zero-padded for (int tk = 0; tk < kv_tile; tk++) { - const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3; + const char * v_data = (const char *)v->data + kv_row(ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3; if (kv_type == GGML_TYPE_F16) { ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV); } else { @@ -9485,8 +9360,12 @@ static void ggml_compute_forward_flash_attn_ext_f16( // When use_ref is set, force the vec-only reference implementation (no tiling, no KV-chunking) const bool use_ref = params->use_ref; + // with kv_rows the loops run over the mask columns, the split KV path does not support it + const ggml_tensor * kv_rows = dst->src[5]; + const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : nek1; + const bool kv_is_f32_or_f16 = (k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_F16); - const bool use_split_kv_path = !use_ref && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512; + const bool use_split_kv_path = !use_ref && !kv_rows && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512; if (use_split_kv_path) { const int64_t chunk_size = (nek1 + nth - 1) / nth; @@ -9565,7 +9444,7 @@ static void ggml_compute_forward_flash_attn_ext_f16( if (use_tiled) { ggml_compute_forward_flash_attn_ext_tiled(params, dst, ir0, ir1); } else { - ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, nek1, nullptr, 0); + ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, n_kv_cols, nullptr, 0); } current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1); @@ -9576,9 +9455,10 @@ static void ggml_compute_forward_flash_attn_ext_f16( void ggml_compute_forward_flash_attn_ext( const ggml_compute_params * params, ggml_tensor * dst) { - switch (dst->op_params[3]) { - case GGML_PREC_DEFAULT: + switch (ggml_prec_get_acc(dst)) { + case GGML_PREC_UNDEFINED: case GGML_PREC_F32: + case GGML_PREC_BF16: { // uses F32 accumulators ggml_compute_forward_flash_attn_ext_f16(params, dst); @@ -10102,7 +9982,7 @@ static void ggml_compute_forward_ssm_scan_f32( } sumf = GGML_F32xt_REDUCE_ONE(sum); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) // todo: RVV implementation const int np = 0; #else @@ -11645,6 +11525,92 @@ void ggml_compute_forward_dsv4_hc_post( } } +// ggml_compute_forward_top_k_mask + +void ggml_compute_forward_top_k_mask( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * mask = dst->src[0]; + const ggml_tensor * top_k = dst->src[1]; + const ggml_tensor * score = dst->src[2]; + const ggml_tensor * blocks = dst->src[3]; + const ggml_tensor * extra = dst->src[4]; + + GGML_ASSERT(dst->type == mask->type); + GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type)); + + const int64_t n_kv = mask->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t n_rows = ggml_nrows(dst); + const int64_t n_top = top_k->ne[0]; + const int64_t n_blk = blocks ? blocks->ne[0] : 1; + const int64_t n_pick = blocks ? blocks->ne[1] : score ? score->ne[0] : n_kv; + const int64_t n_extra = extra ? extra->ne[0] : 0; + + const size_t ts = ggml_type_size(dst->type); + + const int ith = params->ith; + const int nth = params->nth; + + for (int64_t ir = ith; ir < n_rows; ir += nth) { + const int64_t i1 = ir % ne1; + const int64_t i2 = (ir / ne1) % ne2; + const int64_t i3 = ir / (ne1*ne2); + + const char * mask_row = (const char *) mask->data + + (i1 % mask->ne[1])*mask->nb[1] + (i2 % mask->ne[2])*mask->nb[2] + (i3 % mask->ne[3])*mask->nb[3]; + char * dst_row = (char *) dst->data + i1*dst->nb[1] + i2*dst->nb[2] + i3*dst->nb[3]; + + if (dst->type == GGML_TYPE_F16) { + const ggml_fp16_t ninf = GGML_CPU_FP32_TO_FP16(-INFINITY); + for (int64_t c = 0; c < n_kv; ++c) { + ((ggml_fp16_t *) dst_row)[c] = ninf; + } + } else { + for (int64_t c = 0; c < n_kv; ++c) { + ((float *) dst_row)[c] = -INFINITY; + } + } + + auto keep = [&](int32_t c) { + if (c >= 0 && c < n_kv) { + memcpy(dst_row + c*ts, mask_row + c*ts, ts); + } + }; + + const char * top_k_row = (const char *) top_k->data + i1*top_k->nb[1] + i2*top_k->nb[2] + i3*top_k->nb[3]; + + for (int64_t j = 0; j < n_top; ++j) { + const int32_t p = *(const int32_t *) (top_k_row + j*top_k->nb[0]); + if (p < 0 || p >= n_pick) { + continue; + } + if (score) { + const float s = *(const float *) ((const char *) score->data + + p*score->nb[0] + i1*score->nb[1] + i2*score->nb[2] + i3*score->nb[3]); + if (!(s > -INFINITY)) { + continue; + } + } + if (!blocks) { + keep(p); + continue; + } + for (int64_t k = 0; k < n_blk; ++k) { + keep(*(const int32_t *) ((const char *) blocks->data + k*blocks->nb[0] + p*blocks->nb[1])); + } + } + + if (extra) { + const char * extra_row = (const char *) extra->data + i1*extra->nb[1] + i2*extra->nb[2] + i3*extra->nb[3]; + for (int64_t j = 0; j < n_extra; ++j) { + keep(*(const int32_t *) (extra_row + j*extra->nb[0])); + } + } + } +} + // ggml_compute_forward_rwkv_wkv7 static void ggml_compute_forward_rwkv_wkv7_f32( @@ -11680,7 +11646,7 @@ static void ggml_compute_forward_rwkv_wkv7_f32( int64_t h_stride_2d = head_size * head_size; #if defined(GGML_SIMD) - #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v_intrinsic) + #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v) // scalar Route to scalar implementation //TODO: Write SVE code and RVV code for (int64_t t = 0; t < T; t++) { int64_t t_offset = t * t_stride; diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index 2728b08b6..aac527a64 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -56,8 +56,6 @@ void ggml_compute_forward_get_rows(const struct ggml_compute_params * params, st void ggml_compute_forward_get_rows_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_set_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_diag(const struct ggml_compute_params * params, struct ggml_tensor * dst); -void ggml_compute_forward_diag_mask_inf(const struct ggml_compute_params * params, struct ggml_tensor * dst); -void ggml_compute_forward_diag_mask_zero(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_soft_max(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_soft_max_ext_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_rope(const struct ggml_compute_params * params, struct ggml_tensor * dst); @@ -108,6 +106,7 @@ void ggml_compute_forward_lightning_indexer(const struct ggml_compute_params * p void ggml_compute_forward_dsv4_hc_comb(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_dsv4_hc_pre(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_dsv4_hc_post(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_top_k_mask(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_map_custom1(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_map_custom2(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_map_custom3(const struct ggml_compute_params * params, struct ggml_tensor * dst); diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index 5e36459f8..44ea9fae5 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -3,6 +3,7 @@ #include "ggml-cpu-impl.h" #include "simd-mappings.h" +#include "vec.h" #include "ggml-quants.h" #include "quants.h" @@ -62,6 +63,10 @@ void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, i quantize_row_nvfp4_ref(x, y, k); } +void quantize_row_f8_e4m3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + quantize_row_f8_e4m3_ref(x, y, k); +} + // // 2-6 bit quantization in super-blocks // @@ -362,6 +367,49 @@ void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } +void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const ggml_fp8_e4m3_t * GGML_RESTRICT x = vx; + const float * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + int pos = 0; +#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic) + GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO }; + const int np = n - n % GGML_F32_STEP; + for (; pos < np; pos += GGML_F32_STEP) { + for (int j = 0; j < GGML_F32_ARR; ++j) { + const int offset = pos + j*GGML_F32_EPR; + GGML_F32_VEC ax; +#if defined(__AVX512F__) && defined(__AVX512BW__) + const __m128i codes = _mm_loadu_si128((const __m128i *) (x + offset)); + ax = _mm512_i32gather_ps(_mm512_cvtepu8_epi32(codes), ggml_table_f32_f8_e4m3, sizeof(float)); +#elif defined(__AVX2__) && !defined(__AVX512F__) + const __m128i codes = _mm_loadl_epi64((const __m128i *) (x + offset)); + ax = _mm256_i32gather_ps(ggml_table_f32_f8_e4m3, _mm256_cvtepu8_epi32(codes), sizeof(float)); +#else + float values[GGML_F32_EPR]; + for (int k = 0; k < GGML_F32_EPR; ++k) { + values[k] = ggml_table_f32_f8_e4m3[x[offset + k].bits]; + } + ax = GGML_F32_VEC_LOAD(values); +#endif + sum[j] = GGML_F32_VEC_FMA(sum[j], ax, GGML_F32_VEC_LOAD(y + offset)); + } + } + GGML_F32_VEC_REDUCE(sumf, sum); +#endif + for (; pos < n; ++pos) { + sumf += ggml_table_f32_f8_e4m3[x[pos].bits] * y[pos]; + } + *s = sumf; +} + void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index 93ea7eeff..f04be52f5 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -23,6 +23,7 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +void quantize_row_f8_e4m3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -40,6 +41,7 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, // Dot product void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index 2632ae3a0..9282ca299 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -4855,7 +4855,12 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PAR const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]); - GGML_ASSERT(i02 >= 0 && i02 < n_as); + GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as)); + + if (i02 == -1) { + memset((char *) dst->data + id*nb1 + iid1*nb2, 0, ne0*sizeof(float)); + continue; + } MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = { id, iid1 }; matrix_row_counts[i02] += 1; diff --git a/ggml/src/ggml-cpu/simd-gemm.h b/ggml/src/ggml-cpu/simd-gemm.h index 4b9396d54..969ffd13f 100644 --- a/ggml/src/ggml-cpu/simd-gemm.h +++ b/ggml/src/ggml-cpu/simd-gemm.h @@ -5,11 +5,11 @@ #include "simd-mappings.h" // TODO: add support for sizeless vector types -#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic) +#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v) // TODO: untested on avx512 // These are in units of GGML_F32_EPR -#if defined(__AVX512F__) || defined (__ARM_NEON__) +#if defined(__AVX512F__) || defined(__ARM_NEON) static constexpr int GEMM_RM = 4; static constexpr int GEMM_RN = 4; // 16+4+1 = 25/32 #elif defined(__AVX2__) || defined(__AVX__) @@ -149,7 +149,7 @@ static void simd_gemm( C += N; } } -#elif defined(GGML_SIMD) && defined(__riscv_v_intrinsic) +#elif defined(GGML_SIMD) && defined(__riscv_v) // RM accumulators + 1 B vector = RM + 1 <= 8 => RM <= 7 // Microkernel: C[RM x vl] += A[RM x K] * B[K x N] template <int RM> diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index 89a5afa9c..d3d00bf6f 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -14,7 +14,9 @@ #include <arm_neon.h> #endif -#if defined(__riscv_v_intrinsic) +// __riscv_v_intrinsic reports compiler API support and can be defined without +// enabling V; use the ISA feature macro to select RVV code. +#if defined(__riscv_v) #include <riscv_vector.h> #endif @@ -42,6 +44,7 @@ extern "C" { #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) neon_compute_fp32_to_fp16(x) #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x) + #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x) static inline float neon_compute_fp16_to_fp32(ggml_fp16_t h) { __fp16 tmp; @@ -126,6 +129,8 @@ extern float ggml_table_f32_e8m0_half[1 << 8]; // defined in ggml-cpu.c, initialized in ggml_cpu_init() extern float ggml_table_f32_ue4m3[1 << 8]; +extern float ggml_table_f32_f8_e4m3[1 << 8]; + // Use lookup table for E8M0 on x86 (faster than bit manipulation) #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) #define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)] @@ -140,9 +145,25 @@ extern float ggml_table_f32_ue4m3[1 << 8]; #define GGML_CPU_UE4M3_TO_FP32(x) ggml_ue4m3_to_fp32(x) #endif -// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32, -// so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON. -// This is also true for POWER9. +// Pre AVX512 CPUs (AVX2 with F16C, such as i5-8th gen or i7-4790) +// do not produce codegen as checks below don't test for __F16C__ +// but this check does. +#if defined(__F16C__) +// force FP16 -> FP32 conversion to use LUT instead +// see: https://github.com/ggml-org/llama.cpp/issues/30342 +//#define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x) +#define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x) +#endif + +// a branch that defines a GGML_CPU_COMPUTE_* macro must also select it, or the fallback takes over silently +#if defined(GGML_CPU_COMPUTE_FP16_TO_FP32) && !defined(GGML_CPU_FP16_TO_FP32) && !defined(__F16C__) // see: https://github.com/ggml-org/llama.cpp/issues/30342 +#error "a branch defined GGML_CPU_COMPUTE_FP16_TO_FP32 without selecting it as GGML_CPU_FP16_TO_FP32" +#endif +#if defined(GGML_CPU_COMPUTE_FP32_TO_FP16) && !defined(GGML_CPU_FP32_TO_FP16) +#error "a branch defined GGML_CPU_COMPUTE_FP32_TO_FP16 without selecting it as GGML_CPU_FP32_TO_FP16" +#endif + +// fall back to the lookup table / the portable software conversion for whichever direction no branch selected #if !defined(GGML_CPU_FP16_TO_FP32) inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) { uint16_t s; @@ -1223,6 +1244,24 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { #define GGML_F16_STEP GGML_F32_STEP #define GGML_F16_EPR GGML_F32_EPR +static inline uint32x4_t __lzs_f32cx4_to_f16(float32x4_t v_f) { + float32x4_t v_base = vec_mul(vec_mul(vec_abs(v_f), vec_splats(0x1.0p+112f)), vec_splats(0x1.0p-110f)); + + const uint32x4_t v_w = (uint32x4_t)v_f; + const uint32x4_t v_shl1_w = vec_add(v_w, v_w); + const uint32x4_t v_sign = vec_and(v_w, vec_splats(UINT32_C(0x80000000))); + const uint32x4_t v_bias = vec_max(vec_and(v_shl1_w, vec_splats(UINT32_C(0xFF000000))), vec_splats(UINT32_C(0x71000000))); + + v_base = vec_add((float32x4_t)vec_add(vec_sr(v_bias, 1), vec_splats(UINT32_C(0x07800000))), v_base); + + const uint32x4_t v_bits = (uint32x4_t)v_base; + const uint32x4_t v_nonsign = vec_add(vec_and(vec_sr(v_bits, 13), vec_splats(UINT32_C(0x00007C00))), + vec_and(v_bits, vec_splats(UINT32_C(0x00000FFF)))); + const uint32x4_t v_is_nan = (uint32x4_t)vec_cmpgt(v_shl1_w, vec_splats(UINT32_C(0xFF000000))); + + return vec_or(vec_sr(v_sign, 16), vec_sel(v_nonsign, vec_splats(UINT32_C(0x7E00)), v_is_nan)); +} + static inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) { float tmp[4]; @@ -1236,15 +1275,9 @@ static inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) { } static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) { - float arr[4]; - - // note: keep type-cast here to prevent compiler bugs - // see: https://github.com/ggml-org/llama.cpp/issues/12846 - vec_xst(v_y, 0, (float *)(arr)); - - for (int i = 0; i < 4; i++) { - x[i] = GGML_CPU_FP32_TO_FP16(arr[i]); - } + const uint32x4_t v_h = __lzs_f32cx4_to_f16(v_y); + const uint64_t tmp = ((uint64x2_t)vec_pack(v_h, v_h))[0]; + memcpy(x, &tmp, sizeof(tmp)); } #define GGML_F16_VEC GGML_F32x4 @@ -1275,7 +1308,7 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) { #define GGML_BF16_FMA_HI(acc, x, y) \ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y)) -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) // compatible with vlen >= 128 diff --git a/ggml/src/ggml-cpu/spacemit/ime.cpp b/ggml/src/ggml-cpu/spacemit/ime.cpp index 81e86b9dd..dc51ad0aa 100644 --- a/ggml/src/ggml-cpu/spacemit/ime.cpp +++ b/ggml/src/ggml-cpu/spacemit/ime.cpp @@ -198,7 +198,7 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS> class tensor_ case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q5_K: - //case GGML_TYPE_MXFP4: + case GGML_TYPE_MXFP4: forward_mul_mat(params, op); return true; default: @@ -682,7 +682,12 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS> class tensor_ const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]); - GGML_ASSERT(i02 >= 0 && i02 < n_as); + GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as)); + + if (i02 == -1) { + memset((char *) dst->data + id * nb1 + iid1 * nb2, 0, ne0 * sizeof(float)); + continue; + } MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = { id, iid1 }; matrix_row_counts[i02] += 1; @@ -1041,6 +1046,10 @@ class tensor_traits_common : public tensor_traits_base { return true; } case GGML_OP_FLASH_ATTN_EXT: + // K/V row lists are handled by the generic CPU path + if (op->src[5] != nullptr) { + return false; + } forward_flash_attn_ext_f16(params, op); return true; case GGML_OP_CONT: @@ -1145,9 +1154,17 @@ class tensor_traits_common : public tensor_traits_base { ggml_compute_forward_concat(params, op); } return true; - // TODO For GGML_OP_GATED_DELTA_NET - // case GGML_OP_GATED_DELTA_NET: - // return true; + case GGML_OP_GATED_DELTA_NET: + if (params->use_ref) { + return false; + } + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_gated_delta_net_f32(params, op); + return true; + default: + GGML_ABORT("fatal error"); + } default: break; } @@ -1171,7 +1188,7 @@ class tensor_traits_common : public tensor_traits_base { const int64_t DK = nek0; const int64_t DV = nev0; - const bool supported_prec = (dst->op_params[3] == GGML_PREC_F32 || dst->op_params[3] == GGML_PREC_DEFAULT); + const bool supported_prec = (ggml_prec_get_acc(dst) == GGML_PREC_F32 || ggml_prec_get_acc(dst) == GGML_PREC_BF16 || ggml_prec_get_acc(dst) == GGML_PREC_UNDEFINED); const bool supported_types = (q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16); const bool supported_shape = (DK > 0 && DK <= 128 && DV > 0 && DV <= 128); const bool supported_vlen = (__riscv_vlenb() == 128); @@ -1306,11 +1323,10 @@ static const ggml::cpu::tensor_traits * ggml_riscv64_spacemit_get_optimal_repack case GGML_TYPE_Q4_1: { #if defined(RISCV64_SPACEMIT_IME2) - // TODO - // if (cur->ne[1] % 32 == 0 && cur->ne[0] % 256 == 0 && - // (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { - // return &ggml::cpu::riscv64_spacemit::q4_1_32x256_q8_0; - // } + if (cur->ne[1] % 32 == 0 && cur->ne[0] % 256 == 0 && + (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q4_1_32x256_q8_0; + } if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { return &ggml::cpu::riscv64_spacemit::q4_1_32x32_q8_0; @@ -1366,10 +1382,10 @@ static const ggml::cpu::tensor_traits * ggml_riscv64_spacemit_get_optimal_repack case GGML_TYPE_MXFP4: { #if defined(RISCV64_SPACEMIT_IME2) - // TODO - // if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { - // return &ggml::cpu::riscv64_spacemit::mxfp4_32x32_q8_0; - // } + if (ggml_n_dims(cur) == 2 && cur->ne[0] % 32 == 0 && cur->ne[1] % 32 == 0 && + (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::mxfp4_32x32_q8_0; + } #endif } break; @@ -1484,6 +1500,9 @@ static const ggml_backend_buffer_i ggml_backend_riscv64_spacemit_buffer_i = { /* .cpy_tensor = */ nullptr, /* .clear = */ ggml_backend_riscv64_spacemit_buffer_clear, /* .reset = */ nullptr, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; static const char * ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name(ggml_backend_buffer_type_t buft) { @@ -1601,6 +1620,9 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) { return false; } + if (op->src[0]->type == GGML_TYPE_MXFP4 && !ggml_is_contiguous(op->src[1])) { + return false; + } if (op->src[1]->type == GGML_TYPE_F32) { return true; } @@ -1646,7 +1668,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { case GGML_OP_SUM_ROWS: case GGML_OP_GET_ROWS: case GGML_OP_CONCAT: - // case GGML_OP_GATED_DELTA_NET: + case GGML_OP_GATED_DELTA_NET: return (ggml::cpu::tensor_traits *) (&ggml::cpu::riscv64_spacemit::rvv_impl); default: // GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp b/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp index 0c7a036a9..92b84a90b 100644 --- a/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +++ b/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp @@ -2999,8 +2999,107 @@ void gemm_kernel_i8i4_hp_m1(size_t blk_len, "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "ft0", "ft1"); } } else { - // TODO: support quant_b_zp for i8i4 hp kernel - GGML_ABORT("gemm_kernel_i8i4_hp_m1 with quant_b_zp is not supported yet"); + const _Float16 hp_scale_0125 = 0.125; + for (size_t ni = 0; ni < count_n; ni += 32) { + const size_t nb_real = std::min<size_t>(NB_COLS, count_n - ni); + if (nb_real != NB_COLS) { + break; + } + + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * b_tile_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + + asm volatile( + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "mv t4, %[BK] \n\t" + "li t0, 0x4c00 \n\t" + "fmv.h.x fa0, t0 \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + "li t1, 4608 \n\t" + "add s6, %[B], t1 \n\t" + "li t5, 8 \n\t" + "addi t6, %[A], 288 \n\t" + "flh ft1, (t6) \n\t" + "addi t6, %[A], 272 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v16, v18, v18 \n\t" + "vxor.vv v17, v18, v18 \n\t" + "vxor.vv v18, v18, v18 \n\t" + "vxor.vv v19, v18, v18 \n\t" + + "INNER_BLK_LOOP%=: \n\t" + "flh fa1, (t6) \n\t" + "addi t6, t6, 2 \n\t" + "flh ft0, (%[A]) \n\t" + "addi %[A], %[A], 2 \n\t" + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v3, (%[A]) \n\t" + "addi %[A], %[A], 32 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v8, (%[B]) \n\t" + "addi %[B], %[B], 64 \n\t" + "vl4r.v v4, (%[B]) \n\t" + "addi %[B], %[B], 512 \n\t" + "vfmul.vf v8, v8, ft0 \n\t" + "vfmul.vf v9, v8, fa0 \n\t" + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v11, (s6) \n\t" + "addi s6, s6, 32 \n\t" + "vfwcvt.f.xu.v v13, v11 \n\t" + "fmul.h ft2, fa1, %[HP0125] \n\t" + "vsetvli t0, x0, e16, mf2 \n\t" + "vfmul.vv v10, v13, v8 \n\t" + "vfmul.vf v10, v10, ft2 \n\t" + "vfwmacc.vf v31, ft1, v10 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v0, v8, v9, 3 \n\t" + "vsrl.vi v28, v3, 4 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vnpack4.vv v2, v3, v3, 3 \n\t" + "vnpack4.vv v3, v28, v28, 3 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vmadotsu.hp v16, v3, v4, v0, 4, i4 \n\t" + "vmadotsu.hp v17, v3, v5, v0, 5, i4 \n\t" + "vmadotsu.hp v18, v3, v6, v0, 6, i4 \n\t" + "vmadotsu.hp v19, v3, v7, v0, 7, i4 \n\t" + "vmadotu.hp v16, v2, v4, v0, 0, i4 \n\t" + "vmadotu.hp v17, v2, v5, v0, 1, i4 \n\t" + "vmadotu.hp v18, v2, v6, v0, 2, i4 \n\t" + "vmadotu.hp v19, v2, v7, v0, 3, i4 \n\t" + + "addi t5, t5, -1 \n\t" + "bgtz t5, INNER_BLK_LOOP%= \n\t" + + "vpack.vv v8, v16, v17, 1 \n\t" + "vpack.vv v12, v18, v19, 1 \n\t" + "vpack.vv v20, v8, v12, 2 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "addi t4, t4, -1 \n\t" + "vfwmacc.vf v31, ft1, v20 \n\t" + + "addi %[A], t6, 2 \n\t" + "mv %[B], s6 \n\t" + + "bgtz t4, BLK_LOOP%= \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vse32.v v31, (%[DST]) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [BK] "r"(k_blks), [HP0125] "f"(hp_scale_0125) + : "t0", "t1", "t2", "t3", "t4", "t5", "t6", "s6", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", + "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "ft0", "ft1", "ft2"); + } } } @@ -3754,7 +3853,7 @@ void gemm_kernel_i8mxfp4_m1(size_t blk_len, using blk_type = nrow_block_mxfp4<NB_COLS>; GGML_ASSERT(blk_len == K_TILE); - GGML_ASSERT(count_m == 1); + GGML_ASSERT(count_m >= 1); GGML_UNUSED(quant_b_zp); const size_t a_blk_stride = q8_blk_size(blk_len, true); @@ -3923,7 +4022,7 @@ void gemm_kernel_i8mxfp4_m1(size_t blk_len, : : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6", "s7", "v0", "v1", - "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v18", "v19", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); } } @@ -3943,7 +4042,7 @@ void gemm_kernel_i8mxfp4_m4(size_t blk_len, using blk_type = nrow_block_mxfp4<NB_COLS>; GGML_ASSERT(blk_len == K_TILE); - GGML_ASSERT(count_m == 4); + GGML_ASSERT(count_m >= 4); GGML_UNUSED(quant_b_zp); const size_t a_blk_stride = q8_blk_size(blk_len, true); @@ -5687,20 +5786,10 @@ size_t gemm_kernel_i8mxfp4(size_t blk_len, size_t k_blks, size_t ldc) { if (count_m >= 4) { -#if 1 - gemm_kernel_i8mxfp4_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, - k_blks, ldc); -#else gemm_kernel_i8mxfp4_m4(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); -#endif return 4; } else { -#if 1 - gemm_kernel_i8mxfp4_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, - k_blks, ldc); -#else gemm_kernel_i8mxfp4_m1(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); -#endif return 1; } } diff --git a/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp b/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp index 13b84dcbe..e27a006ea 100644 --- a/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +++ b/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp @@ -3175,4 +3175,199 @@ template void forward_get_rows<int16_t>(ggml_compute_params * params, ggml_tenso template void forward_concat<int32_t>(ggml_compute_params * params, ggml_tensor * op); template void forward_concat<int16_t>(ggml_compute_params * params, ggml_tensor * op); +static inline float rvv_dot_f32(const float * x, const float * y, int64_t n) { + size_t vl = __riscv_vsetvlmax_e32m4(); + vfloat32m4_t vsum = __riscv_vfmv_v_f_f32m4(0.0f, vl); + for (int64_t i = 0; i < n; i += vl) { + vl = __riscv_vsetvl_e32m4(n - i); + vfloat32m4_t ax = __riscv_vle32_v_f32m4(x + i, vl); + vfloat32m4_t ay = __riscv_vle32_v_f32m4(y + i, vl); + vsum = __riscv_vfmacc_vv_f32m4_tu(vsum, ax, ay, vl); + } + vl = __riscv_vsetvlmax_e32m4(); + vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); + vs = __riscv_vfredusum_vs_f32m4_f32m1(vsum, vs, vl); + return __riscv_vfmv_f_s_f32m1_f32(vs); +} + +static void forward_gated_delta_net_one_chunk( + const ggml_compute_params * params, + ggml_tensor * dst, + int64_t ir0, + int64_t ir1) { + + ggml_tensor * src_q = dst->src[0]; + ggml_tensor * src_k = dst->src[1]; + ggml_tensor * src_v = dst->src[2]; + ggml_tensor * src_g = dst->src[3]; + ggml_tensor * src_beta = dst->src[4]; + ggml_tensor * src_state = dst->src[5]; + + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + + GGML_ASSERT(ggml_is_contiguous_rows(src_q)); + GGML_ASSERT(ggml_is_contiguous_rows(src_k)); + GGML_ASSERT(ggml_is_contiguous_rows(src_v)); + GGML_ASSERT(ggml_is_contiguous(src_g)); + GGML_ASSERT(ggml_is_contiguous(src_beta)); + GGML_ASSERT(ggml_is_contiguous(src_state)); + + GGML_ASSERT(src_g->ne[0] == 1 || src_g->ne[0] == S_v); + GGML_ASSERT(src_beta->ne[0] == 1); + + GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne); + GGML_TENSOR_LOCALS(size_t, nbq, src_q, nb); + GGML_TENSOR_LOCALS(int64_t, nek, src_k, ne); + GGML_TENSOR_LOCALS(size_t, nbk, src_k, nb); + GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne); + GGML_TENSOR_LOCALS(size_t, nbv, src_v, nb); + GGML_TENSOR_LOCALS(int64_t, neg, src_g, ne); + GGML_TENSOR_LOCALS(size_t, nbg, src_g, nb); + GGML_TENSOR_LOCALS(size_t, nbb, src_beta, nb); + + const bool kda = (neg0 == S_v); + + const int64_t K = ggml_get_op_params_i32(dst, 0); + GGML_ASSERT(K >= 1); + const int64_t state_seq_stride = src_state->nb[3] / sizeof(float); + + const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0); + const int ith = params->ith; + + float * delta = (float *) params->wdata + ith * per_thread + CACHE_LINE_SIZE_F32; + float * state_work = K > 1 ? (delta + S_v) : nullptr; + + const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs; + const int64_t state_size_per_snap = S_v * S_v * H * n_seqs; + float * attn_out_base = (float *) dst->data; + float * state_out_base = (float *) dst->data + attn_score_elems; + + const float * state_in_base = (const float *) src_state->data; + + const int64_t rq3 = nev3 / neq3; + const int64_t rk3 = nev3 / nek3; + + const float scale = 1.0f / sqrtf((float) S_v); + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t iv1 = ir % H; + const int64_t iv3 = ir / H; + + const int64_t iq1 = iv1 % neq1; + const int64_t ik1 = iv1 % nek1; + + const int64_t iq3 = iv3 / rq3; + const int64_t ik3 = iv3 / rk3; + + float * s_out = (K > 1) + ? state_work + : state_out_base + (iv3 * H + iv1) * S_v * S_v; + + const float * s_in = state_in_base + iv3 * state_seq_stride + iv1 * S_v * S_v; + memcpy(s_out, s_in, S_v * S_v * sizeof(float)); + + float * attn_data = attn_out_base + (iv3 * n_tokens * H + iv1) * S_v; + + for (int64_t t = 0; t < n_tokens; t++) { + const float * q_d = (const float *) ((const char *) src_q->data + iq3 * nbq3 + t * nbq2 + iq1 * nbq1); + const float * k_d = (const float *) ((const char *) src_k->data + ik3 * nbk3 + t * nbk2 + ik1 * nbk1); + const float * v_d = (const float *) ((const char *) src_v->data + iv3 * nbv3 + t * nbv2 + iv1 * nbv1); + + const float beta_val = *(const float *) ((const char *) src_beta->data + iv3 * nbb3 + t * nbb2 + iv1 * nbb1); + const float * g_d = (const float *) ((const char *) src_g->data + iv3 * nbg3 + t * nbg2 + iv1 * nbg1); + + if (kda) { + for (int64_t i = 0; i < S_v; ++i) { + delta[i] = expf(g_d[i]); + } + for (int64_t j = 0; j < S_v; ++j) { + float * row = s_out + j * S_v; + const float * d = delta; + int64_t remaining = S_v; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t vr = __riscv_vle32_v_f32m4(row, vl); + vfloat32m4_t vd = __riscv_vle32_v_f32m4(d, vl); + vr = __riscv_vfmul_vv_f32m4(vr, vd, vl); + __riscv_vse32_v_f32m4(row, vr, vl); + row += vl; + d += vl; + remaining -= vl; + } + } + } else { + const float gate = expf(g_d[0]); + float * p = s_out; + int64_t remaining = S_v * S_v; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t v = __riscv_vle32_v_f32m4(p, vl); + v = __riscv_vfmul_vf_f32m4(v, gate, vl); + __riscv_vse32_v_f32m4(p, v, vl); + p += vl; + remaining -= vl; + } + } + + for (int64_t j = 0; j < S_v; ++j) { + const float sum = rvv_dot_f32(s_out + j * S_v, k_d, S_v); + delta[j] = (v_d[j] - sum) * beta_val; + } + + for (int64_t j = 0; j < S_v; ++j) { + float * row = s_out + j * S_v; + const float * k = k_d; + const float dj = delta[j]; + int64_t remaining = S_v; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t vr = __riscv_vle32_v_f32m4(row, vl); + vfloat32m4_t vk = __riscv_vle32_v_f32m4(k, vl); + vr = __riscv_vfmacc_vf_f32m4(vr, dj, vk, vl); + __riscv_vse32_v_f32m4(row, vr, vl); + row += vl; + k += vl; + remaining -= vl; + } + } + + for (int64_t j = 0; j < S_v; ++j) { + const float sum = rvv_dot_f32(s_out + j * S_v, q_d, S_v); + attn_data[j] = sum * scale; + } + + attn_data += S_v * H; + + if (K > 1) { + const int64_t target_slot = n_tokens - 1 - t; + if (target_slot >= 0 && target_slot < K) { + float * curr_state_o = state_out_base + target_slot * state_size_per_snap + + (iv3 * H + iv1) * S_v * S_v; + memcpy(curr_state_o, s_out, S_v * S_v * sizeof(float)); + } + } + } + } +} + +void forward_gated_delta_net_f32(ggml_compute_params * params, ggml_tensor * dst) { + ggml_tensor * src_v = dst->src[2]; + + int ith = params->ith; + int nth = params->nth; + + const auto nr = src_v->ne[1] * src_v->ne[3]; + const auto task_per_thread = (nr + nth - 1) / nth; + + const auto task_begin = ith * task_per_thread; + const auto task_end = std::min((ith + 1) * task_per_thread, nr); + + if (task_begin < task_end) { + forward_gated_delta_net_one_chunk(params, dst, task_begin, task_end); + } +} + } // namespace spacemit_kernels::rvv diff --git a/ggml/src/ggml-cpu/spacemit/rvv_kernels.h b/ggml/src/ggml-cpu/spacemit/rvv_kernels.h index edddf957c..a085613a3 100644 --- a/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +++ b/ggml/src/ggml-cpu/spacemit/rvv_kernels.h @@ -62,6 +62,8 @@ void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op); void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op); +void forward_gated_delta_net_f32(ggml_compute_params * params, ggml_tensor * op); + void forward_cont_with_permute(ggml_compute_params * params, ggml_tensor * op); void forward_cpy_with_permute(ggml_compute_params * params, ggml_tensor * op); diff --git a/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp b/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp index bffa737c7..5819ac07c 100644 --- a/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp +++ b/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp @@ -244,16 +244,6 @@ static void tiled_run_microtile_avx2(const tiled_tile_src0 & src0, const tiled_t const int bs_stride = (nkr == 1) ? TILED_TILE_ROWS : TILED_MICRO; const int bs_off = seff * TILED_MICRO; - // Horizontal sum helper: converts 8x32-bit int lane inside a YMM register to scalar int32 - auto hsum256_epi32 = [](const __m256i v) -> int32_t { - __m128i low = _mm256_castsi256_si128(v); - __m128i high = _mm256_extracti128_si256(v, 1); - __m128i sum128 = _mm_add_epi32(low, high); - sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_SHUFFLE(2, 3, 0, 1))); - sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_SHUFFLE(1, 0, 3, 2))); - return _mm_cvtsi128_si32(sum128); - }; - for (int i = 0; i < TILED_MICRO; i++) { const int ar = i0 + i; const int d_off = seff * TILED_MICRO + ar; @@ -362,15 +352,20 @@ static void tiled_run_microtile_avx2(const tiled_tile_src0 & src0, const tiled_t } } - // OPTIMIZATION 2: Vectorized Epilogue with FMA3 gating - // Reduce accumulators into a 256-bit vector across 8 columns - alignas(32) int32_t s1_vals[GROUP]; - #pragma GCC unroll 8 - for (int t = 0; t < GROUP; t++) { - s1_vals[t] = hsum256_epi32(acc[t]); - } - __m256i s1_vec = _mm256_load_si256((const __m256i *) s1_vals); + // Reduce the eight accumulators to column sums in acc[0]..acc[7] order, with additions modulo 2^32. + const __m256i ab = _mm256_hadd_epi32(acc[0], acc[1]); + const __m256i cd = _mm256_hadd_epi32(acc[2], acc[3]); + const __m256i ef = _mm256_hadd_epi32(acc[4], acc[5]); + const __m256i gh = _mm256_hadd_epi32(acc[6], acc[7]); + + // Each 128-bit half now holds four column sums over input lanes 0..3 or 4..7. + const __m256i abcd = _mm256_hadd_epi32(ab, cd); + const __m256i efgh = _mm256_hadd_epi32(ef, gh); + const __m256i lo = _mm256_permute2x128_si256(abcd, efgh, 0x20); + const __m256i hi = _mm256_permute2x128_si256(abcd, efgh, 0x31); + __m256i s1_vec = _mm256_add_epi32(lo, hi); + // OPTIMIZATION 2: Vectorized Epilogue with FMA3 gating if constexpr (BIAS != 0 && ACTBIAS) { // act-bias correction: raw = desired + 128*sum_s scales[s]*w_bsum[s]; corr precomputed in repack_src0 s1_vec = _mm256_sub_epi32(s1_vec, _mm256_set1_epi32(mins_row[0])); diff --git a/ggml/src/ggml-cpu/unary-ops.cpp b/ggml/src/ggml-cpu/unary-ops.cpp index 1d8344436..f5d57d6f9 100644 --- a/ggml/src/ggml-cpu/unary-ops.cpp +++ b/ggml/src/ggml-cpu/unary-ops.cpp @@ -78,7 +78,7 @@ static inline float op_expm1(float x) { } static inline float op_softplus(float x) { - return (x > 20.0f) ? x : logf(1.0f + expf(x)); + return (x > 20.0f) ? x : log1pf(expf(x)); } static inline float op_floor(float x) { diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index 3918a6f4d..9ff7335b8 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -84,7 +84,7 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G } // reduce sum1,sum2 to sum1 GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) int vl = __riscv_vsetvlmax_e32m8(); vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); vfloat32m8_t vsum; @@ -195,7 +195,7 @@ void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * sumf += (ggml_float)_mm_cvtss_f32(g); #undef LOAD -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfwma) +#elif defined(__riscv_v) && defined(__riscv_zvfbfwma) size_t vl = __riscv_vsetvlmax_e32m4(); // initialize accumulators to all zeroes @@ -318,7 +318,7 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum3_hi); sumf = ggml_sve_sum_f32x2(sum1_lo, sum1_hi); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) #if defined(__riscv_zvfh) int vl = __riscv_vsetvlmax_e32m2(); vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); @@ -377,6 +377,85 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G *s = sumf; } +void ggml_vec_dot_f16_f32(int n, float * GGML_RESTRICT s, size_t bs, const ggml_fp16_t * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) { + assert(nrc == 1); + GGML_UNUSED(nrc); + GGML_UNUSED(bx); + GGML_UNUSED(by); + GGML_UNUSED(bs); + + ggml_float sumf = 0.0; + int i = 0; + + // on AVX512 and AVX with F16C, x is widened to F32 in-register and accumulated in F32 over 4 independent sums +#if defined(__AVX512F__) + __m512 acc0 = _mm512_setzero_ps(); + __m512 acc1 = _mm512_setzero_ps(); + __m512 acc2 = _mm512_setzero_ps(); + __m512 acc3 = _mm512_setzero_ps(); + for (; i + 64 <= n; i += 64) { + acc0 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 0))), _mm512_loadu_ps(y + i + 0), acc0); + acc1 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 16))), _mm512_loadu_ps(y + i + 16), acc1); + acc2 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 32))), _mm512_loadu_ps(y + i + 32), acc2); + acc3 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 48))), _mm512_loadu_ps(y + i + 48), acc3); + } + for (; i + 16 <= n; i += 16) { + acc0 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i))), _mm512_loadu_ps(y + i), acc0); + } + sumf = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3))); +#elif defined(__AVX__) && defined(__F16C__) + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + for (; i + 32 <= n; i += 32) { + acc0 = GGML_F32x8_FMA(acc0, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 0))), _mm256_loadu_ps(y + i + 0)); + acc1 = GGML_F32x8_FMA(acc1, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 8))), _mm256_loadu_ps(y + i + 8)); + acc2 = GGML_F32x8_FMA(acc2, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 16))), _mm256_loadu_ps(y + i + 16)); + acc3 = GGML_F32x8_FMA(acc3, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 24))), _mm256_loadu_ps(y + i + 24)); + } + for (; i + 8 <= n; i += 8) { + acc0 = GGML_F32x8_FMA(acc0, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i))), _mm256_loadu_ps(y + i)); + } + const __m256 acc = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); + __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1)); + r = _mm_add_ps(r, _mm_movehl_ps(r, r)); + r = _mm_add_ss(r, _mm_movehdup_ps(r)); + sumf = _mm_cvtss_f32(r); +#elif defined(GGML_SIMD) + // convert chunks of x to F32 and reuse the F32 SIMD dot + // TODO: convert F16 to F32 in-register and fuse the dot (needs a new SIMD macro); also avoid the per-chunk call overhead for tiny k + float tx[64]; + for (; i + 64 <= n; i += 64) { + const ggml_fp16_t * GGML_RESTRICT xr = x + i; + float * GGML_RESTRICT tr = tx; + for (int j = 0; j < 64; ++j) { + tr[j] = GGML_CPU_FP16_TO_FP32(xr[j]); + } + float v; + ggml_vec_dot_f32(64, &v, 0, tx, 0, y + i, 0, 1); + sumf += v; + } + if (i < n) { + const int nr = n - i; + const ggml_fp16_t * GGML_RESTRICT xr = x + i; + float * GGML_RESTRICT tr = tx; + for (int j = 0; j < nr; ++j) { + tr[j] = GGML_CPU_FP16_TO_FP32(xr[j]); + } + float v; + ggml_vec_dot_f32(nr, &v, 0, tx, 0, y + i, 0, 1); + sumf += v; + i = n; + } +#endif + for (; i < n; ++i) { + sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*y[i]); + } + + *s = sumf; +} + void ggml_vec_silu_f32(const int n, float * y, const float * x) { int i = 0; #if defined(__AVX512F__) && defined(__AVX512DQ__) @@ -401,7 +480,11 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) { for (; i + 3 < n; i += 4) { vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i))); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(ggml_v_silu(vec_xl(0, x + i)), 0, y + i); + } +#elif defined(__riscv_v) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); @@ -438,7 +521,11 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * for (; i + 3 < n; i += 4) { vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i))); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(vec_mul(ggml_v_silu(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i); + } +#elif defined(__riscv_v) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); @@ -452,6 +539,92 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * } } +#ifndef GGML_GELU_QUICK_FP16 +// computed in f32 without the fp16 table, which loses precision on f32 input +void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) { + int i = 0; +#if defined(__AVX512F__) && defined(__AVX512DQ__) + for (; i + 15 < n; i += 16) { + _mm512_storeu_ps(y + i, ggml_v_gelu_quick(_mm512_loadu_ps(x + i))); + } +#elif defined(__AVX2__) && defined(__FMA__) + for (; i + 7 < n; i += 8) { + _mm256_storeu_ps(y + i, ggml_v_gelu_quick(_mm256_loadu_ps(x + i))); + } +#elif defined(__SSE2__) + for (; i + 3 < n; i += 4) { + _mm_storeu_ps(y + i, ggml_v_gelu_quick(_mm_loadu_ps(x + i))); + } +#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__) + const int vlen = svcntw(); + for (; i < n; i += vlen) { + const svbool_t pg = svwhilelt_b32_s32(i, n); + svst1_f32(pg, y + i, ggml_v_gelu_quick(pg, svld1_f32(pg, x + i))); + } +#elif defined(__ARM_NEON) && defined(__aarch64__) + for (; i + 3 < n; i += 4) { + vst1q_f32(y + i, ggml_v_gelu_quick(vld1q_f32(x + i))); + } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(ggml_v_gelu_quick(vec_xl(0, x + i)), 0, y + i); + } +#elif defined(__riscv_v_intrinsic) + for (int vl; i < n; i += vl) { + vl = __riscv_vsetvl_e32m2(n - i); + vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); + vfloat32m2_t vy = ggml_v_gelu_quick_m2(vx, vl); + __riscv_vse32_v_f32m2(&y[i], vy, vl); + } +#endif + for (; i < n; ++i) { + y[i] = ggml_gelu_quick_f32(x[i]); + } +} + +void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) { + int i = 0; +#if defined(__AVX512F__) && defined(__AVX512DQ__) + for (; i + 15 < n; i += 16) { + _mm512_storeu_ps(y + i, _mm512_mul_ps(ggml_v_gelu_quick(_mm512_loadu_ps(x + i)), _mm512_loadu_ps(g + i))); + } +#elif defined(__AVX2__) && defined(__FMA__) + for (; i + 7 < n; i += 8) { + _mm256_storeu_ps(y + i, _mm256_mul_ps(ggml_v_gelu_quick(_mm256_loadu_ps(x + i)), _mm256_loadu_ps(g + i))); + } +#elif defined(__SSE2__) + for (; i + 3 < n; i += 4) { + _mm_storeu_ps(y + i, _mm_mul_ps(ggml_v_gelu_quick(_mm_loadu_ps(x + i)), _mm_loadu_ps(g + i))); + } +#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__) + const int vlen = svcntw(); + for (; i < n; i += vlen) { + const svbool_t pg = svwhilelt_b32_s32(i, n); + svst1_f32(pg, y + i, svmul_f32_x(pg, ggml_v_gelu_quick(pg, svld1_f32(pg, x + i)), svld1_f32(pg, g + i))); + } +#elif defined(__ARM_NEON) && defined(__aarch64__) + for (; i + 3 < n; i += 4) { + vst1q_f32(y + i, vmulq_f32(ggml_v_gelu_quick(vld1q_f32(x + i)), vld1q_f32(g + i))); + } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(vec_mul(ggml_v_gelu_quick(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i); + } +#elif defined(__riscv_v_intrinsic) + for (int vl; i < n; i += vl) { + vl = __riscv_vsetvl_e32m2(n - i); + vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); + vfloat32m2_t vg = __riscv_vle32_v_f32m2(&g[i], vl); + vfloat32m2_t vy = __riscv_vfmul_vv_f32m2(ggml_v_gelu_quick_m2(vx, vl), vg, vl); + __riscv_vse32_v_f32m2(&y[i], vy, vl); + } +#endif + for (; i < n; ++i) { + y[i] = ggml_gelu_quick_f32(x[i]) * g[i]; + } +} +#endif + ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean) { int i = 0; ggml_float sum = 0; @@ -508,7 +681,7 @@ ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const floa val = vec_mul(val, val); sum += (ggml_float)vec_hsum_f32x4(val); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1); for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); @@ -581,7 +754,13 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float vst1q_f32(y + i, val); sum += (ggml_float)vaddvq_f32(val); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + float32x4_t v_val = ggml_v_expf(vec_sub(vec_xl(0, x + i), vec_splats(max))); + vec_xst(v_val, 0, y + i); + sum += (ggml_float)vec_hsum_f32x4(v_val); + } +#elif defined(__riscv_v) vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1); for (int avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m2(n - i); diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index 59fa88f2c..c54ed3f07 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -44,7 +44,6 @@ inline static ggml_float ggml_sve_sum_f32x2(svfloat32_t sum_lo, svfloat32_t sum_ #endif #define GGML_GELU_FP16 -#define GGML_GELU_QUICK_FP16 #define GGML_SOFT_MAX_UNROLL 4 #define GGML_VEC_DOT_UNROLL 2 @@ -71,6 +70,7 @@ extern ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16]; void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc); void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc); void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc); +void ggml_vec_dot_f16_f32(int n, float * GGML_RESTRICT s, size_t bs, const ggml_fp16_t * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc); void ggml_vec_silu_f32(const int n, float * y, const float * x); ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean); //it will also center y ( y = y - mean ) @@ -207,7 +207,7 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG sumf[0] = ggml_sve_sum_f32x2(sum_0_lo, sum_0_hi); sumf[1] = ggml_sve_sum_f32x2(sum_1_lo, sum_1_hi); np = n; - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) #if defined(__riscv_zvfh) size_t vl = __riscv_vsetvlmax_e32m4(); @@ -397,7 +397,7 @@ inline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const svst1_f32(pg, y + np2, ay1); } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl); @@ -436,6 +436,41 @@ inline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const #endif } +// y (F32) += x (F16)*v +// on AVX512, AVX with F16C, NEON and RVV, x is widened to F32 in-register and fused into the multiply-add +inline static void ggml_vec_mad_f16_f32(const int n, float * GGML_RESTRICT y, const ggml_fp16_t * GGML_RESTRICT x, const float v) { + int i = 0; +#if defined(__AVX512F__) + const __m512 vx = _mm512_set1_ps(v); + for (; i + 16 <= n; i += 16) { + const __m512 ax = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i))); + _mm512_storeu_ps(y + i, _mm512_fmadd_ps(ax, vx, _mm512_loadu_ps(y + i))); + } +#elif defined(__AVX__) && defined(__F16C__) + const __m256 vx = _mm256_set1_ps(v); + for (; i + 8 <= n; i += 8) { + const __m256 ax = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i))); + _mm256_storeu_ps(y + i, GGML_F32x8_FMA(_mm256_loadu_ps(y + i), ax, vx)); + } +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) && defined(__ARM_FP16_FORMAT_IEEE) + const float32x4_t vx = vdupq_n_f32(v); + for (; i + 8 <= n; i += 8) { + const float16x8_t ax = vld1q_f16((const __fp16 *)(x + i)); + vst1q_f32(y + i + 0, vfmaq_f32(vld1q_f32(y + i + 0), vcvt_f32_f16(vget_low_f16 (ax)), vx)); + vst1q_f32(y + i + 4, vfmaq_f32(vld1q_f32(y + i + 4), vcvt_f32_f16(vget_high_f16(ax)), vx)); + } +#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin) + for (int vl; i < n; i += vl) { + vl = __riscv_vsetvl_e16m2(n - i); + const vfloat32m4_t ax = __riscv_vfwcvt_f_f_v_f32m4(__riscv_vle16_v_f16m2((const _Float16 *)x + i, vl), vl); + __riscv_vse32_v_f32m4(y + i, __riscv_vfmacc_vf_f32m4(__riscv_vle32_v_f32m4(y + i, vl), v, ax, vl), vl); + } +#endif + for (; i < n; ++i) { + y[i] += GGML_CPU_FP16_TO_FP32(x[i])*v; + } +} + inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, const ggml_fp16_t * GGML_RESTRICT x, const float v) { #if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE) const int sve_register_length = svcntb() * 8; @@ -514,7 +549,7 @@ inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, svst1_f16(pg, (__fp16 *)(y + np2), hy); } np = n; -#elif defined(__riscv_v_intrinsic) // implies __riscv_v_intrinsic +#elif defined(__riscv_v) #if defined (__riscv_zvfh) const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v); const _Float16 scale = *(const _Float16*)(&s); @@ -600,7 +635,7 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int y[i] += x[k][i]*v[k][0]; } } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl); @@ -661,7 +696,7 @@ inline static void ggml_vec_mad1_f32(const int n, float * y, const float * x, co for (int i = 0; i < n; ++i) { y[i] = x[i]*s + b; } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl); @@ -730,7 +765,7 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { ay1 = svmul_f32_m(pg, ay1, vx); svst1_f32(pg, y + i, ay1); } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl); @@ -794,7 +829,7 @@ inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float svst1_f16(pg, (__fp16 *)(y + np), out); } np = n; -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) #if defined(__riscv_zvfh) const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v); const _Float16 scale = *(const _Float16*)(&s); @@ -1054,11 +1089,7 @@ inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * } } #else -inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) { - for (int i = 0; i < n; ++i) { - y[i] = ggml_gelu_quick_f32(x[i]); - } -} +void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x); #endif // Sigmoid Linear Unit (SiLU) function @@ -1143,6 +1174,13 @@ inline static svfloat32_t ggml_v_silu(svbool_t pg, svfloat32_t x) { return svdiv_f32_x(pg, x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static svfloat32_t ggml_v_gelu_quick(svbool_t pg, svfloat32_t x) { + const svfloat32_t one = svdup_n_f32_x(pg, 1.0f); + const svfloat32_t exp_cx = ggml_v_expf(pg, svmul_n_f32_x(pg, x, GELU_QUICK_COEF)); + return svdiv_f32_x(pg, x, svadd_f32_x(pg, one, exp_cx)); +} + #elif defined(__ARM_NEON) && defined(__aarch64__) // adapted from arm limited optimized routine @@ -1182,6 +1220,13 @@ inline static float32x4_t ggml_v_silu(float32x4_t x) { return vdivq_f32(x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static float32x4_t ggml_v_gelu_quick(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t exp_cx = ggml_v_expf(vmulq_n_f32(x, GELU_QUICK_COEF)); + return vdivq_f32(x, vaddq_f32(one, exp_cx)); +} + #elif defined(__AVX512F__) && defined(__AVX512DQ__) // adapted from arm limited optimized routine @@ -1225,6 +1270,13 @@ inline static __m512 ggml_v_silu(__m512 x) { return _mm512_div_ps(x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static __m512 ggml_v_gelu_quick(__m512 x) { + const __m512 one = _mm512_set1_ps(1); + const __m512 exp_cx = ggml_v_expf(_mm512_mul_ps(x, _mm512_set1_ps(GELU_QUICK_COEF))); + return _mm512_div_ps(x, _mm512_add_ps(one, exp_cx)); +} + #elif defined(__AVX2__) && defined(__FMA__) // adapted from arm limited optimized routine @@ -1280,6 +1332,13 @@ inline static __m256 ggml_v_silu(__m256 x) { return _mm256_div_ps(x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static __m256 ggml_v_gelu_quick(__m256 x) { + const __m256 one = _mm256_set1_ps(1); + const __m256 exp_cx = ggml_v_expf(_mm256_mul_ps(x, _mm256_set1_ps(GELU_QUICK_COEF))); + return _mm256_div_ps(x, _mm256_add_ps(one, exp_cx)); +} + #elif defined(__SSE2__) // __AVX2__ / __ARM_NEON #if defined(__FMA__) @@ -1334,7 +1393,61 @@ inline static __m128 ggml_v_silu(__m128 x) { return _mm_div_ps(x, one_plus_exp_neg_x); } -#elif defined(__riscv_v_intrinsic) +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static __m128 ggml_v_gelu_quick(__m128 x) { + const __m128 one = _mm_set1_ps(1); + const __m128 exp_cx = ggml_v_expf(_mm_mul_ps(x, _mm_set1_ps(GELU_QUICK_COEF))); + return _mm_div_ps(x, _mm_add_ps(one, exp_cx)); +} + +#elif defined(__VXE__) || defined(__VXE2__) + +// adapted from arm limited optimized routine +// the maximum error is 1.45358 plus 0.5 ulps +// numbers above 88.38 will flush to infinity +// numbers beneath -103.97 will flush to zero +inline static float32x4_t ggml_v_expf(float32x4_t v_x) { + const float32x4_t v_r = vec_splats(0x1.8p23f); + const float32x4_t v_z = vec_madd(v_x, vec_splats(0x1.715476p+0f), v_r); + const float32x4_t v_n = vec_sub(v_z, v_r); + const float32x4_t v_b = vec_nmsub(v_n, vec_splats(0x1.7f7d1cp-20f), + vec_nmsub(v_n, vec_splats(0x1.62e4p-1f), v_x)); + const uint32x4_t v_e = vec_sl((uint32x4_t)v_z, 23); + const float32x4_t v_k = (float32x4_t)vec_add(v_e, (uint32x4_t)vec_splats(1.0f)); + const uint32x4_t v_c = (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(126.0f)); + const float32x4_t v_u = vec_mul(v_b, v_b); + const float32x4_t v_j = vec_madd( + vec_madd(vec_madd(vec_splats(0x1.0e4020p-7f), v_b, vec_splats(0x1.573e2ep-5f)), v_u, + vec_madd(vec_splats(0x1.555e66p-3f), v_b, vec_splats(0x1.fffdb6p-2f))), + v_u, vec_mul(vec_splats(0x1.ffffecp-1f), v_b)); + if (vec_all_eq(v_c, vec_splats(UINT32_C(0)))) { + return vec_madd(v_j, v_k, v_k); + } + const uint32x4_t v_d = vec_and((uint32x4_t)vec_cmple(v_n, vec_splats(0.0f)), vec_splats(UINT32_C(0x82000000))); + const float32x4_t v_s1 = (float32x4_t)vec_add(v_d, vec_splats(UINT32_C(0x7f000000))); + const float32x4_t v_s2 = (float32x4_t)vec_sub(v_e, v_d); + return vec_sel(vec_sel(vec_madd(v_k, v_j, v_k), vec_mul(vec_madd(v_s2, v_j, v_s2), v_s1), v_c), + vec_mul(v_s1, v_s1), (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(192.0f))); +} + +// computes silu x/(1+exp(-x)) in single precision vector +inline static float32x4_t ggml_v_silu(float32x4_t v_x) { + const float32x4_t v_one = vec_splats(1.0f); + const float32x4_t v_zero = vec_splats(0.0f); + const float32x4_t v_neg_x = vec_sub(v_zero, v_x); + const float32x4_t v_exp_neg_x = ggml_v_expf(v_neg_x); + const float32x4_t v_one_plus_exp_neg_x = vec_add(v_one, v_exp_neg_x); + return vec_div(v_x, v_one_plus_exp_neg_x); +} + +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static float32x4_t ggml_v_gelu_quick(float32x4_t x) { + const float32x4_t one = vec_splats(1.0f); + const float32x4_t exp_cx = ggml_v_expf(vec_mul(x, vec_splats(GELU_QUICK_COEF))); + return vec_div(x, vec_add(one, exp_cx)); +} + +#elif defined(__riscv_v) // adapted from arm limited optimized routine // the maximum error is 1.45358 plus 0.5 ulps @@ -1386,7 +1499,13 @@ inline static vfloat32m2_t ggml_v_silu_m2(vfloat32m2_t x, int vl) { return __riscv_vfdiv_vv_f32m2(x, one_plus_exp_neg_x, vl); } -#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __riscv_v_intrinsic +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static vfloat32m2_t ggml_v_gelu_quick_m2(vfloat32m2_t x, int vl) { + const vfloat32m2_t exp_cx = ggml_v_expf_m2(__riscv_vfmul_vf_f32m2(x, GELU_QUICK_COEF, vl), vl); + return __riscv_vfdiv_vv_f32m2(x, __riscv_vfadd_vf_f32m2(exp_cx, 1.0f, vl), vl); +} + +#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { for (int i = 0; i < n; ++i) { @@ -1531,11 +1650,7 @@ inline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float } } #else -inline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) { - for (int i = 0; i < n; ++i) { - y[i] = ggml_gelu_quick_f32(x[i]) * g[i]; - } -} +void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g); #endif inline static void ggml_vec_geglu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) { diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt index 297d9ffc9..dce8827ae 100644 --- a/ggml/src/ggml-cuda/CMakeLists.txt +++ b/ggml/src/ggml-cuda/CMakeLists.txt @@ -163,7 +163,7 @@ if (CUDAToolkit_FOUND) if (GGML_STATIC) if (WIN32) # As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library - target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas) + target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas CUDA::cublasLt) else () if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "10.1") target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static) @@ -172,7 +172,7 @@ if (CUDAToolkit_FOUND) endif() endif() else() - target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas) + target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas CUDA::cublasLt) endif() if (GGML_CUDA_NO_VMM) diff --git a/ggml/src/ggml-cuda/argsort.cu b/ggml/src/ggml-cuda/argsort.cu index f6a850dda..101afeba2 100644 --- a/ggml/src/ggml-cuda/argsort.cu +++ b/ggml/src/ggml-cuda/argsort.cu @@ -2,7 +2,8 @@ #ifdef GGML_CUDA_USE_CUB # include <cub/cub.cuh> -# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1) + // strided_iterator was added in CCCL 3.1 +# if (CCCL_MAJOR_VERSION > 3 || (CCCL_MAJOR_VERSION == 3 && CCCL_MINOR_VERSION >= 1)) # define STRIDED_ITERATOR_AVAILABLE # include <cuda/iterator> # endif @@ -27,21 +28,21 @@ static __global__ void init_offsets(int * offsets, const int ncols, const int nr } #endif // STRIDED_ITERATOR_AVAILABLE -#ifdef GGML_CUDA_USE_CUB - -// returns the suggested maximum number of rows to process during one argsort_f32_i32_cuda_cub() call -int argsort_f32_i32_cuda_cub_chunk_nrows(const size_t nb01, const int64_t nrows) { - // perform argsort in chunks up to approximately this size (currently 64MB) +// returns the suggested maximum number of rows to process at once, given the temporary buffer bytes per row +int ggml_cuda_chunk_nrows(const size_t row_bytes, const int64_t nrows) { + // process rows in chunks up to approximately this size (currently 64MB) // to avoid excessive temporary buffers memory usage const int chunk_bytes = 1 << 26; // calculate how many rows will fit in one chunk (must be at least one) - const int chunk_nrows = std::max((int) (chunk_bytes / nb01), 1); + const int chunk_nrows = std::max((int) (chunk_bytes / row_bytes), 1); // limit the resulting amount to total nrows return std::min((int64_t) chunk_nrows, nrows); } +#ifdef GGML_CUDA_USE_CUB + void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, const float * x, int * dst, @@ -289,7 +290,7 @@ void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { return; } - const int chunk_nrows = argsort_f32_i32_cuda_cub_chunk_nrows(src0->nb[1], nrows); + const int chunk_nrows = ggml_cuda_chunk_nrows(src0->nb[1], nrows); ggml_cuda_pool & pool = ctx.pool(); diff --git a/ggml/src/ggml-cuda/argsort.cuh b/ggml/src/ggml-cuda/argsort.cuh index c9adfcb98..86df5fda2 100644 --- a/ggml/src/ggml-cuda/argsort.cuh +++ b/ggml/src/ggml-cuda/argsort.cuh @@ -4,8 +4,9 @@ void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +int ggml_cuda_chunk_nrows(const size_t row_bytes, const int64_t nrows); + #ifdef GGML_CUDA_USE_CUB -int argsort_f32_i32_cuda_cub_chunk_nrows(const size_t nb01, const int64_t nrows); void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, const float * x, int * dst, diff --git a/ggml/src/ggml-cuda/batch-get.cu b/ggml/src/ggml-cuda/batch-get.cu new file mode 100644 index 000000000..60c4747b9 --- /dev/null +++ b/ggml/src/ggml-cuda/batch-get.cu @@ -0,0 +1,198 @@ +#include "batch-get.cuh" +#include "common.cuh" +#include "ggml-cpp.h" + +#include <climits> + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +struct ggml_cuda_batch_get_descriptor { + const unsigned char * src; + unsigned char * dst; + size_t size; +}; + +static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) { + // Blocks along y cooperate on each descriptor. + const size_t index = blockIdx.x; + if (index >= count) { + return; + } + const size_t first = (size_t) blockIdx.y * blockDim.x + threadIdx.x; + const size_t stride = (size_t) gridDim.y * blockDim.x; + const auto copy = copies[index]; + // Use 16-, 8-, 4-, or 2-byte copies when both pointers meet that alignment; threads copy separate chunks. + const uintptr_t alignment = (uintptr_t) copy.src | (uintptr_t) copy.dst; + size_t tail = 0; + if (alignment % 16 == 0) { + for (size_t i = first; i < copy.size / 16; i += stride) { + ggml_cuda_memcpy_1<16>(copy.dst + 16*i, copy.src + 16*i); + } + tail = copy.size / 16 * 16; + } else if (alignment % 8 == 0) { + for (size_t i = first; i < copy.size / 8; i += stride) { + ggml_cuda_memcpy_1<8>(copy.dst + 8*i, copy.src + 8*i); + } + tail = copy.size / 8 * 8; + } else if (alignment % 4 == 0) { + for (size_t i = first; i < copy.size / 4; i += stride) { + ggml_cuda_memcpy_1<4>(copy.dst + 4*i, copy.src + 4*i); + } + tail = copy.size / 4 * 4; + } else if (alignment % 2 == 0) { + for (size_t i = first; i < copy.size / 2; i += stride) { + ggml_cuda_memcpy_1<2>(copy.dst + 2*i, copy.src + 2*i); + } + tail = copy.size / 2 * 2; + } + // Copy trailing bytes, or the entire range when no wider alignment is available. + for (size_t i = tail + first; i < copy.size; i += stride) { + ggml_cuda_memcpy_1<1>(copy.dst + i, copy.src + i); + } +} + +struct ggml_cuda_batch_get_mapping { + void * host = nullptr; + unsigned char * device = nullptr; + + ggml_cuda_batch_get_mapping(void * host, size_t size) { + const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable); + if (err != cudaSuccess) { + (void) cudaGetLastError(); + GGML_LOG_DEBUG("%s: failed to register %.2f MiB of pinned memory: %s\n", __func__, + size / 1024.0 / 1024.0, cudaGetErrorString(err)); + return; + } + this->host = host; + if (cudaHostGetDevicePointer((void **) &device, host, 0) != cudaSuccess) { + (void) cudaGetLastError(); + device = nullptr; + return; + } + } + + ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete; + ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete; + + ~ggml_cuda_batch_get_mapping() { + if (host) { + // Release only the registration acquired by this object. + CUDA_CHECK(cudaHostUnregister(host)); + } + } +}; +#endif + +bool ggml_cuda_get_tensor_n(int device, + const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors) { +#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) + GGML_UNUSED(device); + GGML_UNUSED(tensors); + GGML_UNUSED(data); + GGML_UNUSED(offsets); + GGML_UNUSED(sizes); + GGML_UNUSED(n_tensors); + return false; +#else + // Provisional dispatch values based on experiments. + constexpr size_t min_batch_reads = 128; + constexpr size_t max_gather_read_bytes = 128 * 1024; + // Collect non-empty CPU destination ranges. + std::vector<std::pair<uintptr_t, uintptr_t>> ranges; + ranges.reserve(n_tensors); + size_t payload_bytes = 0; + // Number of non-empty reads whose size is at most max_gather_read_bytes. + size_t n_reads_small = 0; + for (size_t i = 0; i < n_tensors; ++i) { + if (sizes[i] == 0) { + continue; + } + const uintptr_t address = (uintptr_t) data[i]; + GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]); + GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]); + payload_bytes += sizes[i]; + n_reads_small += sizes[i] <= max_gather_read_bytes; + ranges.emplace_back(address, address + sizes[i]); + } + if (ranges.empty()) { + return true; + } + // Total number of non-empty reads. + const size_t n_reads = ranges.size(); + if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) { + return false; + } + if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) { + return false; + } + std::sort(ranges.begin(), ranges.end()); + for (size_t i = 1; i < ranges.size(); ++i) { + // Preserve the ordered copy path when CPU destinations overlap. + if (ranges[i].first < ranges[i - 1].second) { + return false; + } + } + // Find the CPU span to register, including gaps between destinations. + const uintptr_t begin = ranges.front().first; + const uintptr_t end = ranges.back().second; + if (end - begin - payload_bytes > payload_bytes) { + return false; + } + // Register and map the existing CPU destination so the kernel can write to it directly. + ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin); + if (!mapping.host) { + return false; + } + // Reads selected for gather; zero means all reads use DMA. + size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0; + // Keep descriptor storage alive until the final stream synchronization. + constexpr size_t max_batch_descriptors = 128 * 1024; + const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor); + ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr); + if (!descriptors) { + n_reads_gather = 0; + } + const auto host_type = ggml_backend_cuda_host_buffer_type(); + ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr); + if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) { + n_reads_gather = 0; + } + if (n_reads_gather) { + auto * copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get())); + auto * device_copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(descriptors.get())); + const int threads = 256; + const size_t nsm = ggml_cuda_info().devices[device].nsm; + size_t i = 0; + for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) { + const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors); + size_t count = 0; + for (; i < n_tensors && count < n_batch; ++i) { + if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) { + continue; + } + copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i], + mapping.device + ((uintptr_t) data[i] - begin), sizes[i]}; + } + const size_t blocks_per_copy = std::min<size_t>(UINT16_MAX, std::max<size_t>(1, (nsm + n_batch - 1) / n_batch)); + const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy); + // Stream ordering keeps each kernel ahead of the next descriptor overwrite. + CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread)); + ggml_cuda_batch_get_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch); + CUDA_CHECK(cudaGetLastError()); + if (first + n_batch < n_reads_gather) { + // Wait before refilling the pinned host descriptor buffer. + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + } + } + } + // Keep the destination pinned for DMA too; overlapping destinations already declined above. + for (size_t i = 0; i < n_tensors; ++i) { + if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) { + CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread)); + } + } + // Wait for GPU writes before releasing the CPU mapping and descriptor buffers. + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + return true; +#endif +} diff --git a/ggml/src/ggml-cuda/batch-get.cuh b/ggml/src/ggml-cuda/batch-get.cuh new file mode 100644 index 000000000..eb9bf0cc8 --- /dev/null +++ b/ggml/src/ggml-cuda/batch-get.cuh @@ -0,0 +1,7 @@ +#pragma once + +#include "ggml-backend.h" + +bool ggml_cuda_get_tensor_n(int device, + const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors); diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu index 89ae2bc78..f7c3a7322 100644 --- a/ggml/src/ggml-cuda/binbcast.cu +++ b/ggml/src/ggml-cuda/binbcast.cu @@ -580,8 +580,7 @@ void ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst float * dst_d = (float *) dst->data; repeat_back_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s00, s01, s02, s03, ne0, ne1, ne2, ne3, stream); } break; - default: { + default: GGML_ASSERT(false); - } break; } } diff --git a/ggml/src/ggml-cuda/clamp.cu b/ggml/src/ggml-cuda/clamp.cu index fe415e7f7..2727b7ec9 100644 --- a/ggml/src/ggml-cuda/clamp.cu +++ b/ggml/src/ggml-cuda/clamp.cu @@ -4,21 +4,42 @@ static __device__ __forceinline__ float op_clamp(float x, float min, float max) return fminf(fmaxf(x, min), max); } +// src and dst may be views: rows are contiguous, dims 1..3 follow the strides (in elements). template <class T> -static __global__ void op_clamp_kernel(const T * x, T * dst, const T min, const T max, const int k) { - const int i = blockDim.x*blockIdx.x + threadIdx.x; +static __global__ void op_clamp_kernel(const T * x, T * dst, const T min, const T max, const uint32_t k, + const uint3 ne0, const uint3 ne1, const uint3 ne2, + const uint32_t s01, const uint32_t s02, const uint32_t s03, + const uint32_t s1, const uint32_t s2, const uint32_t s3) { + const uint32_t i = blockDim.x*blockIdx.x + threadIdx.x; if (i >= k) { return; } - dst[i] = (T)op_clamp((float)x[i], (float)min, (float)max); + const uint2 d0 = fast_div_modulo(i, ne0); // <i / ne0, i0> + const uint2 d1 = fast_div_modulo(d0.x, ne1); // <i / (ne0*ne1), i1> + const uint2 d2 = fast_div_modulo(d1.x, ne2); // <i3, i2> + + const size_t i_src = d0.y + size_t(d1.y)*s01 + size_t(d2.y)*s02 + size_t(d2.x)*s03; + const size_t i_dst = d0.y + size_t(d1.y)*s1 + size_t(d2.y)*s2 + size_t(d2.x)*s3; + + dst[i_dst] = (T)op_clamp((float)x[i_src], (float)min, (float)max); } template <class T> -static void clamp_cuda(const T * x, T * dst, const T min, const T max, const int k, cudaStream_t stream) { - const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE; - op_clamp_kernel<<<num_blocks, CUDA_CLAMP_BLOCK_SIZE, 0, stream>>>(x, dst, min, max, k); +static void clamp_cuda(const T * x, T * dst, const T min, const T max, const ggml_tensor * src0, const ggml_tensor * t, cudaStream_t stream) { + const int64_t k = ggml_nelements(src0); + const size_t ts = sizeof(T); + GGML_ASSERT(k <= std::numeric_limits<uint32_t>::max()); + + const uint3 ne0 = init_fastdiv_values(src0->ne[0]); + const uint3 ne1 = init_fastdiv_values(src0->ne[1]); + const uint3 ne2 = init_fastdiv_values(src0->ne[2]); + + const int64_t num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE; + op_clamp_kernel<<<num_blocks, CUDA_CLAMP_BLOCK_SIZE, 0, stream>>>(x, dst, min, max, (uint32_t) k, ne0, ne1, ne2, + src0->nb[1]/ts, src0->nb[2]/ts, src0->nb[3]/ts, + t->nb[1]/ts, t->nb[2]/ts, t->nb[3]/ts); } @@ -31,6 +52,7 @@ void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst)); float min; float max; @@ -38,8 +60,8 @@ void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { memcpy(&max, (float *) dst->op_params + 1, sizeof(float)); if (src0->type == GGML_TYPE_F16) { - clamp_cuda((const half *)src0_d, (half *)dst_d, (half)min, (half)max, ggml_nelements(src0), stream); + clamp_cuda((const half *)src0_d, (half *)dst_d, (half)min, (half)max, src0, dst, stream); } else { - clamp_cuda((const float *)src0_d, (float *)dst_d, (float)min, (float)max, ggml_nelements(src0), stream); + clamp_cuda((const float *)src0_d, (float *)dst_d, (float)min, (float)max, src0, dst, stream); } } diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 928607c41..414177308 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -7,7 +7,6 @@ #include <cstdint> #include <cstdlib> #include <memory> -#include <mutex> #if defined(GGML_USE_HIP) #define GGML_COMMON_DECL_HIP @@ -52,6 +51,7 @@ #define GGML_CUDA_CC_VOLTA 700 #define GGML_CUDA_CC_TURING 750 #define GGML_CUDA_CC_AMPERE 800 +#define GGML_CUDA_CC_AMPERE_GA10X 860 #define GGML_CUDA_CC_ORIN 870 #define GGML_CUDA_CC_ADA_LOVELACE 890 #define GGML_CUDA_CC_HOPPER 900 @@ -81,6 +81,7 @@ #define GGML_CUDA_CC_RDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a #define GGML_CUDA_CC_RDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA #define GGML_CUDA_CC_RDNA3_5 (GGML_CUDA_CC_OFFSET_AMD + 0x1150) // AI 370, AI Max 395 laptops. +#define GGML_CUDA_CC_RDNA4_M (GGML_CUDA_CC_OFFSET_AMD + 0x1170) // gfx117X #define GGML_CUDA_CC_RDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000 #define GGML_CUDA_CC_IS_AMD(cc) (cc >= GGML_CUDA_CC_OFFSET_AMD) @@ -88,8 +89,9 @@ #define GGML_CUDA_CC_IS_RDNA1(cc) (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2) #define GGML_CUDA_CC_IS_RDNA2(cc) (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3) #define GGML_CUDA_CC_IS_RDNA3_0(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA3_5) -#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4) +#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4_M) #define GGML_CUDA_CC_IS_RDNA3(cc) (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) +#define GGML_CUDA_CC_IS_RDNA4_M(cc) (cc >= GGML_CUDA_CC_RDNA4_M && cc < GGML_CUDA_CC_RDNA4) #define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4) #define GGML_CUDA_CC_IS_GCN_APU(cc) ((cc) == GGML_CUDA_CC_GFX909 || (cc) == GGML_CUDA_CC_GFX90C) #define GGML_CUDA_CC_IS_GCN(cc) ((cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) || GGML_CUDA_CC_IS_GCN_APU(cc)) @@ -130,6 +132,14 @@ static __device__ __forceinline__ void ggml_cuda_syncwarp() { #endif // GGML_USE_HIP } +static __device__ __forceinline__ void ggml_cuda_prefetch_l2(const void * p) { +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); +#else + GGML_UNUSED(p); +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + static __device__ __forceinline__ void ggml_cuda_pdl_sync() { #if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER cudaGridDependencySynchronize(); @@ -276,9 +286,17 @@ static const char * cu_get_error_str(CUresult err) { #define AMD_MFMA_AVAILABLE #endif // defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA) -#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA3)) +#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA4_M)) +#define AMD_WMMA_128B_AVAILABLE +#endif // defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA4_M)) + +#if defined(GGML_USE_HIP) && defined(RDNA3) +#define AMD_WMMA_256B_AVAILABLE +#endif // defined(GGML_USE_HIP) && defined(RDNA3) + +#if defined(AMD_WMMA_128B_AVAILABLE) || defined(AMD_WMMA_256B_AVAILABLE) #define AMD_WMMA_AVAILABLE -#endif // defined(GGML_USE_HIP) && defined(RDNA4) +#endif // defined(AMD_WMMA_128B_AVAILABLE) || defined(AMD_WMMA_256B_AVAILABLE) // The Volta instructions are in principle available on Turing or newer but they are effectively unusable: #if !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA @@ -325,7 +343,7 @@ static bool fast_fp16_hardware_available(const int cc) { // To be used for feature selection of external libraries, e.g. cuBLAS. static bool fp16_mma_hardware_available(const int cc) { return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_VOLTA) || - GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc) || + GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc) || (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2); } @@ -354,7 +372,7 @@ static bool amd_mfma_available(const int cc) { } static bool amd_wmma_available(const int cc) { - return (GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_RDNA3(cc)); + return GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc); } static bool volta_mma_available(const int cc) { @@ -378,6 +396,10 @@ static bool blackwell_mma_available(const int cc) { ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN; } +static bool fp8_mma_hardware_available(const int cc) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE && cc < GGML_CUDA_CC_RUBIN; +} + // Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes. static bool ggml_cuda_is_aligned(const ggml_tensor * tensor, const size_t alignment) { GGML_ASSERT(tensor != nullptr); @@ -712,7 +734,7 @@ static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, i #if defined(GGML_USE_HIP) #if defined(CDNA) || defined(RDNA2) || defined(__gfx906__) c = __builtin_amdgcn_sdot4(a, b, c, false); -#elif defined(RDNA3) || defined(RDNA4) +#elif defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4) c = __builtin_amdgcn_sudot4( true, a, true, b, c, false); #elif defined(RDNA1) || defined(__gfx900__) int tmp1; @@ -757,9 +779,9 @@ static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float2 v acc += v.y*u.y; } -#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA)) +#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA)) #define V_DOT2_F32_F16_AVAILABLE -#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA)) +#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA)) static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const half2 v, const half2 u) { #ifdef V_DOT2_F32_F16_AVAILABLE @@ -873,6 +895,67 @@ static __device__ __forceinline__ float ggml_cuda_ue4m3_to_fp32(uint8_t x) { #endif // defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000 } +static __host__ __device__ __forceinline__ float ggml_cuda_f8_e4m3_to_fp32(uint8_t x) { +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + __nv_fp8_e4m3 xf; + xf.__x = x; + return static_cast<float>(xf); +#else + const uint8_t magnitude = x & 0x7F; + if (magnitude == 0x7F) { + return NAN; + } + + const int exp = (magnitude >> 3) & 0x0F; + const int man = magnitude & 0x07; + float value; + if (exp == 0) { + value = ldexpf((float) man, -9); + } else { + value = ldexpf(1.0f + (float) man / 8.0f, exp - 7); + } + return x & 0x80 ? -value : value; +#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + +static __host__ __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_f8_e4m3(float x) { +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + return __nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3); +#elif !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__) && !defined(__MUSA_ARCH__) + // ggml_cuda_cast also runs on the host, where __float2int_rn is not available. + return ggml_fp32_to_f8_e4m3(x); +#else + const uint8_t sign = signbit(x) ? 0x80 : 0; + x = fabsf(x); + + if (isnan(x)) { + return sign | 0x7F; + } + if (x == 0.0f) { + return sign; + } + if (isinf(x) || x >= 448.0f) { + return sign | 0x7E; + } + if (x < 0.015625f) { + return sign | (uint8_t) __float2int_rn(x * 512.0f); + } + + int exp; + const float mantissa = frexpf(x, &exp) * 2.0f; + int encoded_exp = exp + 6; + int encoded_man = __float2int_rn((mantissa - 1.0f) * 8.0f); + if (encoded_man == 8) { + encoded_man = 0; + encoded_exp++; + } + if (encoded_exp > 15 || (encoded_exp == 15 && encoded_man > 6)) { + return sign | 0x7E; + } + return sign | (uint8_t) (encoded_exp << 3) | (uint8_t) encoded_man; +#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + static __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_ue4m3(float x) { #if defined(BLACKWELL_MMA_AVAILABLE) // This is used for NVFP4 subblock scale quantizations only if (!(x > 0.0f)) { @@ -881,6 +964,7 @@ static __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_ue4m3(float x) { const __nv_fp8_e4m3 xf(x); return xf.__x; #else + GGML_UNUSED(x); NO_DEVICE_CODE; // Used only for NVFP4 Scales for Activations, only for Blackwell #endif // defined(BLACKWELL_MMA_AVAILABLE) } @@ -1195,6 +1279,8 @@ const ggml_cuda_device_info & ggml_cuda_info(); void ggml_cuda_set_device(int device); int ggml_cuda_get_device(); +bool ggml_cuda_pool_debug_enabled(); + struct ggml_cuda_pool { virtual ~ggml_cuda_pool() = default; @@ -1259,199 +1345,22 @@ struct ggml_tensor_extra_gpu { #define USE_CUDA_GRAPH #endif -struct ggml_cuda_graph { -#ifdef USE_CUDA_GRAPH - ~ggml_cuda_graph() { - if (instance != nullptr) { - CUDA_CHECK(cudaGraphExecDestroy(instance)); - } - if (graph != nullptr) { - CUDA_CHECK(cudaGraphDestroy(graph)); - } - } - cudaGraph_t graph = nullptr; - cudaGraphExec_t instance = nullptr; - size_t num_nodes = 0; - std::vector<cudaGraphNode_t> nodes; - bool disable_due_to_gpu_arch = false; - bool warmup_complete = false; - uint64_t uid = 0; - int64_t last_used_time = 0; - struct node_properties { - ggml_tensor node; - void * node_src_data_ptrs[GGML_MAX_SRC]; - int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; - size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; - }; - std::vector<node_properties> node_props; - - bool is_enabled() const { - static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); - return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env); - } -#endif -}; - -struct ggml_cuda_concurrent_event { - std::vector<cudaEvent_t> join_events; - cudaEvent_t fork_event = nullptr; - - int n_streams = 0; - std::unordered_map<const ggml_tensor *, int> stream_mapping; - - // Original order of nodes in this concurrent region (before interleaving) - // Used to restore grouping for fusion within streams - std::vector<const ggml_tensor *> original_order; - - const ggml_tensor * join_node; - - ggml_cuda_concurrent_event() = default; - - ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete; - ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete; - - explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) { - join_events.resize(n_streams); - - for (size_t i = 0; i < join_events.size(); ++i) { - CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming)); - } - - CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming)); - } - - ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept - : join_events(std::move(other.join_events)) - , fork_event(other.fork_event) - , n_streams(other.n_streams) - , stream_mapping(std::move(other.stream_mapping)) - , original_order(std::move(other.original_order)) - , join_node(other.join_node) { - other.fork_event = nullptr; - } - - // 1. check if any branches write to overlapping memory ranges (except the join node) - // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event - // we assume all nodes have the same buffer - bool is_valid() const { - std::vector<std::vector<std::pair<int64_t, int64_t>>> write_ranges; - write_ranges.resize(n_streams); - - // get join_node's memory range to exclude from overlap checking. - // multiple nodes can use join_node's buffer; we synchronize on the join node. - const ggml_tensor * join_t = join_node->view_src ? join_node->view_src : join_node; - const int64_t join_start = (int64_t) join_t->data; - const int64_t join_end = join_start + ggml_nbytes(join_t); - - for (const auto & [tensor, stream] : stream_mapping) { - const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; - const int64_t t_start = (int64_t) t->data; - const int64_t t_end = t_start + ggml_nbytes(t); - - // skip tensors that overlap with join_node's buffer. - if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { - continue; - } - - // concurrent streams begin from 1 - write_ranges[stream - 1].emplace_back(t_start, t_end); - } - - for (int i = 0; i < n_streams; ++i) { - // sorts first by start then by end of write range - std::sort(write_ranges[i].begin(), write_ranges[i].end()); - } - - bool writes_overlap = false; - bool dependent_srcs = false; - for (const auto & [tensor, stream] : stream_mapping) { - const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; - const int64_t t_start = (int64_t) t->data; - const int64_t t_end = t_start + ggml_nbytes(t); - - // skip tensors that overlap with join_node's buffer - if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { - continue; - } - - // check if this buffer's write data overlaps with another stream's - std::pair<int64_t, int64_t> data_range = std::make_pair(t_start, t_end); - for (int i = 0; i < n_streams; ++i) { - if (i == stream - 1) { - continue; - } - auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range); - - if (it != write_ranges[i].end()) { - const std::pair<int64_t, int64_t> & other = *it; - - // std::lower_bound returns the first element where other >= data_range (lexicographically). - // This guarantees other.first >= data_range.first. - // Therefore, overlap occurs iff other.first < data_range.second - // (i.e., the other range starts before this range ends). - if (other.first < data_range.second) { - GGML_LOG_DEBUG("Writes overlap for %s", tensor->name); - writes_overlap = true; - break; - } - } - } - - //check if all srcs are either in branch or don't have a branch - for (int i = 0; i < GGML_MAX_SRC; ++i) { - if (!tensor->src[i]) { - continue; - } - - auto it = stream_mapping.find(tensor->src[i]); - - if (it == stream_mapping.end()) { - continue; - } - - if (it->second != stream) { - dependent_srcs = true; - break; - } - } - - if (dependent_srcs || writes_overlap) { - break; - } - } - - return !writes_overlap && !dependent_srcs; - } - - ~ggml_cuda_concurrent_event() { - if (fork_event != nullptr) { - CUDA_CHECK(cudaEventDestroy(fork_event)); - } - for (cudaEvent_t e : join_events) { - if (e != nullptr) { - CUDA_CHECK(cudaEventDestroy(e)); - } - } - } -}; - -struct ggml_cuda_stream_context { - std::unordered_map<const ggml_tensor *, ggml_cuda_concurrent_event> concurrent_events; - - void reset() { - concurrent_events.clear(); - } -}; +// defined in graph.cuh +struct ggml_cuda_graph; +struct ggml_cuda_stream_context; struct ggml_backend_cuda_context { int device; std::string name; cudaEvent_t copy_event = nullptr; - cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } }; - cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; - void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; - size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {0}; + cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {}; + cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {}; + void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {}; + size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {}; +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 + cublasLtHandle_t cublaslt_handles[GGML_CUDA_MAX_DEVICES] = {}; +#endif int curr_stream_no = 0; @@ -1460,59 +1369,23 @@ struct ggml_backend_cuda_context { // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe) std::unordered_map<const void *, std::unique_ptr<ggml_cuda_graph>> cuda_graphs; - int64_t last_graph_eviction_sweep = 0; + int64_t next_graph_eviction_time = 0; + int64_t last_graph_use_time = 0; - ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { - const int64_t time_now = ggml_time_us(); - - // sweep every 5s, evicting cuda graphs unused for >=10s - if (time_now - last_graph_eviction_sweep >= 5'000'000) { - last_graph_eviction_sweep = time_now; - for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) { - if (time_now - it->second->last_used_time >= 10'000'000) { - it = cuda_graphs.erase(it); - } else { - ++it; - } - } - } - - auto it = cuda_graphs.find(first_node_ptr); - if (it == cuda_graphs.end()) { - it = cuda_graphs.emplace(first_node_ptr, std::make_unique<ggml_cuda_graph>()).first; - } - it->second->last_used_time = time_now; - return it->second.get(); - } + // defined in ggml-cuda.cu, which includes graph.cuh + ggml_cuda_graph * cuda_graph(const void * first_node_ptr, int64_t time_now = ggml_time_us()); // Check if any CUDA graph is enabled for this context (used by kernels that need to know // if graphs are in use without having access to the specific graph key) - bool any_cuda_graph_enabled() const { - for (const auto & [key, graph] : cuda_graphs) { - if (graph && graph->is_enabled()) { - return true; - } - } - return false; - } + bool any_cuda_graph_enabled() const; // Check if any CUDA graph has an instance for this context - bool any_cuda_graph_has_instance() const { - for (const auto & [key, graph] : cuda_graphs) { - if (graph && graph->instance != nullptr) { - return true; - } - } - return false; - } + bool any_cuda_graph_has_instance() const; #endif // USE_CUDA_GRAPH - explicit ggml_backend_cuda_context(int device) : - device(device), - name(GGML_CUDA_NAME + std::to_string(device)) { - } + explicit ggml_backend_cuda_context(int device); - ggml_cuda_stream_context concurrent_stream_context; + std::unique_ptr<ggml_cuda_stream_context> concurrent_stream_context; ~ggml_backend_cuda_context(); @@ -1526,7 +1399,7 @@ struct ggml_backend_cuda_context { cudaStream_t stream() { return stream(device, curr_stream_no); } - ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; } + ggml_cuda_stream_context & stream_context() { return *concurrent_stream_context; } cublasHandle_t cublas_handle() { if (cublas_handles[device][curr_stream_no] == nullptr) { @@ -1534,10 +1407,14 @@ struct ggml_backend_cuda_context { CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no])); CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH)); CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream())); -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2)) +#if (defined(GGML_USE_HIP) && HIP_VERSION >= 70000000) || (!defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2))) if (cublas_workspace_sizes[device] == 0) { +#if defined(GGML_USE_HIP) + cublas_workspace_sizes[device] = 32 * 1024 * 1024; +#else const int cc = ggml_cuda_info().devices[device].cc; cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024; +#endif } CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); @@ -1546,14 +1423,24 @@ struct ggml_backend_cuda_context { return cublas_handles[device][curr_stream_no]; } +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 + cublasLtHandle_t cublaslt_handle() { + if (cublaslt_handles[device] == nullptr) { + ggml_cuda_set_device(device); + CUBLAS_CHECK(cublasLtCreate(&cublaslt_handles[device])); + } + return cublaslt_handles[device]; + } +#endif + // pool std::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; - static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(int device, int stream_no); + static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(ggml_backend_cuda_context & ctx, int device, int stream_no); ggml_cuda_pool & pool(int device) { if (pools[device][curr_stream_no] == nullptr) { - pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no); + pools[device][curr_stream_no] = new_pool_for_device(*this, device, curr_stream_no); } return *pools[device][curr_stream_no]; } @@ -1571,6 +1458,7 @@ struct ggml_cuda_mm_fusion_args_host { const ggml_tensor * gate_scale = nullptr; ggml_glu_op glu_op; float glu_limit = 0.0f; + ggml_prec prec_src1 = GGML_PREC_UNDEFINED; // src1 precision requested by the fused MUL_MATs (MMQ) const ggml_tensor * shared_up = nullptr; const ggml_tensor * shared_gate = nullptr; ggml_tensor * shared_dst = nullptr; @@ -1629,60 +1517,8 @@ struct ggml_cuda_pdl_config { }; -static bool ggml_cuda_kernel_can_use_pdl(const void * kernel) { - const int device = ggml_cuda_get_device(); - - struct cache_key { - int device; - const void * kernel; - - bool operator==(const cache_key & other) const { return device == other.device && kernel == other.kernel; } - }; - - struct cache_key_hash { - // MurmurHash3 mixing function for better hash distribution (vs. just std::hash which in some implementations simply returns the identity) - static size_t hash_mix(size_t x) { - std::uint64_t y = x; - const std::uint64_t m = 0xe9846af9b1a615d; - - y ^= y >> 32; - y *= m; - y ^= y >> 32; - y *= m; - y ^= y >> 28; - - return static_cast<size_t>(y); - } - - size_t operator()(const cache_key & key) const { - // Use a nonzero seed to avoid mapping all-zero keys to zero - size_t h = 42; - h = hash_mix(h + key.device); - h = hash_mix(h + reinterpret_cast<size_t>(key.kernel)); - return h; - } - }; - - static std::mutex cache_mutex; - static std::unordered_map<cache_key, bool, cache_key_hash> cache; - - const cache_key key = { device, kernel }; - std::lock_guard<std::mutex> lock(cache_mutex); - const auto it = cache.find(key); - if (it != cache.end()) { - return it->second; - } - - cudaFuncAttributes attr = {}; - CUDA_CHECK(cudaFuncGetAttributes(&attr, kernel)); - - // PDL device-side primitives are emitted only for PTX versions >= 90. - // We have to guard on a loaded kernel's PTX version so a kernel forward-JIT'ed - // from pre-Hopper PTX to a Hopper-or-newer GPU does not opt into PDL. - const bool can_use_pdl = attr.ptxVersion >= 90; - cache.emplace(key, can_use_pdl); - return can_use_pdl; -} +// whether kernel can be launched with PDL on the current device (also honors GGML_CUDA_PDL=0), defined in ggml-cuda.cu +bool ggml_cuda_kernel_can_use_pdl(const void * kernel); #endif //defined(GGML_CUDA_USE_PDL) @@ -1696,13 +1532,7 @@ static bool ggml_cuda_kernel_can_use_pdl(const void * kernel) { template<typename Kernel, typename... Args> static __inline__ void ggml_cuda_kernel_launch(Kernel kernel, const ggml_cuda_kernel_launch_params & launch_params, Args&&... args) { #if defined(GGML_CUDA_USE_PDL) - - static const bool env_pdl_enabled = []() { - const char * env = getenv("GGML_CUDA_PDL"); - return env == nullptr || std::atoi(env) != 0; - }(); - - if (env_pdl_enabled && ggml_cuda_kernel_can_use_pdl(reinterpret_cast<const void *>(kernel))) { + if (ggml_cuda_kernel_can_use_pdl(reinterpret_cast<const void *>(kernel))) { auto pdl_cfg = ggml_cuda_pdl_config(launch_params); CUDA_CHECK(cudaLaunchKernelEx(&pdl_cfg.cfg, kernel, std::forward<Args>(args)... )); diff --git a/ggml/src/ggml-cuda/concat.cu b/ggml/src/ggml-cuda/concat.cu index 6df89013c..f8dcd95ab 100644 --- a/ggml/src/ggml-cuda/concat.cu +++ b/ggml/src/ggml-cuda/concat.cu @@ -189,7 +189,6 @@ static void concat_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml break; default: GGML_ABORT("Invalid dim: %d", dim); - break; } } } @@ -236,7 +235,6 @@ void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { break; default: GGML_ABORT("Unsupported type size: %zu", ggml_type_size(src0->type)); - break; } } } diff --git a/ggml/src/ggml-cuda/conv2d.cu b/ggml/src/ggml-cuda/conv2d.cu index 10109ad36..c5e1836ff 100644 --- a/ggml/src/ggml-cuda/conv2d.cu +++ b/ggml/src/ggml-cuda/conv2d.cu @@ -180,7 +180,6 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input, const int end = int(int64_t(tiles) * (split + 1) / split_k) * BK; for (int k0 = begin; k0 < end; k0 += BK) { if (k_total % 8 == 0 && uintptr_t(weight) % 16 == 0) { -#pragma unroll for (int i = tid; i < BM * BK / 8; i += nthreads) { const int row = i / (BK / 8), col = 8 * (i % (BK / 8)); const int4 v = m0 + row < oc && k0 + col < k_total ? @@ -189,7 +188,6 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input, *(int4 *) &a_s[row][col / 2] = v; } } else { -#pragma unroll for (int i = tid; i < BM * BK / 2; i += nthreads) { const int row = i / (BK / 2), col = 2 * (i % (BK / 2)); half lo = __float2half(0.0f), hi = lo; @@ -202,7 +200,6 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input, a_s[row][col / 2] = __halves2half2(lo, hi); } } -#pragma unroll for (int k = load_row; k < BK; k += nthreads / (BN / 2)) { const int ki = k0 + k; const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw; @@ -378,8 +375,8 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const auto & device = ggml_cuda_info().devices[ctx.device]; const bool use_mma = turing_mma_available(device.cc) || amd_wmma_available(device.cc) || amd_mfma_available(device.cc); - // MUSA can share the tiling without a native fragment implementation in mma.cuh. - const bool use_simt = GGML_CUDA_CC_IS_MTHREADS(device.cc); + // Devices without a native fragment implementation in mma.cuh can share the tiling. + const bool use_simt = GGML_CUDA_CC_IS_MTHREADS(device.cc) || (GGML_CUDA_CC_IS_AMD(device.cc) && !use_mma && IC * KW * KH >= 64); const bool pointwise = KW == 1 && KH == 1 && ST_X == 1 && ST_Y == 1 && PD_X == 0 && PD_Y == 0; const bool use_blas = pointwise && fast_fp16_hardware_available(device.cc); // Short reductions on small maps do not amortize conversion and launch costs. diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 77fd28894..a8f5047a2 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -541,6 +541,8 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) { return dequantize_row_mxfp4_cuda; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cont_cuda<ggml_fp8_e4m3_t>; case GGML_TYPE_F32: return convert_unary_cont_cuda<float>; case GGML_TYPE_F16: @@ -601,6 +603,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_mxfp4_cuda; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cont_cuda<ggml_fp8_e4m3_t>; case GGML_TYPE_F32: return convert_unary_cont_cuda<float>; case GGML_TYPE_BF16: @@ -658,6 +662,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_mxfp4_cuda; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cont_cuda<ggml_fp8_e4m3_t>; case GGML_TYPE_F16: return convert_unary_cont_cuda<half>; case GGML_TYPE_BF16: @@ -685,6 +691,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) { return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>; case GGML_TYPE_Q8_0: return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>; + case GGML_TYPE_F8_E4M3: + return convert_unary_cuda<ggml_fp8_e4m3_t>; case GGML_TYPE_BF16: return convert_unary_cuda<nv_bfloat16>; default: @@ -710,6 +718,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) { return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>; case GGML_TYPE_Q8_0: return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>; + case GGML_TYPE_F8_E4M3: + return convert_unary_cuda<ggml_fp8_e4m3_t>; case GGML_TYPE_F16: return convert_unary_cuda<half, nv_bfloat16>; default: @@ -735,6 +745,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) { return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>; case GGML_TYPE_Q8_0: return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>; + case GGML_TYPE_F8_E4M3: + return convert_unary_cuda<ggml_fp8_e4m3_t>; case GGML_TYPE_BF16: return convert_unary_cuda<nv_bfloat16, float>; default: diff --git a/ggml/src/ggml-cuda/convert.cuh b/ggml/src/ggml-cuda/convert.cuh index 91255ccf3..fb2a1b3bc 100644 --- a/ggml/src/ggml-cuda/convert.cuh +++ b/ggml/src/ggml-cuda/convert.cuh @@ -35,6 +35,36 @@ template<typename dst_t, typename src_t> __host__ __device__ inline dst_t ggml_cuda_cast(src_t x) { if constexpr (std::is_same_v<dst_t, src_t>) { return x; + } else if constexpr (std::is_same_v<dst_t, ggml_fp8_e4m3_t>) { + return { ggml_cuda_fp32_to_f8_e4m3(ggml_cuda_cast<float>(x)) }; + } else if constexpr (std::is_same_v<src_t, ggml_fp8_e4m3_t>) { +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if CUDART_VERSION >= 13040 + if constexpr (std::is_same_v<dst_t, nv_bfloat16>) { + return static_cast<nv_bfloat16>(__nv_cvt_fp8_to_bf16raw(x.bits, __NV_E4M3)); + } +#endif + if constexpr (std::is_same_v<dst_t, half>) { + return static_cast<half>(__nv_cvt_fp8_to_halfraw(x.bits, __NV_E4M3)); + } +#endif + return ggml_cuda_cast<dst_t>(ggml_cuda_f8_e4m3_to_fp32(x.bits)); +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + } else if constexpr (std::is_same_v<src_t, __nv_fp8x2_e4m3> && std::is_same_v<dst_t, nv_bfloat162>) { +#if CUDART_VERSION >= 13040 + return static_cast<nv_bfloat162>(__nv_cvt_fp8x2_to_bf162raw(x.__x, __NV_E4M3)); +#else + return make_bfloat162( + ggml_cuda_cast<nv_bfloat16>(ggml_cuda_f8_e4m3_to_fp32(x.__x & 0xFF)), + ggml_cuda_cast<nv_bfloat16>(ggml_cuda_f8_e4m3_to_fp32(x.__x >> 8))); +#endif + } else if constexpr (std::is_same_v<src_t, __nv_fp8x2_e4m3> && std::is_same_v<dst_t, float2>) { +#if CUDART_VERSION >= 13040 + return ggml_cuda_cast<float2>(ggml_cuda_cast<nv_bfloat162>(x)); +#else + return __half22float2(static_cast<half2>(__nv_cvt_fp8x2_to_halfraw2(x.__x, __NV_E4M3))); +#endif +#endif } else if constexpr(std::is_same_v<dst_t, nv_bfloat16>) { return __float2bfloat16(float(x)); } else if constexpr(std::is_same_v<src_t, nv_bfloat16>) { diff --git a/ggml/src/ggml-cuda/count-equal.cu b/ggml/src/ggml-cuda/count-equal.cu index 08898115d..411d08c0d 100644 --- a/ggml/src/ggml-cuda/count-equal.cu +++ b/ggml/src/ggml-cuda/count-equal.cu @@ -59,6 +59,5 @@ void ggml_cuda_count_equal(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { } break; default: GGML_ASSERT(false); - break; } } diff --git a/ggml/src/ggml-cuda/cp-async.cuh b/ggml/src/ggml-cuda/cp-async.cuh index 63d0c482f..1071352e7 100644 --- a/ggml/src/ggml-cuda/cp-async.cuh +++ b/ggml/src/ggml-cuda/cp-async.cuh @@ -55,3 +55,23 @@ static __device__ __forceinline__ void cp_async_wait_all() { NO_DEVICE_CODE; #endif // CP_ASYNC_AVAILABLE } + +// Commits the copies issued since the last commit as one group. +static __device__ __forceinline__ void cp_async_commit_group() { +#ifdef CP_ASYNC_AVAILABLE + asm volatile("cp.async.commit_group;"); +#else + NO_DEVICE_CODE; +#endif // CP_ASYNC_AVAILABLE +} + +// Makes each thread wait until at most n of its copy groups are pending. +// Like cp_async_wait_all, this does NOT provide any additional synchronization. +template <int n> +static __device__ __forceinline__ void cp_async_wait_group() { +#ifdef CP_ASYNC_AVAILABLE + asm volatile("cp.async.wait_group %0;" : : "n"(n)); +#else + NO_DEVICE_CODE; +#endif // CP_ASYNC_AVAILABLE +} diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index 90a06b97b..f1f1bf2da 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -477,6 +477,41 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg } else if (ggml_cuda_cpy_as_memcpy_2d(src0, src1, mc_width, mc_height, mc_spitch, mc_dpitch)) { CUDA_CHECK(cudaMemcpy2DAsync(src1_ddc, mc_dpitch, src0_ddc, mc_spitch, mc_width, mc_height, cudaMemcpyDeviceToDevice, main_stream)); + } else if (src0->type == GGML_TYPE_F8_E4M3 && src1->type == GGML_TYPE_F8_E4M3) { + ggml_cpy_scalar_cuda<uint8_t, uint8_t> + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F8_E4M3) { + if (contiguous_srcs) { + ggml_cpy_scalar_contiguous_cuda<float, ggml_fp8_e4m3_t> + (src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_scalar_cuda<float, ggml_fp8_e4m3_t> + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F8_E4M3) { + if (contiguous_srcs) { + ggml_cpy_scalar_contiguous_cuda<half, ggml_fp8_e4m3_t> + (src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_scalar_cuda<half, ggml_fp8_e4m3_t> + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F8_E4M3) { + if (contiguous_srcs) { + ggml_cpy_scalar_contiguous_cuda<nv_bfloat16, ggml_fp8_e4m3_t> + (src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_scalar_cuda<nv_bfloat16, ggml_fp8_e4m3_t> + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } + } else if (src0->type == GGML_TYPE_F8_E4M3 && src1->type == GGML_TYPE_F32) { + if (contiguous_srcs) { + ggml_cpy_scalar_contiguous_cuda<ggml_fp8_e4m3_t, float> + (src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_scalar_cuda<ggml_fp8_e4m3_t, float> + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { if (can_be_transposed) { ggml_cpy_scalar_cuda<float, float, true> diff --git a/ggml/src/ggml-cuda/diagmask.cu b/ggml/src/ggml-cuda/diagmask.cu deleted file mode 100644 index 4b713ba22..000000000 --- a/ggml/src/ggml-cuda/diagmask.cu +++ /dev/null @@ -1,40 +0,0 @@ -#include "diagmask.cuh" - -static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) { - const int col = blockDim.y*blockIdx.y + threadIdx.y; - const int row = blockDim.x*blockIdx.x + threadIdx.x; - - if (col >= ncols) { - return; - } - - const int i = row*ncols + col; - //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i]; - //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU - dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX; -} - -static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) { - const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1); - const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE; - const dim3 block_nums(nrows_x, block_num_x, 1); - diag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past); -} - -void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const float * src0_d = (const float *)src0->data; - float * dst_d = (float *)dst->data; - cudaStream_t stream = ctx.stream(); - - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); - - const int64_t ne00 = src0->ne[0]; - const int64_t ne01 = src0->ne[1]; - const int nrows0 = ggml_nrows(src0); - - const int n_past = ((int32_t *) dst->op_params)[0]; - - diag_mask_inf_f32_cuda(src0_d, dst_d, ne00, nrows0, ne01, n_past, stream); -} diff --git a/ggml/src/ggml-cuda/diagmask.cuh b/ggml/src/ggml-cuda/diagmask.cuh deleted file mode 100644 index 6cdbef17e..000000000 --- a/ggml/src/ggml-cuda/diagmask.cuh +++ /dev/null @@ -1,5 +0,0 @@ -#include "common.cuh" - -#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32 - -void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/dsv4-hc.cu b/ggml/src/ggml-cuda/dsv4-hc.cu index ca1d2dc8a..2d0b3cfea 100644 --- a/ggml/src/ggml-cuda/dsv4-hc.cu +++ b/ggml/src/ggml-cuda/dsv4-hc.cu @@ -145,7 +145,8 @@ static __global__ void dsv4_hc_pre_f32( dst[i0*sd0 + it*sd1] = scale * sum; } -template <bool has_comb> +// gated: post = gate_scale_out*sigmoid(gate_scale_in*post), the fused SCALE -> SIGMOID -> SCALE chain of qwen4exp +template <bool has_comb, bool gated> static __global__ void dsv4_hc_post_f32( const float * x, const float * residual, @@ -167,7 +168,9 @@ static __global__ void dsv4_hc_post_f32( int64_t sc2, int64_t sd0, int64_t sd1, - int64_t sd2) { + int64_t sd2, + float gate_scale_in, + float gate_scale_out) { ggml_cuda_pdl_lc(); const int64_t ir = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; const int64_t nr = n_embd * hc * n_tokens; @@ -182,7 +185,12 @@ static __global__ void dsv4_hc_post_f32( const int64_t idst = (ir / n_embd) % hc; const int64_t it = ir / (n_embd * hc); - float sum = x[i0*sx0 + it*sx1] * post[idst*sp0 + it*sp1]; + float p = post[idst*sp0 + it*sp1]; + if constexpr (gated) { + p = gate_scale_out * (1.0f / (1.0f + expf(-(gate_scale_in * p)))); + } + + float sum = x[i0*sx0 + it*sx1] * p; if constexpr (has_comb) { for (int64_t isrc = 0; isrc < hc; ++isrc) { sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2]; @@ -272,12 +280,19 @@ void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst scale); } -void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + const ggml_tensor * gate_scale_in, const ggml_tensor * gate_scale_out) { const ggml_tensor * x = dst->src[0]; const ggml_tensor * residual = dst->src[1]; - const ggml_tensor * post = dst->src[2]; const ggml_tensor * comb = dst->src[3]; + // when fused, read the raw post weights from before the SCALE -> SIGMOID -> SCALE chain + const bool gated = gate_scale_in != nullptr; + const ggml_tensor * post = gated ? gate_scale_in->src[0] : dst->src[2]; + + const float gate_in = gated ? ggml_get_op_params_f32(gate_scale_in, 0) : 1.0f; + const float gate_out = gated ? ggml_get_op_params_f32(gate_scale_out, 0) : 1.0f; + GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(residual->type == GGML_TYPE_F32); GGML_ASSERT(post->type == GGML_TYPE_F32); @@ -303,7 +318,8 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream()); - auto kernel = comb ? dsv4_hc_post_f32<true> : dsv4_hc_post_f32<false>; + auto kernel = comb ? (gated ? dsv4_hc_post_f32<true, true> : dsv4_hc_post_f32<true, false>) + : (gated ? dsv4_hc_post_f32<false, true> : dsv4_hc_post_f32<false, false>); ggml_cuda_kernel_launch(kernel, launch_params, (const float *) x->data, (const float *) residual->data, (const float *) post->data, comb ? (const float *) comb->data : nullptr, (float *) dst->data, @@ -312,5 +328,6 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float), nbp0 / sizeof(float), nbp1 / sizeof(float), nbc0 / sizeof(float), nbc1 / sizeof(float), nbc2 / sizeof(float), - nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float)); + nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float), + gate_in, gate_out); } diff --git a/ggml/src/ggml-cuda/dsv4-hc.cuh b/ggml/src/ggml-cuda/dsv4-hc.cuh index 2379aaefb..1885e7665 100644 --- a/ggml/src/ggml-cuda/dsv4-hc.cuh +++ b/ggml/src/ggml-cuda/dsv4-hc.cuh @@ -3,4 +3,6 @@ void ggml_cuda_op_dsv4_hc_comb(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst); -void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +// gate_scale_in/gate_scale_out: the SCALE nodes of a fused SCALE -> SIGMOID -> SCALE chain feeding src[2] +void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + const ggml_tensor * gate_scale_in = nullptr, const ggml_tensor * gate_scale_out = nullptr); diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 41d948335..6667ca400 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -25,6 +25,7 @@ typedef void (* fattn_kernel_t)( const char * __restrict__ mask, const char * __restrict__ sinks, const int * __restrict__ KV_max, + const int * __restrict__ KV_rows, float * __restrict__ dst, float2 * __restrict__ dst_meta, const float scale, @@ -145,6 +146,34 @@ static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_bf16( return sum; } +template <int D, int nthreads> +static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_f8_e4m3( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { + + const uint8_t * K_f8 = (const uint8_t *) K_c; + GGML_UNUSED(Q_q8); + GGML_UNUSED(Q_ds_v); + + float sum = 0.0f; + +#pragma unroll + for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += nthreads*4) { + uint8_t tmp[4]; + ggml_cuda_memcpy_1<sizeof(tmp)>(tmp, K_f8 + k_KQ_0 + (threadIdx.x % nthreads)*4); +#pragma unroll + for (int k_KQ_1 = 0; k_KQ_1 < 4; ++k_KQ_1) { + const float k = ggml_cuda_f8_e4m3_to_fp32(tmp[k_KQ_1]); +#ifdef V_DOT2_F32_F16_AVAILABLE + sum += k * __half2float(((const half *) Q_v)[k_KQ_0/nthreads + k_KQ_1]); +#else + sum += k * ((const float *) Q_v)[k_KQ_0/nthreads + k_KQ_1]; +#endif // V_DOT2_F32_F16_AVAILABLE + } + } + + return sum; +} + template<int D, int nthreads> static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q4_0( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { @@ -405,6 +434,19 @@ static __device__ __forceinline__ void dequantize_V_bf16(const void * __restrict } } +template <typename T, int ne> +static __device__ __forceinline__ void dequantize_V_f8_e4m3(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { + static_assert(ne == 2 || ne == 4, "bad ne"); + static_assert(std::is_same_v<T, half> || std::is_same_v<T, float>, "unsupported type"); + ggml_fp8_e4m3_t tmp[ne]; + ggml_cuda_memcpy_1<sizeof(tmp)>(tmp, (const ggml_fp8_e4m3_t *) vx + i0); + +#pragma unroll + for (int l = 0; l < ne; ++l) { + ((T *) dst)[l] = ggml_cuda_cast<T>(tmp[l]); + } +} + template <typename T, int ne> static __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { const block_q4_0 * x = (const block_q4_0 *) vx; @@ -427,17 +469,18 @@ static __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict const int8_t * q8 = (const int8_t *) &q; -#ifdef FP16_AVAILABLE if constexpr (std::is_same_v<T, half>) { +#ifdef FP16_AVAILABLE const half2 d = __half2half2(x[ib].d); #pragma unroll for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same_v<T, float>) { + } else if constexpr (std::is_same_v<T, float>) { const float d = x[ib].d; #pragma unroll @@ -465,8 +508,8 @@ static __device__ __forceinline__ void dequantize_V_q4_1(const void * __restrict const int8_t * q8 = (const int8_t *) &q; -#ifdef FP16_AVAILABLE if constexpr (std::is_same_v<T, half>) { +#ifdef FP16_AVAILABLE const half2 dm = x[ib].dm; const half2 d = __half2half2( __low2half(dm)); const half2 m = __half2half2(__high2half(dm)); @@ -475,9 +518,10 @@ static __device__ __forceinline__ void dequantize_V_q4_1(const void * __restrict for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m; } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same_v<T, float>) { + } else if constexpr (std::is_same_v<T, float>) { const float2 dm = __half22float2(x[ib].dm); #pragma unroll @@ -517,17 +561,18 @@ static __device__ __forceinline__ void dequantize_V_q5_0(const void * __restrict const int8_t * q8 = (const int8_t *) &q; -#ifdef FP16_AVAILABLE if constexpr (std::is_same_v<T, half>) { +#ifdef FP16_AVAILABLE const half2 d = __half2half2(x[ib].d); #pragma unroll for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same_v<T, float>) { + } else if constexpr (std::is_same_v<T, float>) { const float d = x[ib].d; #pragma unroll @@ -565,8 +610,8 @@ static __device__ __forceinline__ void dequantize_V_q5_1(const void * __restrict const int8_t * q8 = (const int8_t *) &q; -#ifdef FP16_AVAILABLE if constexpr (std::is_same_v<T, half>) { +#ifdef FP16_AVAILABLE const half2 dm = x[ib].dm; const half2 d = __half2half2( __low2half(dm)); const half2 m = __half2half2(__high2half(dm)); @@ -575,9 +620,10 @@ static __device__ __forceinline__ void dequantize_V_q5_1(const void * __restrict for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m; } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same_v<T, float>) { + } else if constexpr (std::is_same_v<T, float>) { const float2 dm = __half22float2(x[ib].dm); #pragma unroll @@ -600,17 +646,18 @@ static __device__ __forceinline__ void dequantize_V_q8_0(const void * __restrict int8_t qs[ne]; ggml_cuda_memcpy_1<ne, 2>(qs, x[ib].qs + iqs); -#ifdef FP16_AVAILABLE if constexpr (std::is_same<T, half>::value) { +#ifdef FP16_AVAILABLE const half2 d = __half2half2(x[ib].d); #pragma unroll for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(qs[l0 + 0], qs[l0 + 1]); } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same<T, float>::value) { + } else if constexpr (std::is_same<T, float>::value) { const float d = x[ib].d; #pragma unroll @@ -638,6 +685,8 @@ constexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() { return vec_dot_fattn_vec_KQ_q8_0<D, nthreads>; } else if constexpr (type_K == GGML_TYPE_BF16) { return vec_dot_fattn_vec_KQ_bf16<D, nthreads>; + } else if constexpr (type_K == GGML_TYPE_F8_E4M3) { + return vec_dot_fattn_vec_KQ_f8_e4m3<D, nthreads>; } else { static_assert(type_K == -1, "bad type"); return nullptr; @@ -660,6 +709,8 @@ constexpr __device__ dequantize_V_t get_dequantize_V() { return dequantize_V_q8_0<T, ne>; } else if constexpr (type_V == GGML_TYPE_BF16) { return dequantize_V_bf16<float, ne>; + } else if constexpr (type_V == GGML_TYPE_F8_E4M3) { + return dequantize_V_f8_e4m3<T, ne>; } else { static_assert(type_V == -1, "bad type"); return nullptr; @@ -991,8 +1042,13 @@ void launch_fattn( const bool V_is_K_view = V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)); - const ggml_tensor * mask = dst->src[3]; - const ggml_tensor * sinks = dst->src[4]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + const ggml_tensor * kv_rows = dst->src[5]; + + // with kv_rows the kernel iterates over the mask columns and reads K/V rows through kv_rows + const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : K->ne[1]; + GGML_ASSERT(!kv_rows || !use_sparse); ggml_tensor * KQV = dst; @@ -1113,8 +1169,8 @@ void launch_fattn( // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped. // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or - // multiple sequences of possibly different lengths. - const bool scan_mask = !use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1); + // multiple sequences of possibly different lengths, or kv_rows lists that end with padding. + const bool scan_mask = !use_sparse && mask && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows); if (scan_mask) { const int64_t s31 = mask->nb[1] / sizeof(half2); const int64_t s33 = mask->nb[3] / sizeof(half2); @@ -1123,7 +1179,7 @@ void launch_fattn( const dim3 block_dim_KV_max(FATTN_KQ_STRIDE/2, 1, 1); const int ne_KV_max = blocks_num_KV_max.x*blocks_num_KV_max.y; - const int iter_k = K->ne[1] / FATTN_KQ_STRIDE; + const int iter_k = n_kv_cols / FATTN_KQ_STRIDE; KV_max.alloc(ne_KV_max); ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_KV_max, block_dim_KV_max, 0, main_stream); @@ -1132,13 +1188,19 @@ void launch_fattn( CUDA_CHECK(cudaGetLastError()); } + // K/V are shared by all slices when read through kv_rows + if (kv_rows) { + nb13 = 0; + nb23 = 0; + } + const dim3 block_dim(warp_size, nwarps, 1); int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy. CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared)); GGML_ASSERT(max_blocks_per_sm > 0); int parallel_blocks = max_blocks_per_sm; - const int64_t n_kv = use_sparse ? n_kv_max : K->ne[1]; + const int64_t n_kv = use_sparse ? n_kv_max : n_kv_cols; const int ntiles_KV = (n_kv + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length. dim3 blocks_num; @@ -1255,6 +1317,7 @@ void launch_fattn( mask ? ((const char *) mask->data) : nullptr, sinks ? ((const char *) sinks->data) : nullptr, KV_max.ptr, + kv_rows ? (const int *) kv_rows->data : nullptr, !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 7f0f6abfd..07a6db842 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -36,11 +36,21 @@ struct fattn_mma_config { } \ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_ampere(const int DKQ, const int DV, const int ncols) { + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 8, 128, 2, 128, 24, 24, 24, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 16, 128, 2, 64, 24, 24, 24, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 32, 128, 2, 64, 24, 24, 24, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 64, 128, 2, 64, 24, 24, 24, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 2, 128, 32, 32, 32, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 128, 2, 64, 32, 32, 32, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 128, 2, 64, 32, 32, 32, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 128, 2, 64, 32, 32, 32, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 8, 128, 2, 128, 40, 40, 40, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 16, 128, 2, 64, 40, 40, 40, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 32, 128, 2, 64, 40, 40, 40, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 64, 128, 2, 64, 40, 40, 40, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 128, 2, 128, 40, 40, 40, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 128, 2, 64, 40, 40, 40, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 128, 2, 64, 40, 40, 40, 2, true); @@ -123,16 +133,29 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 64, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 64, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 32, 128, 128, 64, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 32, 128, 64, 128, 2, false); + // TODO tune specifically for Volta return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols); } static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_rdna(const int DKQ, const int DV, const int ncols) { + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 8, 128, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 16, 128, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 32, 128, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 64, 128, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 128, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 128, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 128, 2, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 8, 64, 2, 32, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 16, 64, 2, 32, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 32, 128, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 64, 128, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 64, 2, 32, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 64, 2, 32, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 128, 2, 64, 40, 40, 40, 1, true); @@ -180,11 +203,21 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co } static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_cdna(const int DKQ, const int DV, const int ncols) { + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 8, 256, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 16, 256, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 32, 256, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 64, 256, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 1, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 3, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 8, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 16, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 32, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 64, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 256, 2, 64, 40, 40, 40, 1, true); @@ -420,6 +453,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( // padded slots gather row 0, the -inf mask removes their contribution const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : 0; i_KV = index >= 0 ? index : 0; + } else if (indices) { + // K/V rows given per mask column, negative rows are padding + i_KV = max(indices[k_VKQ_0 + i], 0); } else { i_KV = k_VKQ_0 + i; } @@ -467,8 +503,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( if constexpr (use_sparse) { const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1; src = index >= 0 ? KV + int64_t(index)*stride_KV + k*h2_per_chunk : zero; + } else if (!oob_check || i < i_sup) { + const int64_t i_KV = indices ? max(indices[k_VKQ_0 + i], 0) : k_VKQ_0 + i; + src = KV + i_KV*stride_KV + k*h2_per_chunk; } else { - src = !oob_check || i < i_sup ? KV + int64_t(k_VKQ_0 + i)*stride_KV + k*h2_per_chunk : zero; + src = zero; } ggml_cuda_memcpy_1<16>(swizzle<stride_tile>(tile_KV, i*stride_tile + k*h2_per_chunk, i), src); } @@ -613,6 +652,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse); + constexpr int DKQp = GGML_PAD(DKQ, 16); + constexpr int DVp = GGML_PAD(DV, 16); + constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols); constexpr int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4; constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); @@ -629,12 +671,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( if constexpr (nstages > 1) { static_assert(!oob_check, "OOB check incompatible with multi-stage pipeline"); static_assert(!V_is_K_view, "K data reuse not implemented multi-stage loading"); - static_assert(nbatch_K2 == DKQ/2, "batching not implemented for multi stage loading"); + static_assert(nbatch_K2 == DKQp/2, "batching not implemented for multi stage loading"); constexpr bool use_cp_async = true; cp_async_wait_all(); __syncthreads(); flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> - (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, nullptr); + (V_h2, tile_V, min(nbatch_V2, DV/2), stride_V, k_VKQ_0, k_VKQ_sup, indices); } else { // the sparse mask values are gathered per element, always load them synchronously constexpr bool use_cp_async = nstages == 1 && !use_sparse; @@ -647,11 +689,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( // For MLA K and V have the same data. // Therefore, iterate over K in reverse and later re-use the data if possible. #pragma unroll - for (int k0_start = (DKQ/2-1) - (DKQ/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) { - const int k0_stop = k0_start + nbatch_K2 < DKQ/2 ? k0_start + nbatch_K2 : DKQ/2; + for (int k0_start = (DKQp/2-1) - (DKQp/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) { + const int k0_stop = k0_start + nbatch_K2 < DKQp/2 ? k0_start + nbatch_K2 : DKQp/2; if constexpr (nstages <= 1) { - const int k0_diff = k0_stop - k0_start; + const int k0_diff = min(k0_stop, DKQ/2) - k0_start; // Only load the real head size from global memory. constexpr bool use_cp_async = nstages == 1; flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> (K_h2 + k0_start, tile_K, k0_diff, stride_K, k_VKQ_0, k_VKQ_sup, indices); @@ -685,7 +727,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } } } else { - constexpr int stride_tile_Q = DKQ/2 + 4; + constexpr int stride_tile_Q = DKQp/2 + 4; #pragma unroll for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) { load_ldmatrix(Q_B[0], tile_Q + (threadIdx.y / np)*(T_B_KQ::I*stride_tile_Q) + k_KQ_0, stride_tile_Q); @@ -912,7 +954,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( if constexpr (cols_per_warp == 8) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[cols_per_thread - 1]); #pragma unroll - for (int i = 0; i < DV/T_C_VKQ::I; ++i) { + for (int i = 0; i < DVp/T_C_VKQ::I; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -923,7 +965,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( for (int col = 0; col < cols_per_thread; ++col) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l0 = 0; l0 < T_C_VKQ::ne; l0 += 2) { VKQ_C[i].x[l0 + col] *= KQ_max_scale_h2; @@ -935,7 +977,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( if constexpr (std::is_same_v<decltype(T_C_VKQ::x), half2[T_C_VKQ::ne]>) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -944,7 +986,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } else { static_assert(std::is_same_v<decltype(T_C_VKQ::x), float[T_C_VKQ::ne]>, "bad VKQ type"); #pragma unroll - for (int i = 0; i < DV/T_C_VKQ::J; ++i) { + for (int i = 0; i < DVp/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale[0]; @@ -955,7 +997,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const half2 KQ_max_scale_h2 = make_half2( KQ_max_scale[(threadIdx.x / 2) % 2], KQ_max_scale[(threadIdx.x / 2) % 2]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -991,19 +1033,19 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( (mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); } flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> - (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, nullptr); + (K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, indices); } } // Calculate VKQ tile, need to use logical rather than physical elements for i0 due to transposition of V: #pragma unroll - for (int i0_start = 0; i0_start < DV; i0_start += 2*nbatch_V2) { - static_assert(DV % (2*nbatch_V2) == 0, "bad loop size"); + for (int i0_start = 0; i0_start < DVp; i0_start += 2*nbatch_V2) { + static_assert(DVp % (2*nbatch_V2) == 0, "bad loop size"); const int i0_stop = i0_start + 2*nbatch_V2; if constexpr (nstages <= 1) { - const int i0_diff = i0_stop - i0_start; + const int i0_diff = min(i0_stop, DV) - i0_start; // Only load the real head size from global memory. if (!V_is_K_view || i0_stop > 2*nbatch_K2) { constexpr bool use_cp_async = nstages == 1; flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> @@ -1140,6 +1182,22 @@ template<int ncols> struct mma_tile_sizes<112, ncols> { using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major }; +template<int ncols> struct mma_tile_sizes<40, ncols> { + using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major + using T_A_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major +}; +template<int ncols> struct mma_tile_sizes<72, ncols> { + using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major + using T_A_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major +}; #else template<int DKQ, int ncols> struct mma_tile_sizes { using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR>; // row-major @@ -1165,6 +1223,22 @@ template<int ncols> struct mma_tile_sizes<112, ncols> { using T_B_VKQ = tile<16, 8, half2>; // column-major using T_C_VKQ = tile<16, 8, half2>; // column-major }; +template<int ncols> struct mma_tile_sizes<40, ncols> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile<16, 8, half2>; // column-major + using T_C_KQ = tile<16, 16, float>; // column-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile<16, 8, half2>; // column-major + using T_C_VKQ = tile<16, 8, half2>; // column-major +}; +template<int ncols> struct mma_tile_sizes<72, ncols> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile<16, 8, half2>; // column-major + using T_C_KQ = tile<16, 16, float>; // column-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile<16, 8, half2>; // column-major + using T_C_VKQ = tile<16, 8, half2>; // column-major +}; #endif // RDNA3 #elif defined(AMD_MFMA_AVAILABLE) template<int DKQ, int ncols> struct mma_tile_sizes { @@ -1234,6 +1308,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse); + constexpr int DKQp = GGML_PAD(DKQ, 16); + constexpr int DVp = GGML_PAD(DV, 16); + if (cols_per_warp > ncols) { NO_DEVICE_CODE; return; @@ -1242,7 +1319,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( static_assert(nwarps * (cols_per_warp/ncols2) % ncols1 == 0, "bad nwarps"); constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols); - constexpr int stride_tile_Q = DKQ/2 + 4; + constexpr int stride_tile_Q = DKQp/2 + 4; constexpr int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4; constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); constexpr int stride_tile_KV_max = stride_tile_K > stride_tile_V ? stride_tile_K : stride_tile_V; @@ -1252,17 +1329,17 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( half2 * tile_V = nstages > 1 ? tile_K + nbatch_fa * stride_tile_K : tile_K; half * tile_mask = (half *) (nstages > 1 ? tile_V + nbatch_fa * stride_tile_V : tile_V + nbatch_fa * stride_tile_KV_max); - T_B_KQ Q_B[(Q_in_reg ? DKQ/(2*T_B_KQ::J) : 1)]; + T_B_KQ Q_B[(Q_in_reg ? DKQp/(2*T_B_KQ::J) : 1)]; #if defined(TURING_MMA_AVAILABLE) - T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)]; -#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) - T_C_VKQ VKQ_C[DV % 32 != 0 ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)]; + T_C_VKQ VKQ_C[cols_per_warp == 8 ? DVp/T_C_VKQ::I : DVp/(2*T_C_VKQ::J)]; +#elif defined(AMD_WMMA_256B_AVAILABLE) + T_C_VKQ VKQ_C[DVp % 32 != 0 ? DVp/T_C_VKQ::J : DVp/(2*T_C_VKQ::J)]; #elif defined(AMD_MFMA_AVAILABLE) - T_C_VKQ VKQ_C[ DV/T_C_VKQ::J]; + T_C_VKQ VKQ_C[ DVp/T_C_VKQ::J]; #elif defined(AMD_WMMA_AVAILABLE) - T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; + T_C_VKQ VKQ_C[ DVp/(2*T_C_VKQ::J)]; #else // Volta - T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; + T_C_VKQ VKQ_C[ DVp/(2*T_C_VKQ::J)]; #endif // defined(TURING_MMA_AVAILABLE) float KQ_rowsum[cols_per_thread] = {0.0f}; @@ -1278,8 +1355,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const half2 scale_h2 = make_half2(scale, scale); #pragma unroll for (int stride_k : {warp_size, warp_size/2, warp_size/4, warp_size/8}) { - const int k0_start = stride_k == warp_size ? 0 : DKQ/2 - (DKQ/2) % (2*stride_k); - const int k0_stop = DKQ/2 - (DKQ/2) % (1*stride_k); + const int k0_start = stride_k == warp_size ? 0 : DKQp/2 - (DKQp/2) % (2*stride_k); + const int k0_stop = DKQp/2 - (DKQp/2) % (1*stride_k); const int stride_jc = warp_size / stride_k; if (k0_start == k0_stop) { @@ -1302,7 +1379,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); - const float2 tmp = Q_f2[(jt*ncols1 + j)*stride_Q1 + c*stride_Q2 + k]; + const float2 tmp = DKQp == DKQ || k < DKQ/2 ? + Q_f2[(jt*ncols1 + j)*stride_Q1 + c*stride_Q2 + k] : make_float2(0.0f, 0.0f); tile_Q[jc*stride_tile_Q + k] = scale_h2 * make_half2(tmp.x, tmp.y); } } else { @@ -1322,19 +1400,50 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const int j0 = (threadIdx.y / np) * cols_per_warp; #pragma unroll - for (int k0 = 0; k0 < DKQ/2; k0 += T_B_KQ::J) { + for (int k0 = 0; k0 < DKQp/2; k0 += T_B_KQ::J) { load_ldmatrix(Q_B[k0/T_B_KQ::J], tile_Q + j0*stride_tile_Q + k0, stride_tile_Q); } } __syncthreads(); + // Zero the pad columns of the K/V tiles once, the tile loads only write the first DKQ/2 (DV/2) half2 of a row. + // Done after Q_B is loaded because tile_K may alias tile_Q. + if constexpr (DKQp != DKQ || DVp != DV) { + static_assert(!swz, "pad zero-fill assumes unswizzled tiles"); + static_assert(nbatch_K2 == DKQp/2 && nbatch_V2 == DVp/2, "K/V tile batching not implemented for padded head sizes"); + static_assert(DKQ == DV, "pad zero-fill for DKQ != DV requires separate handling of the aliased V tile"); + + const half2 zero[4] = {{0.0f, 0.0f}, {0.0f, 0.0f}, {0.0f, 0.0f}, {0.0f, 0.0f}}; + const int tid = threadIdx.y*warp_size + threadIdx.x; +#pragma unroll + for (int i0 = 0; i0 < nbatch_fa; i0 += nwarps*warp_size) { + const int i = i0 + tid; + + if (i0 + nwarps*warp_size > nbatch_fa && i >= nbatch_fa) { + break; + } + +#pragma unroll + for (int k = DKQ/2; k < DKQp/2; k += 4) { + ggml_cuda_memcpy_1<16>(tile_K + i*stride_tile_K + k, zero); + } + if constexpr (nstages > 1) { // tile_V is only a separate buffer with multi-stage loading. +#pragma unroll + for (int k = DV/2; k < DVp/2; k += 4) { + ggml_cuda_memcpy_1<16>(tile_V + i*stride_tile_V + k, zero); + } + } + } + __syncthreads(); + } + int kb0 = kb0_start; // Preload mask and K data for first iteration when using cp_async with multiple stages: if constexpr (nstages > 1) { static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading"); - static_assert(nbatch_K2 == DKQ/2, "batching not implemented for multi-stage pipeline"); + static_assert(nbatch_K2 == DKQp/2, "batching not implemented for multi-stage pipeline"); constexpr bool use_cp_async = true; constexpr bool oob_check = false; constexpr int k_VKQ_sup = nbatch_fa; @@ -1343,7 +1452,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); } flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> - (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr); + (K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, kb0*nbatch_fa, k_VKQ_sup, indices); } // kb0_start is always < kb0_stop so the last iter can be executed unconditionally. @@ -1448,7 +1557,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( if constexpr (cols_per_warp == 8) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[cols_per_thread - 1]); #pragma unroll - for (int i = 0; i < DV/T_C_VKQ::I; ++i) { + for (int i = 0; i < DVp/T_C_VKQ::I; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -1459,7 +1568,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( for (int col = 0; col < cols_per_thread; ++col) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l0 = 0; l0 < T_C_VKQ::ne; l0 += 2) { VKQ_C[i].x[l0 + col] *= KQ_max_scale_h2; @@ -1471,7 +1580,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( if constexpr (std::is_same_v<decltype(T_C_VKQ::x), half2[T_C_VKQ::ne]>) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -1480,7 +1589,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } else { static_assert(std::is_same_v<decltype(T_C_VKQ::x), float[T_C_VKQ::ne]>, "bad VKQ type"); #pragma unroll - for (int i = 0; i < DV/T_C_VKQ::J; ++i) { + for (int i = 0; i < DVp/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale[0]; @@ -1491,7 +1600,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const int col = (threadIdx.x / 2) % 2; const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -1505,7 +1614,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( // So also write VKQ accumulators to shared memory in column-major format if np == 1. constexpr int tile_stride = nbatch_combine + 4; - static_assert((DV/2) % nbatch_combine == 0, "bad nbatch_combine"); + static_assert((DVp/2) % nbatch_combine == 0, "bad nbatch_combine"); if constexpr (cols_per_warp == 8) { const int jc_cwmo = (threadIdx.x % (2*T_C_VKQ::J)) / T_C_VKQ::J; // jc combine write meta offset @@ -1649,7 +1758,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } #pragma unroll - for (int k00 = 0; k00 < DV/2; k00 += nbatch_combine) { + for (int k00 = 0; k00 < DVp/2; k00 += nbatch_combine) { if constexpr (cols_per_warp == 8) { static_assert(std::is_same_v<decltype(T_C_VKQ::x), half2[T_C_VKQ::ne]>, "bad VKQ type"); const int jc_cwd = threadIdx.y*T_B_KQ::I + T_B_KQ::get_i(-1); // jc combine write data @@ -1748,6 +1857,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); + if (DVp != DV && k00 + k >= DV/2) { + continue; + } + float2 dstk_val = make_float2(0.0f, 0.0f); #pragma unroll for (int ip = 0; ip < np; ++ip) { @@ -1772,7 +1885,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } } } - if (np > 1) { + if (np > 1 || nbatch_combine != DKQ/2) { __syncthreads(); } } @@ -1802,6 +1915,7 @@ static __global__ void flash_attn_ext_f16( const char * mask_ptr, const char * sinks_ptr, const int * KV_max_ptr, + const int * KV_rows_ptr, float * dst_ptr, float2 * dst_meta_ptr, const float scale, @@ -1866,7 +1980,7 @@ static __global__ void flash_attn_ext_f16( #endif // defined(AMD_WMMA_AVAILABLE) #if defined(AMD_MFMA_AVAILABLE) - if (ncols1*ncols2 < 16 || (DKQ > 256 && ncols1*ncols2 < 32)) { + if (ncols1*ncols2 < 16 || (DKQ > 256 && ncols1*ncols2 < 32 && !use_sparse)) { NO_DEVICE_CODE; return; } @@ -1924,7 +2038,8 @@ static __global__ void flash_attn_ext_f16( const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV); const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr; - const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr; + const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : + KV_rows_ptr ? KV_rows_ptr + int64_t(sequence)*ne11 : nullptr; const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f; @@ -1973,7 +2088,8 @@ static __global__ void flash_attn_ext_f16( const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV); const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr; - const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr; + const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : + KV_rows_ptr ? KV_rows_ptr + int64_t(sequence)*ne11 : nullptr; const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f; @@ -1989,7 +2105,7 @@ static __global__ void flash_attn_ext_f16( (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); #else - GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale, + GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne01, ne02, ne03, nb01, nb02, nb03, @@ -2025,6 +2141,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml const int nwarps = nthreads / warp_size_host; constexpr bool V_is_K_view = DKQ == 576; // Guaranteed by the kernel selection logic in fattn.cu + constexpr int DKQp = GGML_PAD(DKQ, 16); // KV tile strides must match flash_attn_ext_f16_iter / _process_tile. const bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols, cc); @@ -2032,7 +2149,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml const int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(stride_tile_K, stride_tile_V) * sizeof(half2); const size_t nbytes_shared_KV_2stage = nbatch_fa * (stride_tile_K + stride_tile_V) * sizeof(half2); - const size_t nbytes_shared_Q = ncols * (DKQ/2 + 4) * sizeof(half2); + const size_t nbytes_shared_Q = ncols * (DKQp/2 + 4) * sizeof(half2); const size_t nbytes_shared_mask = ncols1 * (nbatch_fa/2 + 4) * sizeof(half2); const size_t nbytes_shared_combine = nwarps*cols_per_warp * (nbatch_combine + 4) * sizeof(half2); @@ -2054,7 +2171,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml bool use_sparse = false; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if !defined(GGML_USE_MUSA) if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2)) { if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, ncols1, ncols2)) { constexpr bool use_sparse_kernel = true; @@ -2077,7 +2194,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml } } } else -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#endif // !defined(GGML_USE_MUSA) { constexpr bool use_sparse_kernel = false; fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>; @@ -2117,28 +2234,36 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 8, 8); \ extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/16, 16); \ +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 40, 40, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 8) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 72, 72, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 8) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 40, 40, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 16) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 72, 72, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 16) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 40, 40, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 32) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 72, 72, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 32) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 40, 40, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 64) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 72, 72, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 64) diff --git a/ggml/src/ggml-cuda/fattn-tile.cu b/ggml/src/ggml-cuda/fattn-tile.cu index e563729a2..accef6315 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cu +++ b/ggml/src/ggml-cuda/fattn-tile.cu @@ -53,8 +53,7 @@ void ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor GGML_ASSERT(V->ne[0] == 512); ggml_cuda_flash_attn_ext_tile_case<576, 512>(ctx, dst); } break; - default: { + default: GGML_ABORT("Unsupported head size"); - } break; } } diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh index b5ea915e1..ba4628fec 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cuh +++ b/ggml/src/ggml-cuda/fattn-tile.cuh @@ -797,6 +797,7 @@ static __global__ void flash_attn_tile( const char * mask_ptr, const char * sinks_ptr, const int * KV_max_ptr, + const int * KV_rows_ptr, float * dst_ptr, float2 * dst_meta_ptr, const float scale, @@ -821,6 +822,7 @@ static __global__ void flash_attn_tile( const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; float * GGML_CUDA_RESTRICT dst = dst_ptr; float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr; + GGML_UNUSED(KV_rows_ptr); // Skip unused kernel variants for faster compilation: @@ -955,7 +957,7 @@ static __global__ void flash_attn_tile( if (ncols2 == 1) { // Branch with out-of-bounds checks. int k_VKQ_0 = blockIdx.y*nbatch_fa; - while (k_VKQ_0 < k_VKQ_max - nbatch_fa) { + while (k_VKQ_0 <= k_VKQ_max - nbatch_fa) { constexpr bool oob_check = false; flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check> (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, @@ -1132,7 +1134,7 @@ static __global__ void flash_attn_tile( } } #else - GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale, + GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne01, ne02, ne03, nb01, nb02, nb03, diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh index 57a285565..02c3c327f 100644 --- a/ggml/src/ggml-cuda/fattn-vec.cuh +++ b/ggml/src/ggml-cuda/fattn-vec.cuh @@ -25,6 +25,7 @@ static __global__ void flash_attn_ext_vec( const char * mask_ptr, const char * sinks_ptr, const int * KV_max_ptr, + const int * KV_rows_ptr, float * dst_ptr, float2 * dst_meta_ptr, const float scale, @@ -50,6 +51,7 @@ static __global__ void flash_attn_ext_vec( const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; float * GGML_CUDA_RESTRICT dst = dst_ptr; float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr; + GGML_UNUSED(KV_rows_ptr); // Skip unused kernel variants for faster compilation: if (use_logit_softcap && !(D == 128 || D == 256)) { @@ -94,7 +96,7 @@ static __global__ void flash_attn_ext_vec( constexpr int V_cols_per_iter = WARP_SIZE / nthreads_V; constexpr vec_dot_KQ_t vec_dot_KQ = get_vec_dot_KQ<type_K, D, nthreads_KQ>(); - constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16; + constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16 && type_K != GGML_TYPE_F8_E4M3; #ifdef V_DOT2_F32_F16_AVAILABLE constexpr dequantize_V_t dequantize_V = get_dequantize_V<type_V, half, V_rows_per_thread>(); #else @@ -201,6 +203,27 @@ static __global__ void flash_attn_ext_vec( } __syncthreads(); + } else if constexpr (type_K == GGML_TYPE_F8_E4M3) { +#pragma unroll + for (int j = 0; j < ncols; ++j) { + const float * Q_j = (const float *) (Q + j*nb01); +#pragma unroll + for (int i0 = 0; i0 < D; i0 += nthreads_KQ*4) { + const int i = i0 + (threadIdx.x % nthreads_KQ)*4; + float4 tmp = make_float4(0.0f, 0.0f, 0.0f, 0.0f); + if (ncols == 1 || ic0 + j < int(ne01.z)) { + ggml_cuda_memcpy_1<sizeof(tmp)>(&tmp, Q_j + i); + } +#ifdef V_DOT2_F32_F16_AVAILABLE + const half2 scale_h2 = make_half2(scale, scale); + Q_reg[j][i0/(2*nthreads_KQ) + 0] = __hmul2(make_half2(tmp.x, tmp.y), scale_h2); + Q_reg[j][i0/(2*nthreads_KQ) + 1] = __hmul2(make_half2(tmp.z, tmp.w), scale_h2); +#else + Q_reg[j][i0/(2*nthreads_KQ) + 0] = make_float2(tmp.x*scale, tmp.y*scale); + Q_reg[j][i0/(2*nthreads_KQ) + 1] = make_float2(tmp.z*scale, tmp.w*scale); +#endif + } + } } else { #ifdef V_DOT2_F32_F16_AVAILABLE const half2 scale_h2 = make_half2(scale, scale); @@ -317,7 +340,7 @@ static __global__ void flash_attn_ext_vec( #endif // V_DOT2_F32_F16_AVAILABLE } - ggml_cuda_syncwarp(); + ggml_cuda_syncwarp(); // Needed for KQ synchronization. #pragma unroll for (int k0 = 0; k0 < WARP_SIZE; k0 += V_cols_per_iter) { @@ -374,6 +397,8 @@ static __global__ void flash_attn_ext_vec( } #endif // V_DOT2_F32_F16_AVAILABLE } + + ggml_cuda_syncwarp(); // Needed for KQ synchronization. } if (sinks && blockIdx.y == 0) { @@ -512,7 +537,7 @@ static __global__ void flash_attn_ext_vec( dst_meta[((sequence*int(ne01.z) + ic0 + tid)*ne02 + head)*gridDim.y + blockIdx.y] = make_float2(KQ_max[tid], KQ_sum[tid]); } #else - GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale, + GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne01, ne02, ne03, nb01, nb02, nb03, @@ -583,6 +608,7 @@ void ggml_cuda_flash_attn_ext_vec_case(ggml_backend_cuda_context & ctx, ggml_ten extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_1); \ extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q8_0); \ extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_BF16); \ + extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_F8_E4M3); \ EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F16) EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_0) @@ -591,6 +617,7 @@ EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_0) EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_1) EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q8_0) EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_BF16) +EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F8_E4M3) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F16) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_0) @@ -599,6 +626,7 @@ EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_0) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_1) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q8_0) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_BF16) +EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F8_E4M3) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F16) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_0) @@ -607,3 +635,4 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16) +EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F8_E4M3) diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index bb34e01f3..cf44ab4f5 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -5,7 +5,8 @@ #include "fattn-vec.cuh" #include "fattn.cuh" -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if !defined(GGML_USE_MUSA) + // one list per group of ncols1 queries: a column is selected if any query of the group can see it template <int ncols1, bool oob> __launch_bounds__(256, 1) @@ -14,10 +15,20 @@ static __global__ void flash_attn_mask_to_sparse_indices( const int n_kv_max, const int64_t s31, const int64_t s33) { ggml_cuda_pdl_sync(); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int values_per_lane = 8; + using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>; + constexpr uint64_t ballot_mask = ~(lane_mask_t)0; + const auto popc = [](const lane_mask_t mask) { + if constexpr (warp_size == 64) { + return __popcll(mask); + } else { + return __popc(mask); + } + }; const int tid = threadIdx.x; - const int warp = tid / WARP_SIZE; - const int lane = tid % WARP_SIZE; + const int warp = tid / warp_size; + const int lane = tid % warp_size; const int sequence = blockIdx.y; const int group = blockIdx.x; @@ -27,7 +38,7 @@ static __global__ void flash_attn_mask_to_sparse_indices( const half * mask = mask_ptr + sequence*s33 + q0*s31; int32_t * indices = indices_ptr + (int64_t(sequence)*gridDim.x + group)*n_kv_max; - __shared__ int warp_offsets[256/WARP_SIZE]; + __shared__ int warp_offsets[256/warp_size]; __shared__ int row_count; __shared__ int chunk_count; @@ -37,11 +48,11 @@ static __global__ void flash_attn_mask_to_sparse_indices( __syncthreads(); for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) { - uint32_t selected_warp[values_per_lane]; + lane_mask_t selected_warp[values_per_lane]; int warp_count = 0; #pragma unroll for (int item = 0; item < values_per_lane; ++item) { - const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane; + const int i = i0 + (warp*values_per_lane + item)*warp_size + lane; bool selected = false; if (i < ne30) { #pragma unroll @@ -49,8 +60,8 @@ static __global__ void flash_attn_mask_to_sparse_indices( selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i])); } } - selected_warp[item] = __ballot_sync(0xFFFFFFFF, selected); - warp_count += __popc(selected_warp[item]); + selected_warp[item] = (lane_mask_t)__ballot_sync(ballot_mask, (int) selected); + warp_count += popc(selected_warp[item]); } if (lane == 0) { @@ -61,7 +72,7 @@ static __global__ void flash_attn_mask_to_sparse_indices( if (tid == 0) { int offset = 0; #pragma unroll - for (int iw = 0; iw < 256/WARP_SIZE; ++iw) { + for (int iw = 0; iw < 256/warp_size; ++iw) { const int count = warp_offsets[iw]; warp_offsets[iw] = offset; offset += count; @@ -70,16 +81,16 @@ static __global__ void flash_attn_mask_to_sparse_indices( } __syncthreads(); - const uint32_t lane_mask = lane == 0 ? 0 : (1u << lane) - 1; + const lane_mask_t lane_mask = lane == 0 ? 0 : ((lane_mask_t)1 << lane) - 1; int warp_item_offset = 0; #pragma unroll for (int item = 0; item < values_per_lane; ++item) { - const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane; - const int dst = row_count + warp_offsets[warp] + warp_item_offset + __popc(selected_warp[item] & lane_mask); - if ((selected_warp[item] & (uint32_t(1) << lane)) && dst < n_kv_max) { + const int i = i0 + (warp*values_per_lane + item)*warp_size + lane; + const int dst = row_count + warp_offsets[warp] + warp_item_offset + popc(selected_warp[item] & lane_mask); + if ((selected_warp[item] & ((lane_mask_t)1 << lane)) && dst < n_kv_max) { indices[dst] = i; } - warp_item_offset += __popc(selected_warp[item]); + warp_item_offset += popc(selected_warp[item]); } __syncthreads(); @@ -101,13 +112,13 @@ static __global__ void flash_attn_mask_to_sparse_indices( // the dependent grid reads indices, signal once the row is complete ggml_cuda_pdl_lc(); } -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#endif // !defined(GGML_USE_MUSA) void ggml_cuda_flash_attn_ext_compact_mask( const ggml_tensor * mask, int32_t * indices, int32_t * counts, int32_t n_queries, int32_t ncols1, int32_t n_kv_max, cudaStream_t stream) { -#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) +#ifdef GGML_USE_MUSA GGML_UNUSED_VARS(mask, indices, counts, n_queries, ncols1, n_kv_max, stream); - GGML_ABORT("sparse flash attention is only supported on NVIDIA CUDA"); + GGML_ABORT("sparse flash attention is not supported on MUSA"); #else const int64_t s31 = mask->nb[1] / sizeof(half); const int64_t s33 = mask->nb[3] / sizeof(half); @@ -122,16 +133,17 @@ void ggml_cuda_flash_attn_ext_compact_mask( ggml_cuda_kernel_launch(kernel, launch_params, (const half *) mask->data, indices, counts, int(mask->ne[0]), n_queries, n_kv_max, s31, s33); CUDA_CHECK(cudaGetLastError()); -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#endif // !defined(GGML_USE_MUSA) } bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2) { -#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) +#ifdef GGML_USE_MUSA GGML_UNUSED_VARS(cc, dst, ncols1, ncols2); return false; #else const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; float max_bias = 0.0f; @@ -139,16 +151,21 @@ bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_ memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + // On CDNA only the DKQ=576 kernel variant with ncols2=16 is compiled with sparse support. + if (amd_mfma_available(cc) && !ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(int(Q->ne[0]), int(V->ne[0]), 1, 16)) { + return false; + } + const int32_t n_kv_max = ggml_get_op_params_i32(dst, 4); // the dense kernel handles up to 64/ncols2 queries per K/V pass, the single-query gather has to beat that const int64_t n_gather = (ncols1 == 1 ? std::min<int64_t>(Q->ne[1], 64/ncols2) : ncols1) * (int64_t) n_kv_max; - return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && + return ((GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc)) || amd_mfma_available(cc)) && dst->src[5] == nullptr && mask != nullptr && n_kv_max > 0 && max_bias == 0.0f && logit_softcap == 0.0f && mask->ne[0] == K->ne[1] && mask->ne[1] >= Q->ne[1] && mask->ne[2] == 1 && - K->ne[1] >= std::max<int64_t>(4096, 2*n_gather); -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + K->ne[1] >= std::max<int64_t>(4096, GGML_CUDA_CC_IS_NVIDIA(cc) ? 2*n_gather : 2LL*n_kv_max); +#endif // !defined(GGML_USE_MUSA) } template <int DKQ, int DV, int ncols2> @@ -156,7 +173,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const ggml_tensor * Q = dst->src[0]; -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if !defined(GGML_USE_MUSA) if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 1, ncols2)) { // a sparse variant at the full tile width gathers the union of its queries once, prefer it for large batches constexpr bool has_wide_sparse = ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 64/ncols2, ncols2); @@ -165,7 +182,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con return; } } -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#endif // !defined(GGML_USE_MUSA) if constexpr (ncols2 <= 8) { if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) { @@ -199,12 +216,15 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; + // with kv_rows the kernel iterates over the mask columns + const int64_t n_kv = dst->src[5] ? dst->src[5]->ne[0] : K->ne[1]; + float max_bias = 0.0f; memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); // Edge cases like no mask, ALiBi, unpadded K/V, or misaligned addresses for large data transfers // are put into the template specialization without GQA optimizations. - bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; + bool use_gqa_opt = mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0; for (const ggml_tensor * t : {Q, K, V, mask}) { if (t == nullptr || ggml_is_quantized(t->type)) { continue; @@ -294,10 +314,18 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg const ggml_tensor * mask = dst->src[3]; switch (Q->ne[0]) { + case 40: + GGML_ASSERT(V->ne[0] == 40); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 40, 40>(ctx, dst); + break; case 64: GGML_ASSERT(V->ne[0] == 64); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 64, 64>(ctx, dst); break; + case 72: + GGML_ASSERT(V->ne[0] == 72); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 72, 72>(ctx, dst); + break; case 80: GGML_ASSERT(V->ne[0] == 80); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 80, 80>(ctx, dst); @@ -365,7 +393,9 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); const int gqa_ratio = Q->ne[2] / K->ne[2]; - if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash + if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst); + } else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash if (cc >= GGML_CUDA_CC_DGX_SPARK) { if (Q->ne[1] <= 8) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst); @@ -412,15 +442,12 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg } break; default: GGML_ABORT("fatal error"); - break; } } #define FATTN_VEC_CASE(D, type_K_case, type_V_case) \ if constexpr (GGML_CUDA_FA_##type_K_case##_##type_V_case) { \ - const bool type_K_okay = type_K == GGML_TYPE_##type_K_case || (type_K == GGML_TYPE_F32 && GGML_TYPE_##type_K_case == GGML_TYPE_F16); \ - const bool type_V_okay = type_V == GGML_TYPE_##type_V_case || (type_V == GGML_TYPE_F32 && GGML_TYPE_##type_V_case == GGML_TYPE_F16); \ - if (head_size == (D) && type_K_okay && type_V_okay) { \ + if (head_size == (D) && type_K == GGML_TYPE_##type_K_case && type_V == GGML_TYPE_##type_V_case) { \ return ggml_cuda_flash_attn_ext_vec_case<D, GGML_TYPE_##type_K_case, GGML_TYPE_##type_V_case>; \ } \ } \ @@ -433,7 +460,9 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg typedef void (* fattn_vec_case_t)(ggml_backend_cuda_context & ctx, ggml_tensor * dst); // Vector kernel for the given head size and K/V types, nullptr if its template instance was not compiled: -static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K, const ggml_type type_V) { +static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K_in, const ggml_type type_V_in) { + const ggml_type type_K = type_K_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_K_in; + const ggml_type type_V = type_V_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_V_in; FATTN_VEC_CASES_ALL_D(F16, F16) FATTN_VEC_CASES_ALL_D(Q4_0, F16) FATTN_VEC_CASES_ALL_D(Q4_1, F16) @@ -441,6 +470,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, F16) FATTN_VEC_CASES_ALL_D(Q8_0, F16) FATTN_VEC_CASES_ALL_D(BF16, F16) + FATTN_VEC_CASES_ALL_D(F8_E4M3, F16) FATTN_VEC_CASES_ALL_D(F16, Q4_0) FATTN_VEC_CASES_ALL_D(Q4_0, Q4_0) @@ -449,6 +479,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q4_0) FATTN_VEC_CASES_ALL_D(Q8_0, Q4_0) FATTN_VEC_CASES_ALL_D(BF16, Q4_0) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q4_0) FATTN_VEC_CASES_ALL_D(F16, Q4_1) FATTN_VEC_CASES_ALL_D(Q4_0, Q4_1) @@ -457,6 +488,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q4_1) FATTN_VEC_CASES_ALL_D(Q8_0, Q4_1) FATTN_VEC_CASES_ALL_D(BF16, Q4_1) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q4_1) FATTN_VEC_CASES_ALL_D(F16, Q5_0) FATTN_VEC_CASES_ALL_D(Q4_0, Q5_0) @@ -465,6 +497,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q5_0) FATTN_VEC_CASES_ALL_D(Q8_0, Q5_0) FATTN_VEC_CASES_ALL_D(BF16, Q5_0) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q5_0) FATTN_VEC_CASES_ALL_D(F16, Q5_1) FATTN_VEC_CASES_ALL_D(Q4_0, Q5_1) @@ -473,6 +506,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q5_1) FATTN_VEC_CASES_ALL_D(Q8_0, Q5_1) FATTN_VEC_CASES_ALL_D(BF16, Q5_1) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q5_1) FATTN_VEC_CASES_ALL_D(F16, Q8_0) FATTN_VEC_CASES_ALL_D(Q4_0, Q8_0) @@ -481,6 +515,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q8_0) FATTN_VEC_CASES_ALL_D(Q8_0, Q8_0) FATTN_VEC_CASES_ALL_D(BF16, Q8_0) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q8_0) FATTN_VEC_CASES_ALL_D(F16, BF16) FATTN_VEC_CASES_ALL_D(Q4_0, BF16) @@ -489,6 +524,16 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, BF16) FATTN_VEC_CASES_ALL_D(Q8_0, BF16) FATTN_VEC_CASES_ALL_D(BF16, BF16) + FATTN_VEC_CASES_ALL_D(F8_E4M3, BF16) + + FATTN_VEC_CASES_ALL_D(F16, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q4_0, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q4_1, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q5_0, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q5_1, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q8_0, F8_E4M3) + FATTN_VEC_CASES_ALL_D(BF16, F8_E4M3) + FATTN_VEC_CASES_ALL_D(F8_E4M3, F8_E4M3) return nullptr; } @@ -532,6 +577,7 @@ static bool ggml_cuda_fattn_kv_type_supported(const ggml_type type) { case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: + case GGML_TYPE_F8_E4M3: return true; default: return false; @@ -549,6 +595,10 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * kv_rows = dst->src[5]; + + // with kv_rows the kernel iterates over the mask columns + const int64_t n_kv = kv_rows ? kv_rows->ne[0] : K->ne[1]; const int gqa_ratio = Q->ne[2] / K->ne[2]; GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); @@ -558,7 +608,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const // The effective batch size for the kernel can be increased by gqa_ratio. // The kernel versions without this optimization are also used for ALiBi, if there is no mask, or if the KV cache is not padded, - bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; + bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0; for (const ggml_tensor * t : {Q, K, V, mask}) { if (t == nullptr || ggml_is_quantized(t->type)) { continue; @@ -630,12 +680,18 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_NONE; } + // only the MMA kernel reads K/V through kv_rows + if (kv_rows) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && mask && Q->ne[0] != 40 && Q->ne[0] != 72 ? + BEST_FATTN_KERNEL_MMA_F16 : BEST_FATTN_KERNEL_NONE; + } + // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes: // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path. const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0; // If Turing tensor cores are available, use them: - if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if (turing_mma_available(cc)) { if (can_use_vector_kernel) { if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { // the sparse gather exists only in the MMA kernel: (DKQ, DV, 1, 8) with GQA > 4 @@ -680,6 +736,11 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_MMA_F16; } + // On CDNA, the sparse MMA kernel is used for DKQ=576 regardless of the head size heuristics. + if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) { + return BEST_FATTN_KERNEL_MMA_F16; + } + // AMD MFMA needs a certain minimum batch size to outscale the tile kernel for large head sizes. if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { if ((Q->ne[0] <= 64 && Q->ne[1] * gqa_ratio_eff > 8)) { diff --git a/ggml/src/ggml-cuda/fp8.cu b/ggml/src/ggml-cuda/fp8.cu new file mode 100644 index 000000000..5fdec4220 --- /dev/null +++ b/ggml/src/ggml-cuda/fp8.cu @@ -0,0 +1,206 @@ +#include "fp8.cuh" + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 + +struct fp8_abs_src { + const float * x; + int64_t ne0; + int64_t ne1; + int64_t ne2; + int64_t s1; + int64_t s2; + int64_t s3; + + __device__ float operator()(int64_t i) const { + const int64_t i0 = i % ne0; + i /= ne0; + const int64_t i1 = i % ne1; + i /= ne1; + const int64_t i2 = i % ne2; + const int64_t i3 = i / ne2; + const float value = fabsf(x[i0 + i1*s1 + i2*s2 + i3*s3]); + return isfinite(value) ? value : 448.0f; + } +}; + +static __global__ void fp8_amax_partials(fp8_abs_src src, int64_t ne, float * partials) { + float amax = 0.0f; + for (int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i < ne; i += (int64_t) blockDim.x*gridDim.x) { + amax = fmaxf(amax, src(i)); + } + + __shared__ float shared[WARP_SIZE]; + amax = block_reduce<block_reduce_method::MAX, 256>(amax, shared); + if (threadIdx.x == 0) { + partials[blockIdx.x] = amax; + } +} + +static __global__ void fp8_amax_final(const float * partials, int n, float * amax) { + float value = 0.0f; + for (int i = threadIdx.x; i < n; i += blockDim.x) { + value = fmaxf(value, partials[i]); + } + + __shared__ float shared[WARP_SIZE]; + value = block_reduce<block_reduce_method::MAX, 256>(value, shared); + if (threadIdx.x == 0) { + *amax = value; + } +} + +static __global__ void quantize_fp8_e4m3( + const float * __restrict__ x, uint8_t * __restrict__ y, const float * __restrict__ amax, + float * __restrict__ scale, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne, int64_t s1, int64_t s2, int64_t s3) { + const int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; + if (i >= ne) { + return; + } + + int64_t tmp = i / ne0; + const int64_t i0 = i - tmp*ne0; + const int64_t i1 = tmp % ne1; + tmp /= ne1; + const int64_t i2 = tmp % ne2; + const int64_t i3 = tmp / ne2; + + const float d = *amax > 0.0f ? *amax / 448.0f : 1.0f; + y[i] = ggml_cuda_fp32_to_f8_e4m3(x[i0 + i1*s1 + i2*s2 + i3*s3] / d); + if (i == 0) { + *scale = d; + } +} + +static void fp8_set_batch(cublasLtMatrixLayout_t desc, int32_t count, int64_t stride) { + CUBLAS_CHECK(cublasLtMatrixLayoutSetAttribute(desc, CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, &count, sizeof(count))); + CUBLAS_CHECK(cublasLtMatrixLayoutSetAttribute(desc, CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, &stride, sizeof(stride))); +} + +static void fp8_destroy_matmul( + cublasLtMatmulDesc_t op_desc, cublasLtMatrixLayout_t a_desc, cublasLtMatrixLayout_t b_desc, + cublasLtMatrixLayout_t d_desc) { + CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(d_desc)); + CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(b_desc)); + CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(a_desc)); + CUBLAS_CHECK(cublasLtMatmulDescDestroy(op_desc)); +} + +bool ggml_cuda_mul_mat_fp8( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const ggml_tensor * weight_scale) { + const int cc = ggml_cuda_info().devices[ctx.device].cc; + if (!fp8_mma_hardware_available(cc) || src0->type != GGML_TYPE_F8_E4M3 || src1->type != GGML_TYPE_F32 || + dst->type != GGML_TYPE_F32 || !ggml_is_contiguous(dst) || src0->ne[0] % 16 != 0 || src0->ne[1] % 16 != 0 || + src0->nb[0] != sizeof(uint8_t) || src0->nb[1] != (size_t) src0->ne[0] || src1->nb[0] != sizeof(float) || + !ggml_cuda_is_aligned(src0, 16) || !ggml_cuda_is_aligned(dst, 16)) { + return false; + } + + GGML_ASSERT(!weight_scale || (weight_scale->type == GGML_TYPE_F32 && ggml_is_contiguous(weight_scale) && ggml_nelements(weight_scale) == 1)); + + GGML_TENSOR_BINARY_OP_LOCALS + GGML_ASSERT(ne10 == ne00); + GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne12 % ne02 == 0); + GGML_ASSERT(ne13 % ne03 == 0); + + cudaStream_t stream = ctx.stream(); + const int64_t ne_src1 = ggml_nelements(src1); + ggml_cuda_pool_alloc<uint8_t> src1_fp8(ctx.pool(), ne_src1); + ggml_cuda_pool_alloc<float> src1_scale(ctx.pool(), 1); + ggml_cuda_pool_alloc<float> src1_amax(ctx.pool(), 1); + + const fp8_abs_src abs_src = { + (const float *) src1->data, ne10, ne11, ne12, + (int64_t) (nb11 / sizeof(float)), (int64_t) (nb12 / sizeof(float)), (int64_t) (nb13 / sizeof(float)) + }; + const int reduce_blocks = std::min<int64_t>((ne_src1 + 255)/256, 1024); + ggml_cuda_pool_alloc<float> reduce_tmp(ctx.pool(), reduce_blocks); + fp8_amax_partials<<<reduce_blocks, 256, 0, stream>>>(abs_src, ne_src1, reduce_tmp.ptr); + fp8_amax_final<<<1, 256, 0, stream>>>(reduce_tmp.ptr, reduce_blocks, src1_amax.ptr); + + quantize_fp8_e4m3<<<(ne_src1 + 255)/256, 256, 0, stream>>>( + (const float *) src1->data, src1_fp8.ptr, src1_amax.ptr, src1_scale.ptr, ne10, ne11, ne12, ne_src1, + nb11 / sizeof(float), nb12 / sizeof(float), nb13 / sizeof(float)); + CUDA_CHECK(cudaGetLastError()); + + cublasLtMatmulDesc_t op_desc; + cublasLtMatrixLayout_t a_desc; + cublasLtMatrixLayout_t b_desc; + cublasLtMatrixLayout_t d_desc; + CUBLAS_CHECK(cublasLtMatmulDescCreate(&op_desc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); + const cublasOperation_t trans_a = CUBLAS_OP_T; + CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(op_desc, CUBLASLT_MATMUL_DESC_TRANSA, &trans_a, sizeof(trans_a))); + CUBLAS_CHECK(cublasLtMatmulDescSetAttribute( + op_desc, CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, &src1_scale.ptr, sizeof(src1_scale.ptr))); + if (weight_scale) { + const float * scale_ptr = (const float *) weight_scale->data; + CUBLAS_CHECK(cublasLtMatmulDescSetAttribute( + op_desc, CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, &scale_ptr, sizeof(scale_ptr))); + } + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // src1 batches that share a src0 matrix are adjacent in the compact src1 copy and in dst, fold them into N + const int64_t n = ne11*(ne02 == 1 ? r2*r3 : r2); + + // batch the remaining matmuls with a fixed src0 stride, or issue one batched call per i3 if there is none + int64_t batch_count = ne02; + int64_t stride_a = nb02; + int64_t n_calls = ne13; + if (ne02 == 1) { + batch_count = ne03; + stride_a = nb03; + n_calls = 1; + } else if (r3 == 1 && (ne03 == 1 || nb03 == ne02*nb02)) { + batch_count = ne02*ne03; + n_calls = 1; + } + + CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&a_desc, CUDA_R_8F_E4M3, ne00, ne01, ne00)); + CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&b_desc, CUDA_R_8F_E4M3, ne10, n, ne10)); + CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&d_desc, CUDA_R_32F, ne0, n, ne0)); + if (batch_count > 1) { + fp8_set_batch(a_desc, batch_count, stride_a); + fp8_set_batch(b_desc, batch_count, ne10*n); + fp8_set_batch(d_desc, batch_count, ne0*n); + } + + // TODO: Allow workspace in the heuristic search for potential performance gains. + // cuBLASLt takes caller-owned workspace per matmul call. + cublasLtMatmulPreference_t preference; + CUBLAS_CHECK(cublasLtMatmulPreferenceCreate(&preference)); + cublasLtMatmulHeuristicResult_t heuristic; + int returned = 0; + const cublasStatus_t status = cublasLtMatmulAlgoGetHeuristic( + ctx.cublaslt_handle(), op_desc, a_desc, b_desc, d_desc, d_desc, preference, 1, &heuristic, &returned); + CUBLAS_CHECK(cublasLtMatmulPreferenceDestroy(preference)); + if (status != CUBLAS_STATUS_SUCCESS || returned == 0) { + fp8_destroy_matmul(op_desc, a_desc, b_desc, d_desc); + return false; + } + + const float alpha = 1.0f; + const float beta = 0.0f; + for (int64_t i3 = 0; i3 < n_calls; ++i3) { + const char * a = (const char *) src0->data + (i3/r3)*nb03; + const uint8_t * b = src1_fp8.ptr + i3*ne12*ne11*ne10; + float * d = (float *) ((char *) dst->data + i3*dst->nb[3]); + CUBLAS_CHECK(cublasLtMatmul(ctx.cublaslt_handle(), op_desc, &alpha, a, a_desc, b, b_desc, + &beta, d, d_desc, d, d_desc, &heuristic.algo, nullptr, 0, stream)); + } + + fp8_destroy_matmul(op_desc, a_desc, b_desc, d_desc); + return true; +} + +#else + +bool ggml_cuda_mul_mat_fp8( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const ggml_tensor * weight_scale) { + GGML_UNUSED_VARS(ctx, src0, src1, dst, weight_scale); + return false; +} + +#endif diff --git a/ggml/src/ggml-cuda/fp8.cuh b/ggml/src/ggml-cuda/fp8.cuh new file mode 100644 index 000000000..069349ed9 --- /dev/null +++ b/ggml/src/ggml-cuda/fp8.cuh @@ -0,0 +1,7 @@ +#pragma once + +#include "common.cuh" + +bool ggml_cuda_mul_mat_fp8( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const ggml_tensor * weight_scale = nullptr); diff --git a/ggml/src/ggml-cuda/fwht.cu b/ggml/src/ggml-cuda/fwht.cu index 67eea594d..78768eace 100644 --- a/ggml/src/ggml-cuda/fwht.cu +++ b/ggml/src/ggml-cuda/fwht.cu @@ -2,6 +2,9 @@ #include "convert.cuh" #include "fwht.cuh" +// wide FWHT blocks use one row per thread block with this many threads +#define GGML_CUDA_FWHT_BLOCK_NT 256 + template <int N, typename T> __launch_bounds__(4*ggml_cuda_get_physical_warp_size(), 1) __global__ void fwht_cuda(const T * src, float * dst, const int64_t n_rows, const float scale) { @@ -59,6 +62,87 @@ __global__ void fwht_cuda(const T * src, float * dst, const int64_t n_rows, cons } } +// Wide blocks: one row per thread block instead of per warp, so each thread keeps N/NT +// values rather than N/32. Stages below the warp width still shuffle, those up to the +// block width go through shared memory, and the rest stay in registers. +template <int N, int NT, typename T> +__launch_bounds__(NT, 1) +__global__ void fwht_cuda_block(const T * src, float * dst, const int64_t n_rows, const float scale) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int NE = N / NT; + static_assert(NE >= 1 && N % NT == 0 && NT % warp_size == 0, "bad FWHT block shape"); + + __shared__ float s[N]; + + const int64_t r = blockIdx.x; + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + const int tid = threadIdx.x; + const int lane = tid % warp_size; + + ggml_cuda_pdl_sync(); + + float reg[NE]; +#pragma unroll + for (int i = 0; i < NE; ++i) { + reg[i] = ggml_cuda_cast<float>(src[i * NT + tid]) * scale; + } + + // stages within a warp: partner differs in the lane bits +#pragma unroll + for (int h = 1; h < warp_size; h *= 2) { +#pragma unroll + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = __shfl_xor_sync(0xFFFFFFFF, val, h, warp_size); + reg[j] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + + // stages across warps: partner differs in the thread-index bits above the lane +#pragma unroll + for (int h = warp_size; h < NT; h *= 2) { +#pragma unroll + for (int j = 0; j < NE; j++) { + s[j * NT + tid] = reg[j]; + } + __syncthreads(); +#pragma unroll + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = s[j * NT + (tid ^ h)]; + reg[j] = (tid & h) == 0 ? val + val2 : val2 - val; + } + __syncthreads(); + } + + // stages above the block width: partner is another register of the same thread +#pragma unroll + for (int h = NT; h < N; h *= 2) { + const int step = h / NT; +#pragma unroll + for (int j = 0; j < NE; j += 2 * step) { +#pragma unroll + for (int k = 0; k < step; k++) { + const float x = reg[j + k]; + const float y = reg[j + k + step]; + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + +#pragma unroll + for (int i = 0; i < NE; ++i) { + dst[i * NT + tid] = reg[i]; + } +} + template <typename T> static bool ggml_cuda_op_fwht_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { const int n = src->ne[0]; @@ -94,7 +178,37 @@ static bool ggml_cuda_op_fwht_impl(ggml_backend_cuda_context & ctx, const ggml_t ggml_cuda_kernel_launch(fwht_cuda<512, T>, launch_params, src_d, dst_d, rows, scale); return true; default: - return false; + break; + } + + // wide blocks: one row per thread block + { + constexpr int nt = GGML_CUDA_FWHT_BLOCK_NT; + + dim3 grid_dims_w(rows, 1, 1); + dim3 block_dims_w(nt, 1, 1); + const ggml_cuda_kernel_launch_params launch_params_w = + ggml_cuda_kernel_launch_params(grid_dims_w, block_dims_w, 0, stream); + + switch (n) { + case 1024: + ggml_cuda_kernel_launch(fwht_cuda_block<1024, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; + case 2048: + ggml_cuda_kernel_launch(fwht_cuda_block<2048, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; + case 4096: + ggml_cuda_kernel_launch(fwht_cuda_block<4096, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; +#if !defined(GGML_USE_MUSA) + // 32 KB of shared memory, above the MUSA limit; falls back there + case 8192: + ggml_cuda_kernel_launch(fwht_cuda_block<8192, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; +#endif // !defined(GGML_USE_MUSA) + default: + return false; + } } } diff --git a/ggml/src/ggml-cuda/gated-delta-net-mma.cu b/ggml/src/ggml-cuda/gated-delta-net-mma.cu new file mode 100644 index 000000000..0684adc1a --- /dev/null +++ b/ggml/src/ggml-cuda/gated-delta-net-mma.cu @@ -0,0 +1,443 @@ +#include "gated-delta-net-mma.cuh" +#include "mma.cuh" + +#include <climits> +#include <mutex> + +namespace ggml_cuda_gated_delta_net_mma { +constexpr int D = 128; +using bf16 = nv_bfloat16; + +template <int R, int C> struct matrix { + bf16 hi[R * C], lo[R * C]; + + static __device__ __forceinline__ int index(int r, int c) { +#ifdef GGML_USE_HIP + return r * C + c; +#else + return 2 * ggml_cuda_mma::swizzle<C / 2, nv_bfloat162>(r * (C / 2) + c / 2, r) + c % 2; +#endif // GGML_USE_HIP + } + + __device__ __forceinline__ void store2(int r, int c, float2 x) { +#ifdef GGML_USE_MUSA + store(r, c, x.x); + store(r, c + 1, x.y); +#else + const int i = index(r, c); + const auto h = __float22bfloat162_rn(x); + const float2 rounded = __bfloat1622float2(h); + *reinterpret_cast<nv_bfloat162 *>(hi + i) = h; + *reinterpret_cast<nv_bfloat162 *>(lo + i) = + __float22bfloat162_rn(make_float2(x.x - rounded.x, x.y - rounded.y)); +#endif // GGML_USE_MUSA + } + + __device__ __forceinline__ void store(int r, int c, float x) { + const int i = index(r, c); + const bf16 h = __float2bfloat16(x); + hi[i] = h; + lo[i] = __float2bfloat16(x - __bfloat162float(h)); + } +}; + +template <int C, int V> struct alignas(128) shared { + matrix<C, D> q, k; + matrix<C, C> p, inverse; + matrix<C, V> delta; + + union { + matrix<D, V> state; + matrix<C, V> solved; + } scratch; + + float lower[C * C], prefix[C], beta[C]; + + static __device__ __forceinline__ int lower_index(int r, int c) { return r * C + c; } +}; + +#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) +using namespace ggml_cuda_mma; +#ifdef GGML_USE_HIP +using acc = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>; +using a_tile = tile<16, 8, nv_bfloat162, DATA_LAYOUT_I_MAJOR_MIRRORED>; +using b_tile = a_tile; +#else +using acc = tile<16, 8, float>; +using a_tile = tile<16, 8, nv_bfloat162>; +using b_tile = tile<8, 8, nv_bfloat162>; +#endif // GGML_USE_HIP + +template <class tile_t, bool TRANS, int R, int C> +__device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col, bool low) { + tile_t t; + const bf16 * data = low ? m.lo : m.hi; +#ifdef GGML_USE_HIP +#pragma unroll + for (int i = 0; i < tile_t::ne; ++i) { + const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i); + const bf16 x = data[TRANS ? m.index(c, r) : m.index(r, c)]; + const bf16 y = data[TRANS ? m.index(c + 1, r) : m.index(r, c + 1)]; + t.x[i] = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y))); + } +#else + const auto * packed = reinterpret_cast<const nv_bfloat162 *>(data); + if constexpr (TRANS) { + load_ldmatrix_trans_swizzled<C / 2>(t, packed, col * (C / 2) + row / 2); + } else { + load_ldmatrix_swizzled<C / 2>(t, packed, row * (C / 2) + col / 2); + } +#endif // GGML_USE_HIP + return t; +} + +// B is addressed as B-transpose. Preserve both BF16 first-order residual terms. +template <int K, bool TA = false, bool TB = false, int AR, int AC, int BR, int BC> +__device__ __forceinline__ void product(acc & c, const matrix<AR, AC> & a, const matrix<BR, BC> & b, int row, int col) { +#pragma unroll + for (int k = 0; k < K; k += 16) { + const auto ah = load<a_tile, TA>(a, row, k, false); + const auto al = load<a_tile, TA>(a, row, k, true); + const auto bh = load<b_tile, TB>(b, col, k, false); + const auto bl = load<b_tile, TB>(b, col, k, true); + mma(c, al, bh); + mma(c, ah, bl); + mma(c, ah, bh); + } +} + +template <int R, int C> __device__ __forceinline__ void store(matrix<R, C> & m, const acc & x, int r, int c) { +#pragma unroll +#ifdef GGML_USE_HIP + for (int i = 0; i < acc::ne; ++i) { + m.store(r + acc::get_i(i), c + acc::get_j(i), x.x[i]); + } +#else + for (int i = 0; i < acc::ne; i += 2) { + m.store2(r + acc::get_i(i), c + acc::get_j(i), make_float2(x.x[i], x.x[i + 1])); + } +#endif // GGML_USE_HIP +} +#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) + +// Each block owns V independent value columns and advances by C tokens. +// Keep recurrent state in FP32; pack high/residual BF16 operands inside the block. +template <int C, int V, int BLOCKS = 1> +static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_mma_args a) { +#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) + static_assert(ggml_cuda_get_physical_warp_size() == 32, "GDN MMA requires 32-lane warps"); + constexpr int WARPS = 8; +#ifdef GGML_USE_HIP + constexpr int N = 16; +#else + constexpr int N = 8; +#endif // GGML_USE_HIP + static_assert(C >= 16 && (C & (C - 1)) == 0 && V % N == 0 && D % V == 0, "invalid GDN tile"); + extern __shared__ __align__(128) unsigned char bytes[]; + auto & s = *reinterpret_cast<shared<C, V> *>(bytes); + const int lane = threadIdx.x, warp = threadIdx.y, tid = warp * 32 + lane; + const int slice = blockIdx.x % (D / V); + const int h = (blockIdx.x / (D / V)) % a.H; + const int seq = blockIdx.x / ((D / V) * a.H); + const int v0 = slice * V; + const int64_t soff = ((int64_t) seq * a.H + h) * D * D; + const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1; + const int64_t voff = seq * a.sv3 + h * a.sv1 + v0; + const int64_t goff = seq * a.sb3 + h * a.sb1; + float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0; + constexpr int STATE_TILES = (D / 16) * (V / N); + constexpr int PER_WARP = STATE_TILES / WARPS; + static_assert(STATE_TILES % WARPS == 0, "state must divide across warps"); + acc state[PER_WARP]; +#pragma unroll + for (int j = 0; j < PER_WARP; ++j) { + const int tile_index = warp + j * WARPS, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N; +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + state[j].x[i] = a.state[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)]; + } + } + auto read_qk = [&](int64_t t, float4 & q, float4 & k) { + t += warp; + q = k = make_float4(0, 0, 0, 0); + if (t < a.n_tokens) { + const int64_t off = qoff + t * a.sq2 + 4 * lane; + if ((((uintptr_t) (a.q + off) | (uintptr_t) (a.k + off)) & 15) == 0) { + q = *reinterpret_cast<const float4 *>(a.q + off); + k = *reinterpret_cast<const float4 *>(a.k + off); + } else { + q = make_float4(a.q[off], a.q[off + 1], a.q[off + 2], a.q[off + 3]); + k = make_float4(a.k[off], a.k[off + 1], a.k[off + 2], a.k[off + 3]); + } + } + }; + // Carry the first eight input rows across chunks to hide global-load latency. + constexpr bool PREFETCH = C == 16 && V == 32; + float4 next_q, next_k; + if constexpr (PREFETCH) { + read_qk(0, next_q, next_k); + } + for (int64_t t0 = 0; t0 < a.n_tokens; t0 += C) { + const int valid = min((int64_t) C, a.n_tokens - t0); +#pragma unroll + for (int j = 0; j < C / 8; ++j) { + const int t = warp + 8 * j, d = 4 * lane; + float4 q, k; + if (PREFETCH && j == 0) { + q = next_q; + k = next_k; + } else { + read_qk(t0 + 8 * j, q, k); + } + s.q.store2(t, d, make_float2(q.x * a.scale, q.y * a.scale)); + s.q.store2(t, d + 2, make_float2(q.z * a.scale, q.w * a.scale)); + s.k.store2(t, d, make_float2(k.x, k.y)); + s.k.store2(t, d + 2, make_float2(k.z, k.w)); + } + // Load gates in parallel before the FP32 prefix scan. + if (warp == 0) { + float carry = 0.f; + for (int base = 0; base < C; base += 32) { + const int t = base + lane; + float g = t < valid ? a.g[goff + (t0 + t) * a.sb2] : 0.f; + const float beta = t < valid ? a.beta[goff + (t0 + t) * a.sb2] : 0.f; +#pragma unroll + for (int offset = 1; offset < 32; offset *= 2) { + const float previous = __shfl_up_sync(0xffffffff, g, offset, 32); + if (lane >= offset) { + g += previous; + } + } + g += carry; + if (t < C) { + s.prefix[t] = g; + s.beta[t] = beta; + } + carry = __shfl_sync(0xffffffff, g, 31, 32); + } + } +#pragma unroll + for (int j = 0; j < PER_WARP; ++j) { + const int ti = warp + j * WARPS; + store(s.scratch.state, state[j], (ti / (V / N)) * 16, (ti % (V / N)) * N); + } + __syncthreads(); + if constexpr (PREFETCH) { + if (t0 + C < a.n_tokens) { + read_qk(t0 + C, next_q, next_k); + } + } + // V32 overlaps Gram work with one output tile per projection warp. + constexpr bool SPLIT_PROJECTION = C == 16 && V == 32; + acc base_output[((C / 16) * (V / N) + WARPS - 1) / WARPS]; + // delta = beta * (v - decay * K S), base output = decay * Q S + auto project = [&]() { + if constexpr (V == 32) { + if (warp < 4) { + return; + } + } + for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : WARPS)) { + const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; + acc qs, ks; + product<D, false, true>(qs, s.q, s.scratch.state, r, c); + product<D, false, true>(ks, s.k, s.scratch.state, r, c); +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + const int t = r + acc::get_i(i), v = c + acc::get_j(i); + const float decay = expf(s.prefix[t]); + const float vv = t < valid ? a.v[voff + (t0 + t) * a.sv2 + v] : 0.f; + s.delta.store(t, v, s.beta[t] * (vv - decay * ks.x[i])); + base_output[ti / WARPS].x[i] = decay * qs.x[i]; + } + } + }; + // A = beta * strict_tril(K K^T * decay), P = tril(Q K^T * decay) + for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : WARPS)) { + const bool gram = ti < (C / 16) * (C / N); + const int t = ti % ((C / 16) * (C / N)), r = (t / (C / N)) * 16, c = (t % (C / N)) * N; + acc x; + if (gram) { + product<D>(x, s.k, s.k, r, c); + } else { + product<D>(x, s.q, s.k, r, c); + } +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + const int rr = r + acc::get_i(i), cc = c + acc::get_j(i); + const float v = rr < valid && cc <= rr ? x.x[i] * expf(s.prefix[rr] - s.prefix[cc]) : 0.f; + if (gram) { + if (cc < rr) { + s.lower[s.lower_index(rr, cc)] = v * s.beta[rr]; + } + } else { + s.p.store(rr, cc, v); + } + } + } + if constexpr (V == 32) { + project(); + } + __syncthreads(); + // inverse = (I + A)^-1 by forward substitution + for (int row = warp; row < C; row += WARPS) { + float x[(C + 31) / 32]; +#pragma unroll + for (int j = 0; j < (C + 31) / 32; ++j) { + const int col = lane + 32 * j; + x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f); + } + for (int pivot = row - 1; pivot > 0; --pivot) { + const float xp = __shfl_sync(0xffffffff, x[pivot / 32], pivot % 32, 32); +#pragma unroll + for (int j = 0; j < (C + 31) / 32; ++j) { + const int col = lane + 32 * j; + if (col < pivot) { + x[j] -= xp * s.lower[s.lower_index(pivot, col)]; + } + } + } +#pragma unroll + for (int j = 0; j < (C + 31) / 32; ++j) { + if (lane + 32 * j < C) { + s.inverse.store(row, lane + 32 * j, x[j]); + } + } + } + if constexpr (V != 32) { + project(); + } + __syncthreads(); + // solved = inverse * delta + for (int ti = warp; ti < (C / 16) * (V / N); ti += WARPS) { + const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; + acc x; + product<C, false, true>(x, s.inverse, s.delta, r, c); + store(s.scratch.solved, x, r, c); + } + __syncthreads(); + // out = base output + P * solved + for (int ti = (SPLIT_PROJECTION ? warp - 4 : warp); ti < (C / 16) * (V / N) && (!SPLIT_PROJECTION || warp >= 4); + ti += WARPS) { + const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; + acc x; + product<C, false, true>(x, s.p, s.scratch.solved, r, c); +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + const int t = r + acc::get_i(i), v = c + acc::get_j(i); + if (t < valid) { + out[(t0 + t) * a.H * D + v] = base_output[ti / WARPS].x[i] + x.x[i]; + } + } + } + // S = decay(chunk) * S + K^T * (solved * decay to chunk end) + for (int i = tid; i < C * V; i += 256) { + const int t = i / V, v = i % V, j = s.scratch.solved.index(t, v); + const float x = __bfloat162float(s.scratch.solved.hi[j]) + __bfloat162float(s.scratch.solved.lo[j]); + s.delta.store(t, v, x * expf(s.prefix[valid - 1] - s.prefix[t])); + } + __syncthreads(); + const float decay = expf(s.prefix[valid - 1]); +#pragma unroll + for (int j = 0; j < PER_WARP; ++j) { + const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + state[j].x[i] *= decay; + } + product<C, true, true>(state[j], s.k, s.delta, r, c); + } + __syncthreads(); + } +#pragma unroll + for (int j = 0; j < PER_WARP; ++j) { + const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + a.state_out[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)] = state[j].x[i]; + } + } +#else + GGML_UNUSED_VARS(a); + NO_DEVICE_CODE; +#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) +} + +template <int C, int V, int BLOCKS = 1> bool init(int device) { + static std::once_flag once[GGML_CUDA_MAX_DEVICES]; + static bool available[GGML_CUDA_MAX_DEVICES] = {}; + std::call_once(once[device], [device] { + ggml_cuda_set_device(device); + if (sizeof(shared<C, V>) > ggml_cuda_info().devices[device].smpbo) { + return; + } + const auto status = cudaFuncSetAttribute((const void *) gdn_single<C, V, BLOCKS>, + cudaFuncAttributeMaxDynamicSharedMemorySize, sizeof(shared<C, V>)); + available[device] = status == cudaSuccess; + if (status != cudaSuccess) { + (void) cudaGetLastError(); + } + }); + return available[device]; +} + +enum class gdn_path { ar, slice32_one_block, slice32_two_blocks, slice64, whole_head }; + +static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a) { + GGML_ASSERT(device >= 0 && device < GGML_CUDA_MAX_DEVICES); + if (!a.eligible || a.H_k != 16 || (a.H != 16 && a.H != 32 && a.H != 48 && a.H != 64) || a.n_tokens < 64 || + a.n_tokens > INT64_MAX - 32 || a.rq3 <= 0 || a.n_seqs <= 0 || a.n_seqs > INT_MAX / (a.H * 4)) { + return gdn_path::ar; + } + const auto & info = ggml_cuda_info().devices[device]; +#ifdef GGML_USE_HIP + if (!GGML_CUDA_CC_IS_RDNA3_5(info.cc) || a.n_tokens < 2048) { + return gdn_path::ar; + } + return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar; +#elif defined(GGML_USE_MUSA) + GGML_UNUSED_VARS(info); + return gdn_path::ar; +#else + // Enable only architectures benchmarked for this schedule. MMA support alone does not imply a speedup. + if (!ampere_mma_available(info.cc) || + (info.cc != GGML_CUDA_CC_AMPERE_GA10X && info.cc != GGML_CUDA_CC_ADA_LOVELACE && + info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_DGX_SPARK)) { + return gdn_path::ar; + } + // Value splitting pays off when whole heads leave at least half the SMs idle. + if (a.H * a.n_seqs * 2 <= info.nsm) { + // A grid that already fits in one wave benefits from a larger register budget. + if (a.H * a.n_seqs * 4 <= info.nsm) { + return init<16, 32, 1>(device) ? gdn_path::slice32_one_block : gdn_path::ar; + } + return init<16, 32, 2>(device) ? gdn_path::slice32_two_blocks : gdn_path::ar; + } + return init<16, 128>(device) ? gdn_path::whole_head : gdn_path::ar; +#endif // GGML_USE_HIP +} +} // namespace +using namespace ggml_cuda_gated_delta_net_mma; + +bool ggml_cuda_gdn_mma_launch(int device, const ggml_cuda_gdn_mma_args & a, cudaStream_t stream) { + if (!a.state_out) { + return false; + } + const auto path = select_gdn_mma_path(device, a); + if (path == gdn_path::ar) { + return false; + } +#ifdef GGML_USE_HIP + gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(32, 8), sizeof(shared<16, 64>), stream>>>(a); +#else + if (path == gdn_path::slice32_two_blocks) { + gdn_single<16, 32, 2><<<a.H * a.n_seqs * 4, dim3(32, 8), sizeof(shared<16, 32>), stream>>>(a); + } else if (path == gdn_path::slice32_one_block) { + gdn_single<16, 32, 1><<<a.H * a.n_seqs * 4, dim3(32, 8), sizeof(shared<16, 32>), stream>>>(a); + } else { + gdn_single<16, 128><<<a.H * a.n_seqs, dim3(32, 8), sizeof(shared<16, 128>), stream>>>(a); + } +#endif // GGML_USE_HIP + CUDA_CHECK(cudaGetLastError()); + return true; +} diff --git a/ggml/src/ggml-cuda/gated-delta-net-mma.cuh b/ggml/src/ggml-cuda/gated-delta-net-mma.cuh new file mode 100644 index 000000000..8eb7c4f33 --- /dev/null +++ b/ggml/src/ggml-cuda/gated-delta-net-mma.cuh @@ -0,0 +1,35 @@ +#pragma once + +#include "common.cuh" + +#include <cstdint> + +struct ggml_cuda_gdn_mma_args { + const float * q; + const float * k; + const float * v; + const float * g; + const float * beta; + const float * state; + float * dst; + float * state_out; + + int64_t H; + int64_t H_k; + int64_t n_tokens; + int64_t n_seqs; + int64_t rq3; + int64_t sq1; + int64_t sq2; + int64_t sq3; + int64_t sv1; + int64_t sv2; + int64_t sv3; + int64_t sb1; + int64_t sb2; + int64_t sb3; + float scale; + bool eligible; +}; + +bool ggml_cuda_gdn_mma_launch(int device, const ggml_cuda_gdn_mma_args & args, cudaStream_t stream); diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index 1b431a724..ae2374cbc 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -1,7 +1,12 @@ #include "gated_delta_net.cuh" +#include "gated-delta-net-mma.cuh" #include "ggml-cuda/common.cuh" -template <int S_v, bool KDA, bool keep_rs_t> +#include <cstdint> + +constexpr int gdn_cols_per_warp = 4; + +template <int S_v, bool KDA, bool keep_rs_t, int cols_per_warp = gdn_cols_per_warp> __global__ void __launch_bounds__((ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v) * 4, 2) gated_delta_net_cuda(const float * q, const float * k, @@ -30,9 +35,19 @@ gated_delta_net_cuda(const float * q, int K) { const uint32_t h_idx = blockIdx.x; const uint32_t sequence = blockIdx.y; - // each warp owns one column, using warp-level primitives to reduce across rows - const int lane = threadIdx.x; - const int col = blockIdx.z * blockDim.y + threadIdx.y; + + constexpr int warp_size = ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v; + static_assert(S_v % warp_size == 0, "S_v must be a multiple of warp_size"); + // the warp is split into cols_per_warp segments of lanes_per_col lanes; each segment owns + // one state column and reduces within itself + constexpr int lanes_per_col = warp_size / cols_per_warp; + constexpr int rows_per_lane = S_v / lanes_per_col; + static_assert(S_v % lanes_per_col == 0, "S_v must be a multiple of lanes_per_col"); + + const int lane = threadIdx.x; + const int col_in_warp = lane / lanes_per_col; // column slot within the warp + const int lane_in_col = lane - col_in_warp * lanes_per_col; // lane within the column's reduction segment + const int col = (blockIdx.z * blockDim.y + threadIdx.y) * cols_per_warp + col_in_warp; const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); const uint32_t iq3 = fastdiv(sequence, rq3_magic); @@ -47,16 +62,13 @@ gated_delta_net_cuda(const float * q, curr_state += state_in_offset + col * S_v; attn_data += (sequence * n_tokens * H + h_idx) * S_v; - constexpr int warp_size = ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v; - static_assert(S_v % warp_size == 0, "S_v must be a multiple of warp_size"); - constexpr int rows_per_lane = (S_v + warp_size - 1) / warp_size; float s_shard[rows_per_lane]; // state is stored transposed: M[col][i] = S[i][col], row col is contiguous ggml_cuda_pdl_sync(); #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; s_shard[r] = curr_state[i]; } @@ -76,7 +88,7 @@ gated_delta_net_cuda(const float * q, float q_reg[rows_per_lane]; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; k_reg[r] = k_t[i]; q_reg[r] = q_t[i]; } @@ -90,7 +102,7 @@ gated_delta_net_cuda(const float * q, for (int r = 0; r < rows_per_lane; r++) { kv_shard += s_shard[r] * k_reg[r]; } - float kv_col = warp_reduce_sum<warp_size>(kv_shard); + float kv_col = warp_reduce_sum<lanes_per_col>(kv_shard); // delta[col] = (v[col] - g * kv[col]) * beta float delta_col = (v_t[col] - g_val * kv_col) * beta_val; @@ -104,9 +116,9 @@ gated_delta_net_cuda(const float * q, attn_partial += s_shard[r] * q_reg[r]; } - float attn_col = warp_reduce_sum<warp_size>(attn_partial); + float attn_col = warp_reduce_sum<lanes_per_col>(attn_partial); - if (lane == 0) { + if (lane_in_col == 0) { attn_data[col] = attn_col * scale; } } else { @@ -114,11 +126,11 @@ gated_delta_net_cuda(const float * q, float kv_shard = 0.0f; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; kv_shard += expf(g_t[i]) * s_shard[r] * k_reg[r]; } - float kv_col = warp_reduce_sum<warp_size>(kv_shard); + float kv_col = warp_reduce_sum<lanes_per_col>(kv_shard); // delta[col] = (v[col] - kv[col]) * beta float delta_col = (v_t[col] - kv_col) * beta_val; @@ -128,14 +140,14 @@ gated_delta_net_cuda(const float * q, float attn_partial = 0.0f; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; s_shard[r] = expf(g_t[i]) * s_shard[r] + k_reg[r] * delta_col; attn_partial += s_shard[r] * q_reg[r]; } - float attn_col = warp_reduce_sum<warp_size>(attn_partial); + float attn_col = warp_reduce_sum<lanes_per_col>(attn_partial); - if (lane == 0) { + if (lane_in_col == 0) { attn_data[col] = attn_col * scale; } } @@ -150,7 +162,7 @@ gated_delta_net_cuda(const float * q, float * curr_state = state + target_slot * state_slot_stride; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; curr_state[col * S_v + i] = s_shard[r]; } } @@ -160,7 +172,7 @@ gated_delta_net_cuda(const float * q, if constexpr (!keep_rs_t) { #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; state[col * S_v + i] = s_shard[r]; } } @@ -179,8 +191,16 @@ static void launch_gated_delta_net( float scale, int64_t state_slot_stride, int K, cudaStream_t stream) { //TODO: Add chunked kernel for even faster pre-fill const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; - const int num_warps = 4; - dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps); + // four columns per warp (see the kernel); shrink the CTA when the wider CTA would leave + // SMs without a CTA, so small head counts keep the device filled + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + const int cols_per_warp = gdn_cols_per_warp; + int num_warps = 4; + while (num_warps > 1 && H*n_seqs*(S_v / (cols_per_warp * num_warps)) < nsm) { + num_warps /= 2; + } + // one CTA covers cols_per_warp*num_warps columns (see the kernel) + dim3 grid_dims(H, n_seqs, (S_v + cols_per_warp * num_warps - 1) / (cols_per_warp * num_warps)); dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1); const uint3 neqk1_magic = init_fastdiv_values(neqk1); @@ -216,10 +236,47 @@ static void launch_gated_delta_net( } default: GGML_ABORT("fatal error"); - break; } } +static ggml_cuda_gdn_mma_args ggml_cuda_gdn_mma_args_from_tensor(const ggml_tensor * dst) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * g = dst->src[3]; + const ggml_tensor * beta = dst->src[4]; + const ggml_tensor * state = dst->src[5]; + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + return { + (const float *) q->data, + (const float *) k->data, + (const float *) v->data, + (const float *) g->data, + (const float *) beta->data, + (const float *) state->data, + (float *) dst->data, + nullptr, + H, + q->ne[1], + v->ne[2], + v->ne[3], + v->ne[3] / q->ne[3], + (int64_t) (q->nb[1] / sizeof(float)), + (int64_t) (q->nb[2] / sizeof(float)), + (int64_t) (q->nb[3] / sizeof(float)), + (int64_t) (v->nb[1] / sizeof(float)), + (int64_t) (v->nb[2] / sizeof(float)), + (int64_t) (v->nb[3] / sizeof(float)), + (int64_t) (beta->nb[1] / sizeof(float)), + (int64_t) (beta->nb[2] / sizeof(float)), + (int64_t) (beta->nb[3] / sizeof(float)), + 1.0f / sqrtf((float) S_v), + g->ne[0] == 1 && ggml_get_op_params_i32(dst, 0) < v->ne[2] && S_v == 128 && q->ne[0] == 128, + }; +} + static void ggml_cuda_op_gated_delta_net_impl( ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_cuda_gated_delta_net_fused_cache * cache) { ggml_tensor * src_q = dst->src[0]; @@ -247,6 +304,8 @@ static void ggml_cuda_op_gated_delta_net_impl( GGML_ASSERT(neq1 == nek1); const int64_t neqk1 = neq1; + GGML_ASSERT(neq3 > 0 && nev3 >= neq3 && nev3 % neq3 == 0); + GGML_ASSERT(nek3 == neq3); const int64_t rq3 = nev3 / neq3; const float * q_d = (const float *) src_q->data; @@ -294,6 +353,13 @@ static void ggml_cuda_op_gated_delta_net_impl( state_slot_stride = cache->slot_stride; } + { + ggml_cuda_gdn_mma_args args = ggml_cuda_gdn_mma_args_from_tensor(dst); + args.state_out = state_d; + if (ggml_cuda_gdn_mma_launch(ctx.device, args, stream)) { + return; + } + } if (kda) { if (keep_rs) { launch_gated_delta_net<true, true>(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index 6b36b8fbc..09b7f02b8 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -316,6 +316,10 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_float((const nv_bfloat16 *) src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_F8_E4M3: + get_rows_cuda_float((const ggml_fp8_e4m3_t *) src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; case GGML_TYPE_Q1_0: get_rows_cuda_q<QK1_0, QR1_0, dequantize_q1_0>(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); @@ -406,7 +410,6 @@ static void ggml_cuda_get_rows_switch_src0_type( break; default: GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0_type)); - break; } } @@ -435,7 +438,6 @@ void get_rows_cuda( break; default: GGML_ABORT("%s: unsupported dst type: %s\n", __func__, ggml_type_name(dst_type)); - break; } } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index e068116f3..fed14ddfc 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4,11 +4,13 @@ #include "ggml-cuda/allreduce.cuh" #include "ggml-cuda/common.cuh" +#include "ggml-cuda/graph.cuh" #include "ggml-cuda/acc.cuh" #include "ggml-cuda/add-id.cuh" #include "ggml-cuda/arange.cuh" #include "ggml-cuda/argmax.cuh" #include "ggml-cuda/argsort.cuh" +#include "ggml-cuda/batch-get.cuh" #include "ggml-cuda/binbcast.cuh" #include "ggml-cuda/clamp.cuh" #include "ggml-cuda/col2im-1d.cuh" @@ -23,14 +25,15 @@ #include "ggml-cuda/cpy.cuh" #include "ggml-cuda/cross-entropy-loss.cuh" #include "ggml-cuda/cumsum.cuh" -#include "ggml-cuda/diagmask.cuh" #include "ggml-cuda/diag.cuh" #include "ggml-cuda/fattn.cuh" +#include "ggml-cuda/fp8.cuh" #include "ggml-cuda/fwht.cuh" #include "ggml-cuda/getrows.cuh" #include "ggml-cuda/im2col.cuh" #include "ggml-cuda/mmf.cuh" #include "ggml-cuda/mmq.cuh" +#include "ggml-cuda/mmq-few.cuh" #include "ggml-cuda/mmvf.cuh" #include "ggml-cuda/mmvq.cuh" #include "ggml-cuda/moe-weighted-reduction.cuh" @@ -70,6 +73,7 @@ #include "ggml-cuda/cumsum.cuh" #include "ggml-cuda/fill.cuh" #include "ggml-cuda/lightning-indexer.cuh" +#include "ggml-cuda/top-k-mask.cuh" #include "ggml.h" #include <algorithm> @@ -138,10 +142,59 @@ int ggml_cuda_get_device() { return id; } +static bool ggml_cuda_env_flag(const char * name) { + const char * env = getenv(name); + return env != nullptr && std::atoi(env) != 0; +} + +static int ggml_cuda_env_int(const char * name, int default_value) { + const char * env = getenv(name); + return env != nullptr ? std::atoi(env) : default_value; +} + +static bool ggml_cuda_env_set(const char * name) { + return getenv(name) != nullptr; +} + +static bool ggml_cuda_unified_memory_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_ENABLE_UNIFIED_MEMORY"); + return enabled; +} + +static bool ggml_cuda_pinned_memory_disabled() { + static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_NO_PINNED"); + return disabled; +} + +static bool ggml_cuda_register_host_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_REGISTER_HOST"); + return enabled; +} + +static bool ggml_cuda_fusion_disabled() { + static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_DISABLE_FUSION"); + return disabled; +} + +static bool ggml_cuda_p2p_enabled() { + static const bool enabled = ggml_cuda_env_set("GGML_CUDA_P2P"); + return enabled; +} + +bool ggml_cuda_pool_debug_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_POOL_DEBUG"); + return enabled; +} + +static int ggml_cuda_op_offload_min_batch() { + static const int min_batch = ggml_cuda_env_int("GGML_OP_OFFLOAD_MIN_BATCH", 32); + return min_batch; +} + static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) { ggml_cuda_set_device(device); cudaError_t err; - if (getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr) { + if (ggml_cuda_unified_memory_enabled()) { err = cudaMallocManaged(ptr, size); #if defined(GGML_USE_HIP) if (err == hipSuccess) { @@ -169,52 +222,36 @@ static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) } #if defined(GGML_USE_HIP) -static int ggml_cuda_parse_id(char devName[]) { +static int ggml_cuda_parse_id(const char * devName) { // A list of possible Target IDs can be found under the rocclr/clr repo in device.cpp // these values are not stable so this is susceptible to breakage // https://github.com/ROCm/clr/blob/amd-staging/rocclr/device/device.cpp - int archMajor = 0x0; - int archMinor = 0x0; - int archNum = GGML_CUDA_CC_OFFSET_AMD; - int archLen = strlen(devName); - char archName[archLen + 1]; + // Extract the base architecture model ("gfx908:sramecc+:xnack-" -> "908") + std::string archName = devName; + archName.erase(0, 3); + archName = archName.substr(0, archName.find(':')); - // strip leading 'gfx' while copying into our buffer - if (archLen > 3) { - strcpy(archName, &devName[3]); - archLen -= 3; - } + const auto hex = [](const std::string & s) { return (int) strtoul(s.c_str(), nullptr, 16); }; - // trim trailing :xnack- or :sramecc- statuses - archLen = strcspn(archName, ":"); - archName[archLen] = '\0'; + int cc = 0; // tease out the version information - if (archLen > 8) { + if (archName.size() > 8) { // versions labeled generic use '-' as delimiter - // strip the trailing "-generic" then iterate through what remains - if ((strstr(archName, "-generic"))) { - archName[archLen - 8] = '\0'; - char * pch; - if ((pch = strtok(archName, "-"))) { - archMajor = (int)strtoul(pch, 0, 16); - if ((pch = strtok(NULL, "-"))) { - archMinor = 0x10 * (int)strtoul(pch, 0, 16); - } + if (archName.compare(archName.size() - 8, 8, "-generic") == 0) { + archName.resize(archName.size() - 8); + const size_t sep = archName.find('-'); + cc = 0x100 * hex(archName.substr(0, sep)); + if (sep != std::string::npos) { + cc += 0x10 * hex(archName.substr(sep + 1)); } } - } else if (archLen >= 3) { - // last two digits should be the minor * 0x10 + stepping - archMinor = (int)strtoul(&archName[archLen - 2], 0, 16); - archName[archLen - 2] = '\0'; - - // only the major version remains - archMajor = (int)strtoul(archName, 0, 16); + } else if (archName.size() >= 3) { + // ordinary IDs are the version in hex + cc = hex(archName); } - archNum += archMajor * 0x100; - archNum += archMinor; - return archNum; + return GGML_CUDA_CC_OFFSET_AMD + cc; } #endif // defined(GGML_USE_HIP) @@ -383,7 +420,7 @@ static ggml_cuda_device_info ggml_cuda_init() { // configure logging to stdout // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr)); - if (getenv("GGML_CUDA_P2P") != nullptr) { + if (ggml_cuda_p2p_enabled()) { for (int id = 0; id < info.physical_device_count; ++id) { CUDA_CHECK(cudaSetDevice(id)); for (int id_other = 0; id_other < info.physical_device_count; ++id_other) { @@ -413,6 +450,7 @@ const ggml_cuda_device_info & ggml_cuda_info() { struct ggml_cuda_pool_leg : public ggml_cuda_pool { static const int MAX_BUFFERS = 256; + ggml_backend_cuda_context & ctx; int device; struct ggml_cuda_buffer { void * ptr = nullptr; @@ -422,7 +460,8 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {}; size_t pool_size = 0; - explicit ggml_cuda_pool_leg(int device) : + ggml_cuda_pool_leg(ggml_backend_cuda_context & ctx, int device) : + ctx(ctx), device(device) { } @@ -444,6 +483,20 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { } } + // captured CUDA graphs can still use freed buffers, so they must be captured again + void invalidate_cuda_graphs() { +#ifdef USE_CUDA_GRAPH + int n_captured = 0; + for (const auto & [key, graph] : ctx.cuda_graphs) { + n_captured += graph->instances.size(); + } + if (n_captured > 0) { + GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: memory freed, %d captured CUDA graphs must be captured again\n", device, n_captured); + } + ctx.cuda_graphs.clear(); +#endif // USE_CUDA_GRAPH + } + void * alloc(size_t size, size_t * actual_size) override { #ifdef DEBUG_CUDA_MALLOC int nnz = 0; @@ -494,6 +547,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { device, look_ahead_size/1024.0/1024.0, cached_bytes/1024.0/1024.0); CUDA_CHECK(cudaDeviceSynchronize()); clear_pool(); + invalidate_cuda_graphs(); err = ggml_cuda_device_malloc(&ptr, look_ahead_size, device); if (err == cudaSuccess) { GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: retry succeeded\n", device); @@ -522,6 +576,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { ggml_cuda_set_device(device); CUDA_CHECK(cudaFree(ptr)); pool_size -= size; + invalidate_cuda_graphs(); } }; @@ -535,6 +590,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { CUdeviceptr pool_addr = 0; size_t pool_used = 0; size_t pool_size = 0; + size_t max_size = CUDA_POOL_VMM_MAX_SIZE; size_t granularity; #if defined(GGML_USE_HIP) std::vector<std::pair<CUdeviceptr, size_t>> mappings; @@ -556,7 +612,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { #else CU_CHECK(cuMemUnmap(pool_addr, pool_size)); #endif - CU_CHECK(cuMemAddressFree(pool_addr, CUDA_POOL_VMM_MAX_SIZE)); + CU_CHECK(cuMemAddressFree(pool_addr, max_size)); } } @@ -572,7 +628,21 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { size_t reserve_size = size - avail; reserve_size = granularity * ((reserve_size + granularity - 1) / granularity); - GGML_ASSERT(pool_size + reserve_size <= CUDA_POOL_VMM_MAX_SIZE); + // reserve virtual address space (if not already reserved) + if (pool_addr == 0) { + CUresult err = cuMemAddressReserve(&pool_addr, max_size, 0, 0, 0); + while (err == CUDA_ERROR_OUT_OF_MEMORY && (max_size >> 1) >= reserve_size && (max_size >> 1) >= granularity) { + max_size >>= 1; + err = cuMemAddressReserve(&pool_addr, max_size, 0, 0, 0); + } + CU_CHECK(err); + if (max_size < CUDA_POOL_VMM_MAX_SIZE) { + GGML_LOG_WARN(GGML_CUDA_NAME " pool[%d]: reserved %.2f GiB of virtual memory instead of %.2f GiB\n", + device, max_size / (1024.0 * 1024.0 * 1024.0), CUDA_POOL_VMM_MAX_SIZE / (1024.0 * 1024.0 * 1024.0)); + } + } + + GGML_ASSERT(pool_size + reserve_size <= max_size); // allocate more physical memory CUmemAllocationProp prop = {}; @@ -582,11 +652,6 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { CUmemGenericAllocationHandle handle; CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0)); - // reserve virtual address space (if not already reserved) - if (pool_addr == 0) { - CU_CHECK(cuMemAddressReserve(&pool_addr, CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0)); - } - // map at the end of the pool CUdeviceptr start_ptr = (CUdeviceptr)((char *)(pool_addr) + pool_size); CU_CHECK(cuMemMap(start_ptr, reserve_size, 0, handle, 0)); @@ -598,7 +663,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { CU_CHECK(cuMemRelease(handle)); // VMM Bug fix for P2P access if GGML_CUDA_P2P is set, or if NCCL build - bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr; + bool use_peer_access = ggml_cuda_p2p_enabled(); #if defined(GGML_USE_NCCL) use_peer_access = true; #endif // defined(GGML_USE_NCCL) @@ -676,15 +741,168 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { }; #endif // defined(GGML_USE_VMM) -std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(int device, - [[maybe_unused]] int stream_no) { +// Debug pool for NVIDIA compute-sanitizer, e.g. +// `GGML_CUDA_POOL_DEBUG=1 compute-sanitizer --tool memcheck ./build/bin/test-backend-ops -b CUDA0` +// Allocations are exact-size and freed on release, +// allowing memcheck to catch out-of-bounds and use-after-free. +struct ggml_cuda_pool_debug : public ggml_cuda_pool { + int device; + size_t pool_size = 0; // leak check (error if not zero at destruction) + + explicit ggml_cuda_pool_debug(int device) : + device(device) { + } + + ~ggml_cuda_pool_debug() { + GGML_ASSERT(pool_size == 0); + } + + void * alloc(size_t size, size_t * actual_size) override { + void * ptr; + CUDA_CHECK(ggml_cuda_device_malloc(&ptr, size, device)); + *actual_size = size; + pool_size += size; + return ptr; + } + + void free(void * ptr, size_t size) override { + ggml_cuda_set_device(device); + CUDA_CHECK(cudaFree(ptr)); + pool_size -= size; + } +}; + +std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(ggml_backend_cuda_context & ctx, + int device, + [[maybe_unused]] int stream_no) { + if (ggml_cuda_pool_debug_enabled()) { + return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_debug(device)); + } #if defined(GGML_USE_VMM) if (ggml_cuda_info().devices[device].vmm) { return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_vmm(device)); } #endif // defined(GGML_USE_VMM) - return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(device)); + return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(ctx, device)); +} + +ggml_backend_cuda_context::ggml_backend_cuda_context(int device) : + device(device), + name(GGML_CUDA_NAME + std::to_string(device)), + concurrent_stream_context(std::make_unique<ggml_cuda_stream_context>()) { +} + +#ifdef USE_CUDA_GRAPH +ggml_cuda_graph * ggml_backend_cuda_context::cuda_graph(const void * first_node_ptr, int64_t time_now) { + // Give all cached graphs 10s to be reused after an idle period, without updating each graph. + if (!cuda_graphs.empty() && time_now - last_graph_use_time >= 10'000'000) { + next_graph_eviction_time = time_now + 10'000'000; + } + last_graph_use_time = time_now; + + // Sweep every 5s during active use, evicting graphs unused for at least 10s. + if (time_now >= next_graph_eviction_time) { + next_graph_eviction_time = time_now + 5'000'000; + for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) { + if (it->first != first_node_ptr && time_now - it->second->last_used_time >= 10'000'000) { + it = cuda_graphs.erase(it); + } else { + ++it; + } + } + } + + auto it = cuda_graphs.find(first_node_ptr); + if (it == cuda_graphs.end()) { + it = cuda_graphs.emplace(first_node_ptr, std::make_unique<ggml_cuda_graph>()).first; + } + it->second->last_used_time = time_now; + return it->second.get(); +} + +bool ggml_backend_cuda_context::any_cuda_graph_enabled() const { + for (const auto & [key, graph] : cuda_graphs) { + if (graph && graph->is_enabled()) { + return true; + } + } + return false; +} + +bool ggml_backend_cuda_context::any_cuda_graph_has_instance() const { + for (const auto & [key, graph] : cuda_graphs) { + if (graph && !graph->instances.empty()) { + return true; + } + } + return false; } +#endif // USE_CUDA_GRAPH + +#if defined(GGML_CUDA_USE_PDL) +bool ggml_cuda_kernel_can_use_pdl(const void * kernel) { + static const bool env_pdl_enabled = []() { + const char * env = getenv("GGML_CUDA_PDL"); + return env == nullptr || std::atoi(env) != 0; + }(); + if (!env_pdl_enabled) { + return false; + } + + const int device = ggml_cuda_get_device(); + + struct cache_key { + int device; + const void * kernel; + + bool operator==(const cache_key & other) const { return device == other.device && kernel == other.kernel; } + }; + + struct cache_key_hash { + // MurmurHash3 mixing function for better hash distribution (vs. just std::hash which in some implementations simply returns the identity) + static size_t hash_mix(size_t x) { + std::uint64_t y = x; + const std::uint64_t m = 0xe9846af9b1a615d; + + y ^= y >> 32; + y *= m; + y ^= y >> 32; + y *= m; + y ^= y >> 28; + + return static_cast<size_t>(y); + } + + size_t operator()(const cache_key & key) const { + // Use a nonzero seed to avoid mapping all-zero keys to zero + size_t h = 42; + h = hash_mix(h + key.device); + h = hash_mix(h + reinterpret_cast<size_t>(key.kernel)); + return h; + } + }; + + static std::mutex cache_mutex; + static std::unordered_map<cache_key, bool, cache_key_hash> cache; + + const cache_key key = { device, kernel }; + std::lock_guard<std::mutex> lock(cache_mutex); + const auto it = cache.find(key); + if (it != cache.end()) { + return it->second; + } + + cudaFuncAttributes attr = {}; + CUDA_CHECK(cudaFuncGetAttributes(&attr, kernel)); + + // PDL device-side primitives are emitted only for PTX versions >= 90. + // We have to guard on a loaded kernel's PTX version so a kernel forward-JIT'ed + // from pre-Hopper PTX to a Hopper-or-newer GPU does not opt into PDL. + const bool can_use_pdl = attr.ptxVersion >= 90; + cache.emplace(key, can_use_pdl); + return can_use_pdl; +} +#endif // defined(GGML_CUDA_USE_PDL) // destroying a cuBLAS handle while a graph is being captured in a different thread can result in a CUDA error // this lock is used to ensure that no cuBLAS handle is destroyed while a graph is being captured @@ -712,6 +930,11 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { CUDA_CHECK(cudaFree(cublas_workspaces[i][j])); } } +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 + if (cublaslt_handles[i] != nullptr) { + CUBLAS_CHECK(cublasLtDestroy(cublaslt_handles[i])); + } +#endif } } @@ -762,7 +985,8 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer if (padded_size > original_size) { ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size)); + CUDA_CHECK(cudaMemsetAsync((char *)tensor->data + original_size, 0, padded_size - original_size, cudaStreamPerThread)); + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); } } return GGML_STATUS_SUCCESS; @@ -812,6 +1036,51 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); } +static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + + ggml_cuda_set_device(ctx->device); + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + CUDA_CHECK(cudaMemcpyAsync( + (char *) tensors[i]->data + offsets[i], + data[i], + sizes[i], + cudaMemcpyHostToDevice, + cudaStreamPerThread)); + } + + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); +} + +static void ggml_backend_cuda_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + + ggml_cuda_set_device(ctx->device); + + if (!ggml_cuda_pinned_memory_disabled() && + ggml_cuda_get_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) { + return; + } + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + CUDA_CHECK(cudaMemcpyAsync( + data[i], + (const char *) tensors[i]->data + offsets[i], + sizes[i], + cudaMemcpyDeviceToHost, + cudaStreamPerThread)); + } + + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); +} + static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { if (ggml_backend_buffer_is_cuda(src->buffer)) { ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context; @@ -837,6 +1106,54 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co GGML_UNUSED(buffer); } +static bool ggml_backend_cuda_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) { + ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + + for (size_t i = 0; i < n_tensors; i++) { + ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer; + if (!ggml_backend_buffer_is_cuda(src_buf)) { + return false; + } + } + +#ifdef GGML_CUDA_NO_PEER_COPY + for (size_t i = 0; i < n_tensors; i++) { + ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer; + ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer; + ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src_buf->context; + ggml_backend_cuda_buffer_context * dst_ctx_i = (ggml_backend_cuda_buffer_context *)dst_buf->context; + const int src_physical = ggml_cuda_get_physical_device(src_ctx->device); + const int dst_physical = ggml_cuda_get_physical_device(dst_ctx_i->device); + if (src_physical != dst_physical) { + return false; + } + } +#endif + + ggml_cuda_set_device(dst_ctx->device); + + for (size_t i = 0; i < n_tensors; i++) { + ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer; + ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer; + ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src_buf->context; + ggml_backend_cuda_buffer_context * dst_ctx_i = (ggml_backend_cuda_buffer_context *)dst_buf->context; + const int src_physical = ggml_cuda_get_physical_device(src_ctx->device); + const int dst_physical = ggml_cuda_get_physical_device(dst_ctx_i->device); + if (src_physical == dst_physical) { + CUDA_CHECK(cudaMemcpyAsync(dsts[i]->data, srcs[i]->data, ggml_nbytes(srcs[i]), cudaMemcpyDeviceToDevice, cudaStreamPerThread)); + } else { +#ifdef GGML_CUDA_NO_PEER_COPY + GGML_ABORT("peer copy not supported"); +#else + CUDA_CHECK(cudaMemcpyPeerAsync(dsts[i]->data, dst_physical, srcs[i]->data, src_physical, ggml_nbytes(srcs[i]), cudaStreamPerThread)); +#endif + } + } + + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + return true; +} + static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; @@ -857,6 +1174,9 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = { /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor, /* .clear = */ ggml_backend_cuda_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ ggml_backend_cuda_buffer_set_tensor_n, + /* .get_tensor_n = */ ggml_backend_cuda_buffer_get_tensor_n, + /* .cpy_tensor_n = */ ggml_backend_cuda_buffer_cpy_tensor_n, }; // cuda buffer type @@ -977,6 +1297,18 @@ struct ggml_backend_cuda_comm_context { ggml_cuda_ar_pipeline * ar_pipeline = nullptr; + // Number of ranks in the communicator when each rank is a separate process. Zero unless + // comm_init_rank created this context. + int world_size = 0; + + // Number of ranks taking part in the reduction. For a communicator from comm_init_rank this is + // world_size. Otherwise it is the number of backends this process holds. The NCCL reduction + // uses it to choose the tensor size below which it reduces in F32 rather than converting to + // BF16 first. + size_t n_ranks() const { + return world_size > 0 ? (size_t) world_size : backends.size(); + } + #ifdef GGML_USE_NCCL std::vector<ncclComm_t> comms; #endif // GGML_USE_NCCL @@ -1004,6 +1336,7 @@ static bool ggml_backend_cuda_comm_allreduce_nccl( } const size_t n_backends = comm_ctx->backends.size(); + const size_t n_ranks = comm_ctx->n_ranks(); for (size_t i = 0; i < n_backends; ++i) { GGML_ASSERT(tensors[i] != nullptr); @@ -1013,7 +1346,7 @@ static bool ggml_backend_cuda_comm_allreduce_nccl( // For small tensors, simply reduce them as FP32. // The following heuristic for how "small" a tensor should be is based on RTX 4090s connected via 16x PCIe 4.0. - if ((n_backends <= 2 && ne < 32768) || (n_backends == 3 && ne < 131072) || (n_backends >= 4 && ne < 262144)) { + if ((n_ranks <= 2 && ne < 32768) || (n_ranks == 3 && ne < 131072) || (n_ranks >= 4 && ne < 262144)) { for (size_t i = 0; i < n_backends; ++i) { if ((tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) comm_ctx->backends[i]->context; @@ -1244,6 +1577,90 @@ static void * ggml_backend_cuda_comm_init(ggml_backend_t * backends, size_t n_ba return ret; } +#ifdef GGML_USE_NCCL +// Builds the communicator with ncclCommInitRank, for the case where each rank is its own +// process. Blocks until all world_size ranks have joined. +static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank, int world_size, + ggml_backend_comm_bootstrap_callback bootstrap_callback, void * bootstrap_callback_data) { + if (world_size < 1 || rank < 0 || rank >= world_size) { + GGML_LOG_ERROR("%s: invalid rank %d for world size %d\n", __func__, rank, world_size); + return nullptr; + } + + // Every rank sends whether its local checks passed, and rank 0 also sends the NCCL unique id. + // So all ranks either join together or fail together, and no rank waits in ncclCommInitRank alone. + struct bootstrap_msg { + uint8_t ok; + ncclUniqueId id; + }; + bootstrap_msg local; + memset(&local, 0, sizeof(local)); + local.ok = 1; + + if (!ggml_backend_is_cuda(backend)) { + GGML_LOG_ERROR("%s: backend is not a CUDA backend\n", __func__); + local.ok = 0; + } else { + // NCCL requires one distinct physical GPU per rank, see ggml_backend_cuda_comm_init_nccl. + const ggml_cuda_device_info & info = ggml_cuda_info(); + if (info.device_count > info.physical_device_count) { + GGML_LOG_ERROR("%s: CUDA virtual devices are in use, which NCCL does not support\n", __func__); + local.ok = 0; + } + } + if (local.ok && rank == 0) { + const ncclResult_t rc = ncclGetUniqueId(&local.id); + if (rc != ncclSuccess) { + GGML_LOG_ERROR("%s: ncclGetUniqueId failed (%s)\n", __func__, ncclGetErrorString(rc)); + local.ok = 0; + } + } + + std::vector<bootstrap_msg> all(world_size); + if (!bootstrap_callback(bootstrap_callback_data, &local, all.data(), sizeof(local))) { + GGML_LOG_ERROR("%s: bootstrap exchange failed on rank %d\n", __func__, rank); + return nullptr; + } + for (int i = 0; i < world_size; i++) { + if (!all[i].ok) { + GGML_LOG_ERROR("%s: rank %d failed its setup checks\n", __func__, i); + return nullptr; + } + } + + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; + + auto * ret = new ggml_backend_cuda_comm_context; + ret->backends.push_back(backend); + ret->dev_ids.push_back(cuda_ctx->device); + ret->world_size = world_size; + ret->comms.resize(1); + ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_nccl; + + // On DGX Spark, NCCL picks 64 channels by default, which is slower and uses more memory than 8. + // NCCL_MAX_CTAS still overrides this. + ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + config.maxCTAs = 8; + } + + ggml_cuda_set_device(cuda_ctx->device); + const ncclResult_t rc = ncclCommInitRankConfig(&ret->comms[0], world_size, all[0].id, rank, &config); + if (rc != ncclSuccess) { + GGML_LOG_ERROR("%s: ncclCommInitRankConfig failed for rank %d/%d (%s)\n", + __func__, rank, world_size, ncclGetErrorString(rc)); + ret->comms.clear(); + delete ret; + return nullptr; + } + + GGML_LOG_INFO("%s: joined NCCL communicator as rank %d/%d on device %d\n", + __func__, rank, world_size, cuda_ctx->device); + return ret; +} +#endif // GGML_USE_NCCL + // Top-level dispatch -- calls the function pointer chosen by comm_init. // Returns false to let the meta-backend's butterfly run. static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { @@ -1271,7 +1688,7 @@ static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buff } static void * ggml_cuda_host_malloc(size_t size) { - if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { + if (ggml_cuda_pinned_memory_disabled()) { return nullptr; } @@ -1303,6 +1720,12 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggm return buffer; } +static size_t ggml_backend_cuda_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + // very large pinned allocations can fail (e.g. on Windows) and fall back to pageable memory, smaller ones succeed + return 4ull*1024*1024*1024; +} + ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = { /* .iface = */ { @@ -1310,7 +1733,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer, /* .alloc_buffer_n = */ NULL, /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_max_size = */ ggml_backend_cuda_host_buffer_type_get_max_size, /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, /* .get_alloc_size_n = */ NULL, /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, @@ -1409,13 +1832,13 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const using traits = batched_mul_mat_traits<compute_type>; using cuda_t = typename traits::cuda_type; - GGML_ASSERT(ggml_is_contiguous(dst)); - - // Byte offsets and tensor dimensions are currently used in an inconsistent way for dst. - // As long as dst is contiguous this does not matter though. + // F32 chunks can have padding between rows to preserve the original destination stride. + GGML_ASSERT(ggml_is_contiguous(dst) || + (compute_type == GGML_TYPE_F32 && ggml_is_contiguous_1(dst))); GGML_TENSOR_BINARY_OP_LOCALS + const int64_t ldc = nb1 / sizeof(float); const int64_t ne_dst = ggml_nelements(dst); cudaStream_t main_stream = ctx.stream(); cublasHandle_t cublas_h = ctx.cublas_handle(); @@ -1510,7 +1933,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const if (compute_type == GGML_TYPE_F16) { prefer_f32_output = cc == GGML_CUDA_CC_VOLTA || GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_CDNA(cc); } else if (compute_type == GGML_TYPE_BF16) { - prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc); + prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_RDNA4_M(cc) && !GGML_CUDA_CC_IS_CDNA(cc); } if (prefer_f32_output) { @@ -1545,14 +1968,14 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ne01, ne11, ne10, (const float *) alpha, (const float *) src0_ptr, s01, (const float *) src1_ptr, s11, - (const float *) beta, (float *) dst_ptr, ne0)); + (const float *) beta, (float *) dst_ptr, ldc)); } else if (ne12 == 1 && ne13 == 1) { CUBLAS_CHECK( cublasGemmEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, src0_ptr, cu_data_type_a, s01, src1_ptr, cu_data_type_b, s11, - beta, dst_ptr, cu_data_type, ne0, + beta, dst_ptr, cu_data_type, ldc, cu_compute_type, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); } else if (r2 == 1 && r3 == 1 && is_src0_cont_2 && is_src1_cont_2) { @@ -1567,7 +1990,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ne01, ne11, ne10, alpha, src0_ptr, cu_data_type_a, s01, sma, // strideA src1_ptr, cu_data_type_b, s11, smb, // strideB - beta, dst_ptr, cu_data_type, ne0, ne1*ne0, // strideC + beta, dst_ptr, cu_data_type, ldc, ne1*ldc, // strideC ne12*ne13, cu_compute_type, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); @@ -1605,7 +2028,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ne01, ne11, ne10, alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, s01, (const void **) (ptrs_src.get() + 1*ne23), cu_data_type_b, s11, - beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne0, + beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ldc, ne23, cu_compute_type, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); @@ -1620,9 +2043,9 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const int cc = ggml_cuda_info().devices[ctx.device].cc; - const ggml_prec prec = (ggml_prec) ggml_get_op_params_i32(dst, 0); + const ggml_prec prec = ggml_prec_get_acc(dst); ggml_type compute_type = src0->type; - if (ggml_is_quantized(compute_type)) { + if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) { compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32; } else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) { compute_type = GGML_TYPE_F32; @@ -1658,6 +2081,32 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml } } + constexpr size_t max_src0_convert_size = 512ull * 1024 * 1024; + const size_t src0_f32_size = ggml_nelements(src0) * sizeof(float); + + if (compute_type == GGML_TYPE_F32 && + (src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) && + src0_f32_size > max_src0_convert_size) { + const size_t f32_row_size = src0_f32_size / src0->ne[1]; + const int64_t rows_per_chunk = std::max<int64_t>(1, (int64_t) (max_src0_convert_size / f32_row_size)); + + if (rows_per_chunk < src0->ne[1]) { + ggml_tensor src0_chunk = *src0; + ggml_tensor dst_chunk = *dst; + + for (int64_t i01 = 0; i01 < src0->ne[1]; i01 += rows_per_chunk) { + src0_chunk.ne[1] = std::min(rows_per_chunk, src0->ne[1] - i01); + src0_chunk.data = (char *) src0->data + i01*src0->nb[1]; + + dst_chunk.ne[0] = src0_chunk.ne[1]; + dst_chunk.data = (char *) dst->data + i01*dst->nb[0]; + + ggml_cuda_mul_mat_cublas_impl<GGML_TYPE_F32>(ctx, &src0_chunk, src1, &dst_chunk); + } + return; + } + } + switch (compute_type) { case GGML_TYPE_F32: ggml_cuda_mul_mat_cublas_impl<GGML_TYPE_F32>(ctx, src0, src1, dst); @@ -1767,7 +2216,7 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, return true; } -static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { +static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor, const ggml_tensor * gate = nullptr) { ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; const ggml_tensor * dst = tensor; @@ -1775,12 +2224,17 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { const bool is_mul_mat_id = tensor->op == GGML_OP_MUL_MAT_ID; bool use_mul_mat_vec_f = - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || + src0->type == GGML_TYPE_F8_E4M3) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; - use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]); + use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0, cc, warp_size, is_mul_mat_id ? src1->ne[2] : src1->ne[1]); + + if (gate && !ggml_cuda_should_use_mmvf(gate->src[0], cc, warp_size, is_mul_mat_id ? src1->ne[2] : src1->ne[1])) { + return false; + } //we only support fusion for ncols_dst = 1 if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) { @@ -1804,11 +2258,11 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) { ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src; - bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear && src1->type == GGML_TYPE_F32 && - dst->type == GGML_TYPE_F32 && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE; + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + bool use_mul_mat_vec_q = ggml_cuda_should_use_mmvq(src0->type, cc, src1->ne[1]) && !bad_padding_clear && + src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; // fusion is not universally faster on Pascal - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; if (cc <= GGML_CUDA_CC_PASCAL) { return false; } @@ -1876,6 +2330,119 @@ static bool ggml_cuda_match_shared_expert(const ggml_cgraph * graph, int routed_ return true; } +// The weight scale that scale_node (MUL) applies to the output of mm_node (MUL_MAT), or nullptr. +// Shared by ggml_cuda_try_fuse and graph_optimize so that both accept the same patterns. +static const ggml_tensor * ggml_cuda_get_mul_mat_scale(const ggml_tensor * scale_node, const ggml_tensor * mm_node) { + const bool scale_lhs_mm = scale_node->src[0] == mm_node; + const bool scale_rhs_mm = scale_node->src[1] == mm_node; + if (!scale_lhs_mm && !scale_rhs_mm) { + return nullptr; + } + + const ggml_tensor * scale = scale_lhs_mm ? scale_node->src[1] : scale_node->src[0]; + if ((mm_node->src[0]->type != GGML_TYPE_NVFP4 && mm_node->src[0]->type != GGML_TYPE_F8_E4M3) || scale_node->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != 1 || + !ggml_are_same_shape(scale_node, mm_node)) { + return nullptr; + } + + return scale; +} + +// The same for MUL_MAT_ID with one scale per expert: RESHAPE, REPEAT and GET_ROWS (by the ids) of the scales, then the MUL. +static const ggml_tensor * ggml_cuda_get_mul_mat_id_scale(const ggml_tensor * reshape, const ggml_tensor * repeat, const ggml_tensor * getrows, + const ggml_tensor * scale_node, const ggml_tensor * mm_node) { + if (repeat->src[0] != reshape || getrows->src[0] != repeat || getrows->src[1] != mm_node->src[2]) { + return nullptr; + } + if (!((scale_node->src[0] == mm_node && scale_node->src[1] == getrows) || + (scale_node->src[0] == getrows && scale_node->src[1] == mm_node))) { + return nullptr; + } + + const ggml_tensor * scale = reshape->src[0]; + if ((mm_node->src[0]->type != GGML_TYPE_NVFP4 && mm_node->src[0]->type != GGML_TYPE_F8_E4M3) || scale_node->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != mm_node->src[0]->ne[2] || + !ggml_are_same_shape(scale_node, mm_node)) { + return nullptr; + } + + return scale; +} + +// Buffer- and batch-size independent, so graph_optimize produces the same graph topology for every ubatch size (see #29986). +// up_scale/gate_scale: the MUL nodes that apply the NVFP4 weight scales (per tensor or per expert) between the MUL_MATs and the GLU. +static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu, const int device, + const ggml_tensor * up_scale = nullptr, const ggml_tensor * gate_scale = nullptr) { + const ggml_tensor * x_up = up->src[0]; + const ggml_tensor * y = up->src[1]; + + if ((up->op != GGML_OP_MUL_MAT && up->op != GGML_OP_MUL_MAT_ID) || !ggml_cuda_mmq_fusion_supported(x_up->type) || y->type != GGML_TYPE_F32) { + return false; + } + + // The fused kernel is only compiled for weights whose rows fill whole tiles (no fallback). + if (ggml_cuda_mmq_needs_fallback(x_up->ne[1])) { + return false; + } + + if (!ggml_cuda_should_fuse_mul_mat(up, gate, glu, nullptr, nullptr, up_scale, gate_scale)) { + return false; + } + + // The fused kernel runs both MUL_MATs at one src1 precision, see ggml_cuda_mul_mat_q. + if (ggml_prec_get_src(up, 1) != ggml_prec_get_src(gate, 1)) { + return false; + } + + // The fused write-back applies SWIGLU_OAI with the default alpha and limit of ggml_cuda_op_swiglu_oai_single. + if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI && (ggml_get_op_params_f32(glu, 2) != 1.702f || ggml_get_op_params_f32(glu, 3) != 7.0f)) { + return false; + } + + // NVIDIA only: the fused write-back relies on the MMA accumulator layout. + return turing_mma_available(ggml_cuda_info().devices[device].cc); +} + +static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu, + const ggml_tensor * up_scale = nullptr, const ggml_tensor * gate_scale = nullptr) { + // A hinted matmul can be fused, but its unfused path may be faster (e.g. FWHT for GGML_HINT_SRC0_IS_HADAMARD). + if (ggml_get_op_params_i32(up, 1) != GGML_HINT_NONE || ggml_get_op_params_i32(gate, 1) != GGML_HINT_NONE) { + return false; + } + + // The padding of a compute-buffer weight that is a view cannot be cleared, see ggml_cuda_mul_mat. + const auto bad_padding_clear = [](const ggml_tensor * x) { + return ggml_backend_buffer_get_usage(x->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_nbytes(x) != ggml_backend_buffer_get_alloc_size(x->buffer, x) && x->view_src; + }; + const int device = ggml_cuda_get_device(); + if (!ggml_cuda_mul_mat_q_fusion_matches(up, gate, glu, device, up_scale, gate_scale) || bad_padding_clear(up->src[0]) || bad_padding_clear(gate->src[0])) { + return false; + } + + // The batch size checks are done here, not in ggml_cuda_mul_mat_q_fusion_matches. + const ggml_tensor * x_up = up->src[0]; + const ggml_tensor * y = up->src[1]; + const int cc = ggml_cuda_info().devices[device].cc; + + // For MUL_MAT_ID: same kernel choice as ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU). + const bool is_id = up->op == GGML_OP_MUL_MAT_ID; + const int64_t ncols = is_id ? y->ne[2] : y->ne[1]; + const int64_t nchannels = is_id ? x_up->ne[2] : y->ne[2]*y->ne[3]; + const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(x_up->type, cc) : ggml_cuda_should_use_mmvq(x_up->type, cc, ncols); + // do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat + if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, cc)) { + return false; + } + if (use_mmvq || !ggml_cuda_should_use_mmq(x_up->type, cc, ncols, /*n_experts =*/ is_id ? x_up->ne[2] : 0)) { + return false; + } + + // The fused kernel needs one block per tile (no stream-k split), so only fuse if the tiling is efficient. + return ggml_cuda_mmq_fusion_is_efficient(x_up->type, ggml_prec_get_src(up, 1), x_up->ne[1], ncols, nchannels, device); +} + static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_TENSOR_BINARY_OP_LOCALS @@ -1896,17 +2463,17 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor const int cc = ggml_cuda_info().devices[ctx.device].cc; const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - if (ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11)) { - // The custom F16 vector kernel can be used over batched cuBLAS GEMM. + if (ggml_cuda_should_use_mmvf(src0, cc, warp_size, ne11)) { + // The custom vector kernel can be used over batched cuBLAS GEMM. // But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); return; } - // A transposed vector can still use MMVQ (i.e. ne01 == 1) + // A transposed vector can still use MMVF (i.e. ne01 == 1) if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1 && src0->type == GGML_TYPE_F32 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) - && ggml_cuda_should_use_mmvf(src1->type, cc, warp_size, src1->ne, src1->nb, /*ne11 =*/ 1)) { + && ggml_cuda_should_use_mmvf(src1, cc, warp_size, /*ne11 =*/ 1)) { ggml_tensor dst_vec = *dst; dst_vec.ne[0] = ne11; dst_vec.ne[1] = 1; @@ -1920,14 +2487,23 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst); return; } - if (ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { - ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); + // from a per-type number of src1 columns up to 24, few-row MMQ is faster than MMVQ and MMQ + const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, cc); + if (!use_mmq_few && ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { + ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); + return; + } + if (use_mmq_few) { + ggml_cuda_mul_mat_q_few(ctx, src0, src1, dst); return; } if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); return; } + if (src0->type == GGML_TYPE_F8_E4M3 && ggml_cuda_mul_mat_fp8(ctx, src0, src1, dst)) { + return; + } ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); } @@ -1936,6 +2512,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int cc) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { return true; @@ -1946,7 +2523,10 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c if (dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc)) { return false; } - } else if (GGML_CUDA_CC_IS_AMD(cc)) { + } else if (src0->type == GGML_TYPE_F8_E4M3 && + ggml_cuda_should_use_mmvf(src0, cc, warp_size, dst->ne[2])) { + return false; + } else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) { return false; } } @@ -1955,7 +2535,7 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c return false; } - if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { + if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { return false; } @@ -1972,7 +2552,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_TENSOR_BINARY_OP_LOCALS - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { @@ -1984,8 +2565,12 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst); return; } + } else if (src0->type == GGML_TYPE_F8_E4M3 && + ggml_cuda_should_use_mmvf(src0, cc, warp_size, ne2)) { + ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst); + return; } else { - if (GGML_CUDA_CC_IS_AMD(cc)) { + if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) { ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst); return; } @@ -1997,7 +2582,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * return; } - if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { + if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { ggml_cuda_mul_mat_f(ctx, src0, src1, ids, dst); return; } @@ -2011,7 +2596,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_ASSERT(nb2 % nb1 == 0); const ggml_type type_src1_sorted = (src0->type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) - || ggml_is_quantized(src0->type) ? GGML_TYPE_F32 : src0->type; + || ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3 ? GGML_TYPE_F32 : src0->type; const ggml_type type_dst_sorted = GGML_TYPE_F32; const size_t ts_src1_sorted = ggml_type_size(type_src1_sorted); const size_t ts_dst_sorted = ggml_type_size(type_dst_sorted); @@ -2156,7 +2741,6 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg ggml_cuda_dup(ctx, dst); break; case GGML_OP_ADD: - case GGML_OP_ADD1: // TODO: more efficient implementation ggml_cuda_op_add(ctx, dst); break; case GGML_OP_ADD_ID: @@ -2351,9 +2935,6 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_DIAG: ggml_cuda_op_diag(ctx, dst); break; - case GGML_OP_DIAG_MASK_INF: - ggml_cuda_op_diag_mask_inf(ctx, dst); - break; case GGML_OP_SOFT_MAX: ggml_cuda_op_soft_max(ctx, dst); break; @@ -2471,6 +3052,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_LIGHTNING_INDEXER: ggml_cuda_lightning_indexer(ctx, dst); break; + case GGML_OP_TOP_K_MASK: + ggml_cuda_op_top_k_mask(ctx, dst); + break; default: return false; } @@ -2614,6 +3198,11 @@ static bool ggml_cuda_is_view_or_noop(const ggml_tensor * t) { } #ifdef USE_CUDA_GRAPH +static bool ggml_cuda_graph_debug_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_GRAPH_DEBUG"); + return enabled; +} + static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { bool use_cuda_graph = true; @@ -2633,9 +3222,9 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { // the mul_mat_id fallback path synchronizes the stream, so we cannot use CUDA graphs // ref: https://github.com/ggml-org/llama.cpp/pull/18958 use_cuda_graph = false; -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__); -#endif + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__); + } } } @@ -2659,7 +3248,9 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx if (cgraph->uid != 0 && cgraph->uid == graph->uid) { - GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid); + } GGML_ASSERT((int)graph->node_props.size() == cgraph->n_nodes); return false; } @@ -2693,33 +3284,107 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx return res; } -static void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) { - ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK +static std::vector<ggml_cuda_graph_cluster> ggml_cuda_graph_get_clusters(cudaGraph_t cuda_graph, int device) { + std::vector<ggml_cuda_graph_cluster> clusters; + static const bool runtime_affected = [] { + int version = 0; + CUDA_CHECK(cudaRuntimeGetVersion(&version)); + // TODO: upper bound once fixed in the CUDA runtime + return version >= 11080; + }(); + if (!runtime_affected || ggml_cuda_info().devices[device].cc < GGML_CUDA_CC_HOPPER) { + return clusters; + } + size_t n_nodes = 0; + CUDA_CHECK(cudaGraphGetNodes(cuda_graph, nullptr, &n_nodes)); + std::vector<cudaGraphNode_t> nodes(n_nodes); + CUDA_CHECK(cudaGraphGetNodes(cuda_graph, nodes.data(), &n_nodes)); + for (size_t i = 0; i < n_nodes; ++i) { + cudaGraphNodeType type; + CUDA_CHECK(cudaGraphNodeGetType(nodes[i], &type)); + if (type != cudaGraphNodeTypeKernel) { + continue; + } + cudaLaunchAttributeValue val = {}; + CUDA_CHECK(cudaGraphKernelNodeGetAttribute(nodes[i], cudaLaunchAttributeClusterDimension, &val)); + if (val.clusterDim.x * val.clusterDim.y * val.clusterDim.z > 1) { + clusters.push_back({ i, val.clusterDim.x, val.clusterDim.y, val.clusterDim.z }); + } + } + return clusters; +} +#endif // USE_CUDA_GRAPH_CLUSTER_CHECK + +static void ggml_cuda_graph_update_executable(ggml_cuda_graph_instance & instance, cudaGraph_t cuda_graph, int device) { +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK + std::vector<ggml_cuda_graph_cluster> clusters = ggml_cuda_graph_get_clusters(cuda_graph, device); + if (instance.clusters != clusters) { + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: cluster dims changed, re-instantiating CUDA graph\n", __func__); + } + CUDA_CHECK(cudaGraphExecDestroy(instance)); + CUDA_CHECK(cudaGraphInstantiate(&instance.handle, cuda_graph, nullptr, nullptr, 0)); + instance.clusters = std::move(clusters); + return; + } +#else + GGML_UNUSED(device); +#endif // USE_CUDA_GRAPH_CLUSTER_CHECK #if CUDART_VERSION >= 12000 cudaGraphExecUpdateResultInfo result_info; - cudaError_t stat = cudaGraphExecUpdate(graph->instance, graph->graph, &result_info); + cudaError_t stat = cudaGraphExecUpdate(instance, cuda_graph, &result_info); #else cudaGraphNode_t errorNode; cudaGraphExecUpdateResult result_info; - cudaError_t stat = cudaGraphExecUpdate(graph->instance, graph->graph, &errorNode, &result_info); + cudaError_t stat = cudaGraphExecUpdate(instance, cuda_graph, &errorNode, &result_info); #endif // CUDART_VERSION >= 12000 if (stat == cudaErrorGraphExecUpdateFailure) { -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__); -#endif + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__); + } // The pre-existing graph exec cannot be updated due to violated constraints // so instead clear error and re-instantiate (void)cudaGetLastError(); - CUDA_CHECK(cudaGraphExecDestroy(graph->instance)); - graph->instance = nullptr; - CUDA_CHECK(cudaGraphInstantiate(&graph->instance, graph->graph, NULL, NULL, 0)); + CUDA_CHECK(cudaGraphExecDestroy(instance)); + instance.handle = nullptr; + CUDA_CHECK(cudaGraphInstantiate(&instance.handle, cuda_graph, nullptr, nullptr, 0)); } else { GGML_ASSERT(stat == cudaSuccess); } } + +// number of ggml nodes per CUDA graph chunk, 0 captures the whole ggml graph as one CUDA graph +static int ggml_cuda_graph_chunk_size() { + static const int chunk_size = [] { + const char * env = getenv("GGML_CUDA_GRAPH_CHUNK_SIZE"); + return env != nullptr ? atoi(env) : 512; + }(); + return chunk_size; +} + +// end the capture of chunk `idx`, update or create its executable and launch it +static void ggml_cuda_graph_launch_chunk(ggml_backend_cuda_context * cuda_ctx, ggml_cuda_graph * graph, size_t idx) { + cudaGraph_t cuda_graph = nullptr; + CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &cuda_graph)); + + if (idx < graph->instances.size()) { + ggml_cuda_graph_update_executable(graph->instances[idx], cuda_graph, cuda_ctx->device); + } else { + GGML_ASSERT(idx == graph->instances.size()); + graph->instances.emplace_back(); + CUDA_CHECK(cudaGraphInstantiate(&graph->instances[idx].handle, cuda_graph, nullptr, nullptr, 0)); +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK + graph->instances[idx].clusters = ggml_cuda_graph_get_clusters(cuda_graph, cuda_ctx->device); +#endif // USE_CUDA_GRAPH_CLUSTER_CHECK + } + CUDA_CHECK(cudaGraphDestroy(cuda_graph)); + + CUDA_CHECK(cudaGraphLaunch(graph->instances[idx], cuda_ctx->stream())); +} #endif // USE_CUDA_GRAPH static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope, @@ -2747,9 +3412,10 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope, return false; } - // Only norm/neox shaders have the fusion code + // Only norm/neox/multi shaders have the fusion code const int mode = ((const int32_t *) rope->op_params)[2]; - if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX) { + if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && + mode != GGML_ROPE_TYPE_MROPE && mode != GGML_ROPE_TYPE_IMROPE) { return false; } @@ -2872,6 +3538,138 @@ static int ggml_cuda_try_gdn_cache_fusion( return skip; } +// match ssm_scan + the strided cpy that scatters its state snapshots into the cache, so the kernel writes them and skips the cpy +static int ggml_cuda_try_ssm_scan_cache_fusion( + const ggml_cgraph * cgraph, int node_idx, ggml_cuda_ssm_scan_fused_cache & fused_state_cpy) { + const ggml_tensor * ssm = cgraph->nodes[node_idx]; + // the kernel skips the snapshot tail, so the scan output must not be a graph output + if (ssm->op != GGML_OP_SSM_SCAN || ssm->type != GGML_TYPE_F32 || (ssm->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + + const int64_t K = ggml_get_op_params_i32(ssm, 0); // snapshot slot count + + const ggml_tensor * s = ssm->src[0]; + const ggml_tensor * x = ssm->src[1]; + const ggml_tensor * A = ssm->src[3]; + + const int64_t d_state = s->ne[0]; + const int64_t D = d_state * s->ne[1] * x->ne[1]; // d_state * head_dim * n_head + const int64_t n_tok = x->ne[2]; + const int64_t n_seqs = x->ne[3]; + + // only the mamba-2 kernels (group scan and SSD) write to the cache; mamba-1 still uses the cpy + if (A->nb[1] != sizeof(float) || (d_state != 96 && d_state != 128 && d_state != 256)) { + return 0; + } + + // the scan reads its input rows from the cache (picked by ids), so with more than one seq a seq can read a row that another seq writes in the same launch + if (n_seqs != 1) { + return 0; + } + + const int64_t n_written = std::min<int64_t>(n_tok, K); + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, ggml_nelements(x)); + + // snapshot cpy is the first real node after the scan (skip views/no-ops) + const ggml_tensor * cpy = nullptr; + int skip = 0; + for (int j = node_idx + 1; j < cgraph->n_nodes && cpy == nullptr; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_cuda_is_view_or_noop(n)) { + continue; + } + if (n->op != GGML_OP_CPY || (n->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + cpy = n; + skip = j - node_idx; + } + if (cpy == nullptr) { + return 0; + } + + const ggml_tensor * src = cpy->src[0]; // view of the scan snapshot tail + const ggml_tensor * dst = cpy->src[1]; // cache view the kernel writes to + + // src must be this scan's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != ssm || src->view_offs != tail_off || + !ggml_is_contiguous(src)) { + return 0; + } + + // dst is the [D, n_seqs, n_written] cache view; require nb[1] == D, the per-seq stride the kernel takes from src0->nb[3] + const std::array<int64_t, GGML_MAX_DIMS> expected_ne = { D, n_seqs, n_written, 1 }; + if (dst->op != GGML_OP_VIEW || dst->type != GGML_TYPE_F32 || dst->data == nullptr || + !std::equal(expected_ne.begin(), expected_ne.end(), dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || dst->nb[1] != (size_t) ggml_row_size(GGML_TYPE_F32, D)) { + return 0; + } + + fused_state_cpy.data = (float *) dst->data; // rollback slot 0 (newest) + fused_state_cpy.slot_stride = K > 1 ? (int64_t) (dst->nb[2] / sizeof(float)) : 0; + return skip; +} + +// expert group selection from build_moe_ffn (n_expert_groups > 1) +// src0/src1 index into this table, -1 is the (biased) probs, -2 is not checked +static const struct { + ggml_op op; + int src0; + int src1; +} topk_moe_group_pattern[] = { + { GGML_OP_RESHAPE, -1, -2 }, // 0: [n_exp_per_group, n_expert_groups, n_tokens] + { GGML_OP_RESHAPE, 0, -2 }, // 1: [1, n_exp_per_group, n_expert_groups, n_tokens] + { GGML_OP_ARGSORT, 0, -2 }, // 2 + { GGML_OP_VIEW, 2, -2 }, // 3: top-2 per group + { GGML_OP_GET_ROWS, 1, 3 }, // 4 + { GGML_OP_RESHAPE, 4, -2 }, // 5 + { GGML_OP_SUM_ROWS, 5, -2 }, // 6: group scores + { GGML_OP_RESHAPE, 6, -2 }, // 7 + { GGML_OP_ARGSORT, 7, -2 }, // 8 + { GGML_OP_VIEW, 8, -2 }, // 9: top n_group_used groups + { GGML_OP_GET_ROWS, 0, 9 }, // 10 + { GGML_OP_FILL, 0, -2 }, // 11 + { GGML_OP_SET_ROWS, 10, 9 }, // 12: unused groups -> -inf + { GGML_OP_RESHAPE, 12, -2 }, // 13: [n_expert, n_tokens] +}; + +static bool ggml_cuda_topk_moe_group_fusion(const struct ggml_cgraph * cgraph, int node_idx, const ggml_tensor * selection, ggml_cuda_topk_moe_args & args) { +#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000 + GGML_UNUSED_VARS(cgraph, node_idx, selection, args); + return false; +#else + const int n_ops = (int) std::size(topk_moe_group_pattern); + if (node_idx + n_ops > cgraph->n_nodes) { + return false; + } + + ggml_tensor ** nodes = cgraph->nodes + node_idx; + auto src_ok = [&](const ggml_tensor * src, int idx) { + return idx == -2 || src == (idx == -1 ? selection : nodes[idx]); + }; + + for (int i = 0; i < n_ops; i++) { + const auto & p = topk_moe_group_pattern[i]; + if (nodes[i]->op != p.op || !src_ok(nodes[i]->src[0], p.src0) || !src_ok(nodes[i]->src[1], p.src1)) { + return false; + } + } + + const int n_groups = nodes[0]->ne[1]; + const int n_group_used = nodes[9]->ne[0]; + + if (nodes[3]->ne[0] != 2 || nodes[12]->src[2] != nodes[11] || ggml_get_op_params_f32(nodes[11], 0) != -INFINITY || + n_groups > WARP_SIZE) { + return false; + } + + args.n_groups = n_groups; + args.n_group_used = n_group_used; + return true; +#endif // defined(GGML_USE_HIP) && HIP_VERSION < 60300000 +} + static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) { args.sigmoid = false; args.sqrt_softplus = false; @@ -2879,6 +3677,8 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod args.delayed_softmax = false; args.prob_bias = false; args.norm = false; + args.n_groups = 0; + args.n_group_used = 0; const int n_nodes = cgraph->n_nodes; ggml_tensor ** nodes = cgraph->nodes; @@ -2921,18 +3721,20 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod } // src of bias add is the unreshaped probs (-2 instead of -1) - if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == nodes[node_idx - 2]) { + const ggml_tensor * selection = nodes[node_idx - 2]; + if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == selection) { args.prob_bias = true; + selection = nodes[node_idx]; node_idx++; } - // RESHAPE/ADD -> ARGSORT - if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT) { - return false; + + if (ggml_cuda_topk_moe_group_fusion(cgraph, node_idx, selection, args)) { + node_idx += (int) std::size(topk_moe_group_pattern); + selection = nodes[node_idx - 1]; } - if (args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 1]) { - return false; - } else if (!args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 2]) { + // RESHAPE/ADD/group mask -> ARGSORT + if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT || nodes[node_idx]->src[0] != selection) { return false; } @@ -2943,6 +3745,11 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod nodes[node_idx]->src[0] != nodes[node_idx - 1]) { return false; } + + // kept groups must hold all used experts + if (args.n_groups > 0 && nodes[node_idx]->ne[0] > args.n_group_used * (probs_reshaped->ne[1] / args.n_groups)) { + return false; + } node_idx++; if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_GET_ROWS) { @@ -3325,6 +4132,26 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, } } + // scale_out*sigmoid(scale_in*post) as the hc_post weights (qwen4exp hc_combine) + std::initializer_list<enum ggml_op> hc_post_gate_ops = { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_DSV4_HC_POST }; + + if (is_equal(hc_post_gate_ops, ops) && ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 3 })) { + const ggml_tensor * scale_in = cgraph->nodes[node_idx]; + const ggml_tensor * sigmoid = cgraph->nodes[node_idx + 1]; + const ggml_tensor * scale_out = cgraph->nodes[node_idx + 2]; + + // the chain must feed hc_post->src[2]; a bias on either scale is not handled + if (ggml_check_edges(cgraph, node_idx, {{1, 0, 0}, {2, 0, 1}, {3, 2, 2}}) && + ggml_get_unary_op(sigmoid) == GGML_UNARY_OP_SIGMOID && + ggml_get_op_params_f32(scale_in, 1) == 0.0f && ggml_get_op_params_f32(scale_out, 1) == 0.0f && + scale_in->src[0]->type == GGML_TYPE_F32 && scale_out->type == GGML_TYPE_F32 && + ggml_are_same_shape(scale_in->src[0], scale_out)) { + int out_nodes[] = { node_idx + 3 }; + return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1); + } + return false; + } + if (!ggml_can_fuse(cgraph, node_idx, ops)) { return false; } @@ -3487,6 +4314,11 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, return false; } + // TODO: make the fused softcap kernel stride aware, it reads src and writes dst as dense + if (!ggml_is_contiguous(scale->src[0]) || !ggml_is_contiguous(scale2)) { + return false; + } + if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) { return false; } @@ -3505,8 +4337,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, // try and fuse nodes and return the number of nodes to skip static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { - static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); - if (disable_fusion) { + if (ggml_cuda_fusion_disabled()) { return 0; } @@ -3558,6 +4389,20 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } + // ssm_scan -> cpy: scatter recurrent-state snapshots into the cache + if (node->op == GGML_OP_SSM_SCAN) { + ggml_cuda_ssm_scan_fused_cache fused_state_cpy; + const int nodes_to_skip = ggml_cuda_try_ssm_scan_cache_fusion(cgraph, i, fused_state_cpy); + if (nodes_to_skip > 0) { +#ifdef GGML_CUDA_DEBUG + GGML_LOG_INFO("%s: fused ssm_scan snapshot copies for %s (skipped %d nodes)\n", + __func__, node->name, nodes_to_skip); +#endif + ggml_cuda_op_ssm_scan_fused_cache(*cuda_ctx, node, fused_state_cpy); + return nodes_to_skip; + } + } + //topk-moe if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || cgraph->nodes[i]->op == GGML_OP_ARGSORT) { @@ -3584,17 +4429,18 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } else { ops.insert(ops.end(), { GGML_OP_SOFT_MAX }); } - const int i_probs = i + (int) ops.size() - 1; // last node of the gating activation - + ops.push_back(GGML_OP_RESHAPE); if (args.prob_bias) { - bias = cgraph->nodes[i_probs + 2]->src[1]; - ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, - GGML_OP_GET_ROWS }); - out_nodes[0] = i_probs + 4; - } else { - ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }); - out_nodes[0] = i_probs + 3; + ops.push_back(GGML_OP_ADD); + bias = cgraph->nodes[i + ops.size() - 1]->src[1]; + } + if (args.n_groups > 0) { + for (const auto & p : topk_moe_group_pattern) { + ops.push_back(p.op); + } } + ops.insert(ops.end(), { GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }); + out_nodes[0] = i + ops.size() - 2; ids = cgraph->nodes[out_nodes[0]]; if (args.norm) { @@ -3730,43 +4576,6 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph bool fused_mul_mat_vec = false; int fused_node_count = 0; - auto get_mul_mat_scale = [](const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * { - const bool scale_lhs_mm = scale_node->src[0] == mm_node; - const bool scale_rhs_mm = scale_node->src[1] == mm_node; - if (!scale_lhs_mm && !scale_rhs_mm) { - return nullptr; - } - - const ggml_tensor * scale = scale_lhs_mm ? scale_node->src[1] : scale_node->src[0]; - if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 || - scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != 1 || - !ggml_are_same_shape(scale_node, mm_node)) { - return nullptr; - } - - return scale; - }; - - auto get_mul_mat_id_scale = [](const ggml_tensor * reshape, const ggml_tensor * repeat, const ggml_tensor * getrows, - const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * { - if (repeat->src[0] != reshape || getrows->src[0] != repeat || getrows->src[1] != mm_node->src[2]) { - return nullptr; - } - if (!((scale_node->src[0] == mm_node && scale_node->src[1] == getrows) || - (scale_node->src[0] == getrows && scale_node->src[1] == mm_node))) { - return nullptr; - } - - const ggml_tensor * scale = reshape->src[0]; - if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 || - scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != mm_node->src[0]->ne[2] || - !ggml_are_same_shape(scale_node, mm_node)) { - return nullptr; - } - - return scale; - }; - auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) -> const ggml_tensor * { if (op_bias == GGML_OP_ADD) { if (bias_node->src[0] == mul_node) { @@ -3833,8 +4642,8 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph continue; } - const ggml_tensor * gate_scale = get_mul_mat_scale(gate_scale_n, gate_n); - const ggml_tensor * up_scale = get_mul_mat_scale(up_scale_n, up_n); + const ggml_tensor * gate_scale = ggml_cuda_get_mul_mat_scale(gate_scale_n, gate_n); + const ggml_tensor * up_scale = ggml_cuda_get_mul_mat_scale(up_scale_n, up_n); if (!gate_scale || !up_scale) { continue; } @@ -3859,12 +4668,27 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.glu_op = ggml_get_glu_op(glu); fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) { + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); fused_mul_mat_vec = true; fused_node_count = n_ops; break; } + + if (!with_bias && ggml_cuda_should_fuse_mul_mat_q(up_n, gate_n, glu, up_scale_n, gate_scale_n)) { + fusion_data.prec_src1 = ggml_prec_get_src(up_n, 1); + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } } if (fused_mul_mat_vec) { @@ -3929,9 +4753,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph continue; } - const ggml_tensor * gate_scale = get_mul_mat_id_scale(cgraph->nodes[gate_idx + 1], cgraph->nodes[gate_idx + 2], + const ggml_tensor * gate_scale = ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[gate_idx + 1], cgraph->nodes[gate_idx + 2], cgraph->nodes[gate_idx + 3], gate_scale_n, gate_n); - const ggml_tensor * up_scale = get_mul_mat_id_scale(cgraph->nodes[up_idx + 1], cgraph->nodes[up_idx + 2], + const ggml_tensor * up_scale = ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[up_idx + 1], cgraph->nodes[up_idx + 2], cgraph->nodes[up_idx + 3], up_scale_n, up_n); if (!gate_scale || !up_scale) { continue; @@ -3953,12 +4777,27 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.glu_op = ggml_get_glu_op(glu); fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) { + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); fused_mul_mat_vec = true; fused_node_count = n_ops; break; } + + if (!with_bias && ggml_cuda_should_fuse_mul_mat_q(up_n, gate_n, glu, up_scale_n, gate_scale_n)) { + fusion_data.prec_src1 = ggml_prec_get_src(up_n, 1); + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } } if (fused_mul_mat_vec) { @@ -4002,7 +4841,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph const ggml_tensor * src1 = up_n->src[1]; const ggml_tensor * ids = up_n->src[2]; - if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n)) { + if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) { ggml_cuda_mm_fusion_args_host fusion_data{}; fusion_data.gate = gate_n->src[0]; fusion_data.x_bias = up_bias_tensor; @@ -4045,7 +4884,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph const ggml_tensor * src1 = up->src[1]; const ggml_tensor * ids = up->src[2]; - if (ggml_cuda_should_fuse_mul_mat_vec_f(up)) { + if (ggml_cuda_should_fuse_mul_mat_vec_f(up, gate)) { ggml_cuda_mm_fusion_args_host fusion_data{}; fusion_data.gate = gate->src[0]; fusion_data.glu_op = ggml_get_glu_op(glu); @@ -4068,6 +4907,19 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fused_node_count = 3; break; } + + if (ggml_cuda_should_fuse_mul_mat_q(up, gate, glu)) { + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate->src[0]; + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + fusion_data.prec_src1 = ggml_prec_get_src(up, 1); + + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = 3; + break; + } } } @@ -4123,9 +4975,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph const ggml_tensor * scale = nullptr; if (op == GGML_OP_MUL_MAT) { - scale = get_mul_mat_scale(scale_node, mm_node); + scale = ggml_cuda_get_mul_mat_scale(scale_node, mm_node); } else { - scale = get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], scale_node, mm_node); + scale = ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], scale_node, mm_node); } if (!scale) { continue; @@ -4150,6 +5002,36 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.x_bias = bias; fusion_data.x_scale = scale; + if (ggml_cuda_should_fuse_mul_mat_vec_f(mm_node)) { + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + + if (op == GGML_OP_MUL_MAT && !with_bias && src0->type == GGML_TYPE_F8_E4M3 && + ggml_get_op_params_i32(mm_node, 3) != GGML_PREC_F32 && + ggml_cuda_mul_mat_fp8(*cuda_ctx, src0, src1, scale_node, scale)) { + return 1; + } + + if (!with_bias && src0->type == GGML_TYPE_NVFP4 && src1->type == GGML_TYPE_F32 && mm_node->type == GGML_TYPE_F32) { + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + const bool is_id = op == GGML_OP_MUL_MAT_ID; + const int64_t ncols = is_id ? src1->ne[2] : src1->ne[1]; + const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(src0->type, cc) : + ggml_cuda_should_use_mmvq(src0->type, cc, ncols); + const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && + src0->view_src; + if (!use_mmvq && !bad_padding_clear && + ggml_cuda_should_use_mmq(src0->type, cc, ncols, /*n_experts =*/ is_id ? src0->ne[2] : 0)) { + fusion_data.prec_src1 = ggml_prec_get_src(mm_node, 1); + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); + return n_ops - 1; + } + } + if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) { ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); fused_mul_mat_vec = true; @@ -4274,6 +5156,11 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 1; } + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_DSV4_HC_POST }, { GGML_UNARY_OP_SIGMOID })) { + ggml_cuda_op_dsv4_hc_post(*cuda_ctx, cgraph->nodes[i + 3], node, cgraph->nodes[i + 2]); + return 3; + } + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE }, { GGML_UNARY_OP_TANH })) { ggml_cuda_op_softcap(*cuda_ctx, cgraph->nodes[i + 2], node); return 2; @@ -4293,6 +5180,13 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud ggml_cuda_concurrent_event * concurrent_event = nullptr; bool should_launch_concurrent_events = false; +#ifdef USE_CUDA_GRAPH + ggml_cuda_graph * graph = use_cuda_graph ? cuda_ctx->cuda_graph(graph_key) : nullptr; + const int chunk_size = ggml_cuda_graph_chunk_size(); + int chunk_end = chunk_size; + size_t n_chunks = 0; +#endif // USE_CUDA_GRAPH + const auto try_launch_concurrent_event = [&](const ggml_tensor * node) { if (stream_ctx.concurrent_events.find(node) != stream_ctx.concurrent_events.end()) { concurrent_event = &stream_ctx.concurrent_events[node]; @@ -4382,6 +5276,15 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; +#ifdef USE_CUDA_GRAPH + // fused nodes are skipped together, so a cut never splits a fusion + // a concurrent region must join before its capture can end + if (use_cuda_graph && chunk_size > 0 && i >= chunk_end && !is_concurrent_event_active) { + ggml_cuda_graph_launch_chunk(cuda_ctx, graph, n_chunks++); + CUDA_CHECK(cudaStreamBeginCapture(cuda_ctx->stream(), cudaStreamCaptureModeRelaxed)); + chunk_end = i + chunk_size; + } +#endif // USE_CUDA_GRAPH if (is_concurrent_event_active) { GGML_ASSERT(concurrent_event); @@ -4464,14 +5367,14 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } #ifdef USE_CUDA_GRAPH - ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); if (use_cuda_graph && cuda_graph_update_required) { // End CUDA graph capture - if (graph->graph != nullptr) { - CUDA_CHECK(cudaGraphDestroy(graph->graph)); - graph->graph = nullptr; - } + ggml_cuda_graph_launch_chunk(cuda_ctx, graph, n_chunks++); - CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &graph->graph)); + // drop the chunks left over from a longer graph + for (size_t i = n_chunks; i < graph->instances.size(); ++i) { + CUDA_CHECK(cudaGraphExecDestroy(graph->instances[i])); + } + graph->instances.resize(n_chunks); graph_evaluated_or_captured = true; // CUDA graph has been captured std::lock_guard<std::mutex> lock(ggml_cuda_lock); @@ -4483,16 +5386,11 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } } - if (use_cuda_graph) { - ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); - if (graph->instance == nullptr) { // Create executable graph from captured graph. - CUDA_CHECK(cudaGraphInstantiate(&graph->instance, graph->graph, NULL, NULL, 0)); - } - if (cuda_graph_update_required) { // Update graph executable - ggml_cuda_graph_update_executable(cuda_ctx, graph_key); + if (use_cuda_graph && !cuda_graph_update_required) { + // the chunks were launched during capture, otherwise launch them here + for (cudaGraphExec_t instance : graph->instances) { + CUDA_CHECK(cudaGraphLaunch(instance, cuda_ctx->stream())); } - // Launch graph - CUDA_CHECK(cudaGraphLaunch(graph->instance, cuda_ctx->stream())); #else GGML_UNUSED(graph_key); graph_evaluated_or_captured = true; @@ -4504,9 +5402,9 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) { ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); - if (graph->graph == nullptr) { + if (graph->instances.empty()) { if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_VOLTA) { - if (!graph->disable_due_to_gpu_arch) { + if (!graph->disable_due_to_gpu_arch && ggml_cuda_graph_debug_enabled()) { GGML_LOG_DEBUG("%s: disabling CUDA graphs due to GPU architecture\n", __func__); } graph->disable_due_to_gpu_arch = true; @@ -4533,7 +5431,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); if (graph->is_enabled()) { - const bool graph_compatible = ggml_cuda_graph_check_compability(cgraph); + // graph->uid is set only for a compatible graph (see ggml_cuda_graph_update_required), so a reused graph needs no new check + const bool graph_reused = cgraph->uid != 0 && cgraph->uid == graph->uid; + const bool graph_compatible = graph_reused || ggml_cuda_graph_check_compability(cgraph); if (graph_compatible) { const bool properties_changed = ggml_cuda_graph_update_required(cuda_ctx, cgraph); @@ -4541,7 +5441,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, // Warmup: need at least 2 calls with no property change on the 2nd call if (!properties_changed) { graph->warmup_complete = true; - GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__); + } use_cuda_graph = true; cuda_graph_update_required = true; } @@ -4551,10 +5453,12 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, if (properties_changed) { // Properties changed - reset warmup, execute directly until stable again graph->warmup_complete = false; - GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__); + } } else { use_cuda_graph = true; - cuda_graph_update_required = graph->instance == nullptr; + cuda_graph_update_required = graph->instances.empty(); } } } @@ -4604,7 +5508,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; - static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); + const bool disable_fusion = ggml_cuda_fusion_disabled(); auto add_alloc_deps = [&](size_t start, size_t last_node) { @@ -4644,6 +5548,46 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph } } for (int i = 0; i < cgraph->n_nodes; ++i) { + // Keep the inputs of the fused gate/up/GLU MMQ kernel alive until the GLU so that they cannot alias its output (under op offload the weights are buffer copies). + if (i + 2 < cgraph->n_nodes && cgraph->nodes[i + 2]->op == GGML_OP_GLU) { + ggml_tensor * glu = cgraph->nodes[i + 2]; + ggml_tensor * gate = glu->src[0]; + ggml_tensor * up = glu->src[1]; + const bool ok = (gate == cgraph->nodes[i] && up == cgraph->nodes[i + 1]) || (gate == cgraph->nodes[i + 1] && up == cgraph->nodes[i]); + if (ok && ggml_cuda_mul_mat_q_fusion_matches(up, gate, glu, cuda_ctx->device)) { + add_alloc_deps(i, i + 2); + i += 2; + continue; + } + } + // Same with the NVFP4 scales: MUL_MAT(gate), MUL(gate scale), MUL_MAT(up), MUL(up scale), GLU. + if (i + 4 < cgraph->n_nodes && cgraph->nodes[i + 4]->op == GGML_OP_GLU) { + ggml_tensor * glu = cgraph->nodes[i + 4]; + ggml_tensor * gate_scale = cgraph->nodes[i + 1]; + ggml_tensor * up_scale = cgraph->nodes[i + 3]; + if (glu->src[0] == gate_scale && glu->src[1] == up_scale && + ggml_cuda_mul_mat_q_fusion_matches(cgraph->nodes[i + 2], cgraph->nodes[i], glu, cuda_ctx->device, up_scale, gate_scale) && + ggml_cuda_get_mul_mat_scale(gate_scale, cgraph->nodes[i]) && ggml_cuda_get_mul_mat_scale(up_scale, cgraph->nodes[i + 2])) { + add_alloc_deps(i, i + 4); + i += 4; + continue; + } + } + // Same with per-expert NVFP4 scales: MUL_MAT_ID, RESHAPE, REPEAT, GET_ROWS, MUL for gate and for up, GLU. + if (i + 10 < cgraph->n_nodes && cgraph->nodes[i + 10]->op == GGML_OP_GLU) { + ggml_tensor * glu = cgraph->nodes[i + 10]; + ggml_tensor * gate_scale = cgraph->nodes[i + 4]; + ggml_tensor * up_scale = cgraph->nodes[i + 9]; + if (glu->src[0] == gate_scale && glu->src[1] == up_scale && + ggml_cuda_mul_mat_q_fusion_matches(cgraph->nodes[i + 5], cgraph->nodes[i], glu, cuda_ctx->device, up_scale, gate_scale) && + ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], gate_scale, cgraph->nodes[i]) && + ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[i + 6], cgraph->nodes[i + 7], cgraph->nodes[i + 8], up_scale, cgraph->nodes[i + 5])) { + add_alloc_deps(i, i + 10); + i += 10; + continue; + } + } + ggml_cuda_moe_weighted_reduction_match match; if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) { params->add_alloc_dep(params->user_data, const_cast<ggml_tensor *>(match.experts), match.dst); @@ -5024,7 +5968,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot } bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) { - if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) { + if (!ggml_cuda_register_host_enabled()) { return false; } @@ -5047,7 +5991,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) { } void ggml_backend_cuda_unregister_host_buffer(void * buffer) { - if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) { + if (!ggml_cuda_register_host_enabled()) { return; } @@ -5174,8 +6118,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); // Check if UMA is explicitly enabled via environment variable - bool uma_env = getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr; - bool is_uma = prop.integrated > 0 || uma_env; + bool is_uma = prop.integrated > 0 || ggml_cuda_unified_memory_enabled(); if (is_uma) { // For UMA systems (like DGX Spark), use system memory info @@ -5216,7 +6159,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back props->device_id = ctx->pci_bus_id.empty() ? nullptr : ctx->pci_bus_id.c_str(); ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); - bool host_buffer = getenv("GGML_CUDA_NO_PINNED") == nullptr; + const bool host_buffer = !ggml_cuda_pinned_memory_disabled(); #ifdef GGML_CUDA_NO_PEER_COPY bool events = false; #else @@ -5290,13 +6233,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g if (op->src[0]->type == GGML_TYPE_BF16 && ggml_get_unary_op(op) == GGML_UNARY_OP_XIELU) { return false; } - // TODO: should become: - //return ggml_is_contiguous_rows(op->src[0]); - return ggml_is_contiguous(op->src[0]); + return op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16; default: return false; } - break; case GGML_OP_GLU: switch (ggml_get_glu_op(op)) { case GGML_GLU_OP_REGLU: @@ -5310,11 +6250,13 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g (ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_CLAMP)) { return false; } + if (op->src[0]->type == GGML_TYPE_F16 && ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI) { + return false; + } return ggml_is_contiguous_1(op->src[0]); default: return false; } - break; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: { @@ -5326,7 +6268,16 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16 && !ggml_cuda_op_mul_mat_use_fwht(op)) { return false; } - if (op->op == GGML_OP_MUL_MAT_ID && ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { + const int cc_dev = ggml_cuda_info().devices[dev_ctx->device].cc; + const bool f32_src1_ok = GGML_CUDA_CC_IS_NVIDIA(cc_dev) && cc_dev < GGML_CUDA_CC_AMPERE; + const enum ggml_prec prec_src1 = ggml_prec_get_src(op, 1); + const bool fp8_id_mmvf = op->op == GGML_OP_MUL_MAT_ID && + a->type == GGML_TYPE_F8_E4M3 && b->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + ggml_cuda_should_use_mmvf(a, cc_dev, ggml_cuda_info().devices[dev_ctx->device].warp_size, b->ne[2]); + if (prec_src1 == GGML_PREC_F32 && !fp8_id_mmvf && (op->op == GGML_OP_MUL_MAT_ID || !f32_src1_ok || a->type != GGML_TYPE_F32)) { + return false; + } + if (prec_src1 == GGML_PREC_BF16 && (op->op == GGML_OP_MUL_MAT_ID || a->type != GGML_TYPE_F32)) { return false; } #ifdef GGML_USE_MUSA @@ -5354,6 +6305,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5374,7 +6326,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g default: return false; } - } break; + } case GGML_OP_OUT_PROD: return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32; case GGML_OP_GET_ROWS: @@ -5391,6 +6343,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5413,16 +6366,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g default: return false; } - } break; + } case GGML_OP_GET_ROWS_BACK: - { - return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1; - } break; + return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1; case GGML_OP_SET_ROWS: { return ( ( - (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_F8_E4M3 || op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && op->src[0]->type == GGML_TYPE_F32 @@ -5431,18 +6382,28 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g ) ) && (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); - } break; + } case GGML_OP_SET: { const ggml_type t = op->type; return (t == GGML_TYPE_F32 || t == GGML_TYPE_I32) && t == op->src[0]->type && t == op->src[1]->type; - } break; + } case GGML_OP_CPY: { ggml_type src0_type = op->src[0]->type; ggml_type src1_type = op->src[1]->type; + if (src0_type == GGML_TYPE_F8_E4M3 && src1_type == GGML_TYPE_F8_E4M3) { + return true; + } + if ((src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16 || src0_type == GGML_TYPE_BF16) && + src1_type == GGML_TYPE_F8_E4M3) { + return true; + } + if (src0_type == GGML_TYPE_F8_E4M3 && src1_type == GGML_TYPE_F32) { + return true; + } if ((src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16 || src0_type == GGML_TYPE_F16) && (src1_type == GGML_TYPE_F32 || src1_type == GGML_TYPE_BF16 || src1_type == GGML_TYPE_F16) ) { @@ -5494,20 +6455,18 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return true; } return false; - } break; + } case GGML_OP_DUP: return true; case GGML_OP_ARGMAX: case GGML_OP_COUNT_EQUAL: - { - return true; - } break; + return true; case GGML_OP_REPEAT: { // the CUDA REPEAT path only implements F32/F16; other types assert at runtime ggml_type src0_type = op->src[0]->type; return src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16; - } break; + } case GGML_OP_REPEAT_BACK: return op->type == GGML_TYPE_F32 && (op->src[0]->ne[2]*op->src[0]->ne[3]) <= (1 << 15); case GGML_OP_CONCAT: @@ -5544,7 +6503,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g ) ) ); - } break; + } case GGML_OP_CONV_TRANSPOSE_1D: { ggml_type src0_type = op->src[0]->type; @@ -5553,7 +6512,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return true; } return false; - } break; + } case GGML_OP_COL2IM_1D: { ggml_type src0_type = op->src[0]->type; @@ -5561,33 +6520,32 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g op->type == src0_type && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op); - } break; + } case GGML_OP_SILU_BACK: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; - break; case GGML_OP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_L2_NORM: return ggml_is_contiguous_rows(op->src[0]); case GGML_OP_RMS_NORM_BACK: return ggml_is_contiguous(op->src[0]); - break; case GGML_OP_NONE: case GGML_OP_RESHAPE: case GGML_OP_VIEW: case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: case GGML_OP_ADD_ID: - case GGML_OP_ADD1: case GGML_OP_SQR: case GGML_OP_SQRT: case GGML_OP_SIN: case GGML_OP_COS: - case GGML_OP_CLAMP: case GGML_OP_LOG: return true; case GGML_OP_SCALE: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type; + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type && + ggml_is_contiguous_rows(op->src[0]); + case GGML_OP_CLAMP: + return ggml_is_contiguous_rows(op->src[0]); case GGML_OP_ADD: case GGML_OP_SUB: case GGML_OP_MUL: @@ -5604,9 +6562,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g if (op->src[3]->ne[0] == 1) { // Mamba2 - // (kernel only supports (d_state == 96 || d_state == 128 || d_state == 256) && d_head % 16 == 0) + // (kernel only supports (d_state == 64 || d_state == 96 || d_state == 128 || d_state == 256) && d_head % 16 == 0) const int64_t d_state = op->src[0]->ne[0]; - return (d_state == 96 || d_state == 128 || d_state == 256) && op->src[0]->ne[1] % 16 == 0; + return (d_state == 64 || d_state == 96 || d_state == 128 || d_state == 256) && op->src[0]->ne[1] % 16 == 0; } else { if (K > 1) { return false; @@ -5623,8 +6581,6 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g } case GGML_OP_CONT: return true; - case GGML_OP_DIAG_MASK_INF: - return true; case GGML_OP_SOFT_MAX: return true; case GGML_OP_SOFT_MAX_BACK: { @@ -5633,7 +6589,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return max_bias == 0.0f; } case GGML_OP_ROLL: - if(op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0])) { + if(op->src[0]->type == GGML_TYPE_F32) { return true; } return false; @@ -5661,13 +6617,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g //return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]); return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); case GGML_OP_SUM: - return ggml_is_contiguous_rows(op->src[0]); + return ggml_is_contiguously_allocated(op->src[0]); case GGML_OP_TOP_K: -#if defined(GGML_USE_HIP) || defined(GGML_CUDA_USE_CUB) - return true; -#else - return op->src[0]->ne[0] <= 1024; -#endif // defined(GGML_USE_HIP) || defined(GGML_CUDA_USE_CUB) + return op->src[0]->ne[0] <= INT_MAX; case GGML_OP_ARGSORT: #ifndef GGML_CUDA_USE_CUB { @@ -5679,7 +6631,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return ncols_pad * sizeof(int) <= ggml_cuda_info().devices[dev_ctx->device].smpb; } #else - return true; + return op->src[0]->ne[0] <= INT_MAX; #endif case GGML_OP_SUM_ROWS: return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); @@ -5724,6 +6676,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return true; case GGML_OP_LIGHTNING_INDEXER: return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op); + case GGML_OP_TOP_K_MASK: + return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32; default: return false; @@ -5864,12 +6818,21 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t { const auto & info = ggml_cuda_info(); + bool has_fp8 = false; + bool has_fp4 = false; for (int id = 0; id < info.device_count; ++id) { - if (blackwell_mma_available(info.devices[id].cc)) { - features.push_back({ "BLACKWELL_NATIVE_FP4", "1"}); + has_fp8 |= fp8_mma_hardware_available(info.devices[id].cc); + has_fp4 |= blackwell_mma_available(info.devices[id].cc); + if (has_fp8 && has_fp4) { break; } } + if (has_fp8) { + features.push_back({ "NATIVE_FP8", "1"}); + } + if (has_fp4) { + features.push_back({ "BLACKWELL_NATIVE_FP4", "1"}); + } } #undef _STRINGIFY @@ -5896,6 +6859,11 @@ static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, con if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { return (void *)ggml_backend_cuda_comm_allreduce_tensor; } +#ifdef GGML_USE_NCCL + if (strcmp(name, "ggml_backend_comm_init_rank") == 0) { + return (void *)ggml_backend_cuda_comm_init_rank; + } +#endif // GGML_USE_NCCL if (strcmp(name, "ggml_backend_register_host_buffer") == 0) { return (void *)ggml_backend_cuda_register_host_buffer; } @@ -5925,7 +6893,7 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { std::lock_guard<std::mutex> lock(mutex); if (!initialized) { ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context; - const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32; + const int min_batch_size = ggml_cuda_op_offload_min_batch(); const ggml_cuda_device_info & info = ggml_cuda_info(); const bool virtual_devices = info.device_count > info.physical_device_count; diff --git a/ggml/src/ggml-cuda/graph.cuh b/ggml/src/ggml-cuda/graph.cuh new file mode 100644 index 000000000..7b9c27789 --- /dev/null +++ b/ggml/src/ggml-cuda/graph.cuh @@ -0,0 +1,221 @@ +#pragma once + +// CUDA graph capture and concurrent stream state, only needed by ggml-cuda.cu + +#include "common.cuh" + +#include <algorithm> +#include <unordered_map> +#include <utility> +#include <vector> + +#ifdef USE_CUDA_GRAPH +// cudaGraphExecUpdate keeps stale kernel cluster dims (cuBLAS sm90), re-instantiate when they change +// TODO: add a test, fails on H100 (driver 590.48.01, CUDA runtime 12.9.79, cuBLAS 12.9.1.4) for F16 MUL_MAT with +// ne01 1200 <-> 1040 (k=768, n=1024) between graph updates, see https://github.com/ggml-org/llama.cpp/pull/31284 +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 +#define USE_CUDA_GRAPH_CLUSTER_CHECK +#endif + +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK +// cluster dims of a clustered kernel node +struct ggml_cuda_graph_cluster { + size_t node; + uint32_t x, y, z; + + bool operator==(const ggml_cuda_graph_cluster & other) const { + return node == other.node && x == other.x && y == other.y && z == other.z; + } +}; +#endif + +struct ggml_cuda_graph_instance { + cudaGraphExec_t handle = nullptr; +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK + std::vector<ggml_cuda_graph_cluster> clusters; +#endif + + operator cudaGraphExec_t() const { return handle; } +}; +#endif // USE_CUDA_GRAPH + +struct ggml_cuda_graph { +#ifdef USE_CUDA_GRAPH + ~ggml_cuda_graph() { + for (cudaGraphExec_t instance : instances) { + CUDA_CHECK(cudaGraphExecDestroy(instance)); + } + } + // the ggml graph is captured as a series of CUDA graphs, so the GPU can run one chunk while the next is enqueued + std::vector<ggml_cuda_graph_instance> instances; + size_t num_nodes = 0; + std::vector<cudaGraphNode_t> nodes; + bool disable_due_to_gpu_arch = false; + bool warmup_complete = false; + uint64_t uid = 0; + int64_t last_used_time = 0; + struct node_properties { + ggml_tensor node; + void * node_src_data_ptrs[GGML_MAX_SRC]; + int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; + size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; + }; + std::vector<node_properties> node_props; + + bool is_enabled() const { + static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); + return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env || ggml_cuda_pool_debug_enabled()); + } +#endif +}; + +struct ggml_cuda_concurrent_event { + std::vector<cudaEvent_t> join_events; + cudaEvent_t fork_event = nullptr; + + int n_streams = 0; + std::unordered_map<const ggml_tensor *, int> stream_mapping; + + // Original order of nodes in this concurrent region (before interleaving) + // Used to restore grouping for fusion within streams + std::vector<const ggml_tensor *> original_order; + + const ggml_tensor * join_node; + + ggml_cuda_concurrent_event() = default; + + ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete; + ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete; + + explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) { + join_events.resize(n_streams); + + for (size_t i = 0; i < join_events.size(); ++i) { + CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming)); + } + + CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming)); + } + + ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept + : join_events(std::move(other.join_events)) + , fork_event(other.fork_event) + , n_streams(other.n_streams) + , stream_mapping(std::move(other.stream_mapping)) + , original_order(std::move(other.original_order)) + , join_node(other.join_node) { + other.fork_event = nullptr; + } + + // 1. check if any branches write to overlapping memory ranges (except the join node) + // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event + // we assume all nodes have the same buffer + bool is_valid() const { + std::vector<std::vector<std::pair<int64_t, int64_t>>> write_ranges; + write_ranges.resize(n_streams); + + // get join_node's memory range to exclude from overlap checking. + // multiple nodes can use join_node's buffer; we synchronize on the join node. + const ggml_tensor * join_t = join_node->view_src ? join_node->view_src : join_node; + const int64_t join_start = (int64_t) join_t->data; + const int64_t join_end = join_start + ggml_nbytes(join_t); + + for (const auto & [tensor, stream] : stream_mapping) { + const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; + const int64_t t_start = (int64_t) t->data; + const int64_t t_end = t_start + ggml_nbytes(t); + + // skip tensors that overlap with join_node's buffer. + if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { + continue; + } + + // concurrent streams begin from 1 + write_ranges[stream - 1].emplace_back(t_start, t_end); + } + + for (int i = 0; i < n_streams; ++i) { + // sorts first by start then by end of write range + std::sort(write_ranges[i].begin(), write_ranges[i].end()); + } + + bool writes_overlap = false; + bool dependent_srcs = false; + for (const auto & [tensor, stream] : stream_mapping) { + const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; + const int64_t t_start = (int64_t) t->data; + const int64_t t_end = t_start + ggml_nbytes(t); + + // skip tensors that overlap with join_node's buffer + if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { + continue; + } + + // check if this buffer's write data overlaps with another stream's + std::pair<int64_t, int64_t> data_range = std::make_pair(t_start, t_end); + for (int i = 0; i < n_streams; ++i) { + if (i == stream - 1) { + continue; + } + auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range); + + if (it != write_ranges[i].end()) { + const std::pair<int64_t, int64_t> & other = *it; + + // std::lower_bound returns the first element where other >= data_range (lexicographically). + // This guarantees other.first >= data_range.first. + // Therefore, overlap occurs iff other.first < data_range.second + // (i.e., the other range starts before this range ends). + if (other.first < data_range.second) { + GGML_LOG_DEBUG("Writes overlap for %s", tensor->name); + writes_overlap = true; + break; + } + } + } + + //check if all srcs are either in branch or don't have a branch + for (int i = 0; i < GGML_MAX_SRC; ++i) { + if (!tensor->src[i]) { + continue; + } + + auto it = stream_mapping.find(tensor->src[i]); + + if (it == stream_mapping.end()) { + continue; + } + + if (it->second != stream) { + dependent_srcs = true; + break; + } + } + + if (dependent_srcs || writes_overlap) { + break; + } + } + + return !writes_overlap && !dependent_srcs; + } + + ~ggml_cuda_concurrent_event() { + if (fork_event != nullptr) { + CUDA_CHECK(cudaEventDestroy(fork_event)); + } + for (cudaEvent_t e : join_events) { + if (e != nullptr) { + CUDA_CHECK(cudaEventDestroy(e)); + } + } + } +}; + +struct ggml_cuda_stream_context { + std::unordered_map<const ggml_tensor *, ggml_cuda_concurrent_event> concurrent_events; + + void reset() { + concurrent_events.clear(); + } +}; diff --git a/ggml/src/ggml-cuda/lightning-indexer.cu b/ggml/src/ggml-cuda/lightning-indexer.cu index 57cb1c1dc..b7939243b 100644 --- a/ggml/src/ggml-cuda/lightning-indexer.cu +++ b/ggml/src/ggml-cuda/lightning-indexer.cu @@ -2,21 +2,22 @@ #include "lightning-indexer.cuh" #include "fattn-common.cuh" #include "convert.cuh" +#include "mma.cuh" -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) -#if defined(TURING_MMA_AVAILABLE) +// tokens scored per block by the tile kernel +#define LIGHTNING_INDEXER_TILE_TOKENS 8 -typedef union { - int2 i2; - half2 h2[2]; -} half4; +// heads whose queries the tile kernel stages per pass, MUSA arch 21 caps static shared memory +// at 28 KB and the queries of four heads do not fit there next to the key tile +#if defined(GGML_USE_MUSA) && defined(__MUSA_ARCH__) && __MUSA_ARCH__ < 220 +#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 2 +#else +#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 4 +#endif -// TODO add support for AMD cards via rocWMMA -#include <mma.h> -namespace wmma = nvcuda::wmma; template <int WARPS_PER_BLOCK, int K_VECS_PER_BLOCK, int64_t N_EMBD, int64_t N_HEAD, ggml_type TYPE_K> -static __global__ void lightning_indexer_kernel_wmma( +static __global__ void lightning_indexer_kernel_mma( const float * Q, const char * K, const float * W, const half * M, float * dst, int64_t n_stream, int64_t n_batch, int64_t n_kv, size_t nb1, size_t nb2, size_t nb3, @@ -26,219 +27,161 @@ static __global__ void lightning_indexer_kernel_wmma( size_t nbm1, size_t nbm2, size_t nbm3, int64_t nem3 ) { - - constexpr int THREADS_PER_BLOCK = WARPS_PER_BLOCK * WARP_SIZE; - constexpr int HEADS_PER_INNER_LOOP = 8; - constexpr int K_EMBD_PER_INNER_LOOP = 16; + constexpr int MMA_DIM = 16; + constexpr int WAVE_SIZE = ggml_cuda_get_physical_warp_size(); + constexpr int THREADS_PER_BLOCK = WARPS_PER_BLOCK * WAVE_SIZE; constexpr int N_EMBD_PADDED = N_EMBD + 8; + constexpr int K_VECS_PADDED = K_VECS_PER_BLOCK + 8; + constexpr int HEAD_CHUNK = MMA_DIM; + constexpr int N_HEAD_CHUNKS = N_HEAD / HEAD_CHUNK; + constexpr int N_KV_TILES = K_VECS_PER_BLOCK / MMA_DIM; + constexpr int N_EMBD_TILES = N_EMBD / MMA_DIM; + + static_assert(N_HEAD % MMA_DIM == 0, "N_HEAD must be a multiple of 16"); + static_assert(N_EMBD % MMA_DIM == 0, "N_EMBD must be a multiple of 16"); + static_assert(K_VECS_PER_BLOCK % MMA_DIM == 0, "K_VECS_PER_BLOCK must be a multiple of 16"); + +#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + using tile_qk = ggml_cuda_mma::tile<MMA_DIM, MMA_DIM, float, ggml_cuda_mma::DATA_LAYOUT_J_MAJOR>; +#else + using tile_qk = ggml_cuda_mma::tile<MMA_DIM, MMA_DIM, float>; +#endif const int i_batch = blockIdx.y; const int i_stream = blockIdx.z; const int i_warp = threadIdx.y; const int i_lane = threadIdx.x; - const int tid = i_warp * WARP_SIZE + i_lane; + const int tid = i_warp * WAVE_SIZE + i_lane; - // each block processes K_VECS_PER_BLOCK K vectors const int start_kv = blockIdx.x * K_VECS_PER_BLOCK; const char * q_base = (const char *) Q + i_batch*nbq2 + i_stream*nbq3; const float * w_base = (const float *) ((const char *) W + i_batch*nbw1 + i_stream*nbw3); - // phase 1 - load weights and first Q tile to shared memory - __shared__ float w_shared[N_HEAD]; - __shared__ int2 q_shared_h[HEADS_PER_INNER_LOOP][N_EMBD_PADDED / 4]; + __shared__ half q_shared[2][HEAD_CHUNK][N_EMBD_PADDED]; + __shared__ half k_shared[K_VECS_PER_BLOCK][N_EMBD_PADDED]; + __shared__ float qk_chunk[HEAD_CHUNK][K_VECS_PADDED]; + __shared__ float score_shared[K_VECS_PER_BLOCK]; - if (tid < N_HEAD) { - w_shared[tid] = w_base[tid]; + for (int i = tid; i < N_HEAD; i += THREADS_PER_BLOCK) { + w_shared[i] = w_base[i]; } - - // total number of half4 elements in HEADS_PER_INNER_LOOP x N_EMBD Q tile - constexpr int N_Q_TILE = HEADS_PER_INNER_LOOP * (N_EMBD / 4); - // number of registers needed in each thread to store Q tile in thread block - constexpr int N_Q_NEXT = (N_Q_TILE + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; - -#pragma unroll - for (int i_q = tid; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { - const int i_head = i_q / (N_EMBD / 4); - const int i_embd = i_q % (N_EMBD / 4); - const float4 q = *(const float4 *) (q_base + i_head*nbq1 + i_embd*sizeof(float4)); - half4 q_packed; - q_packed.h2[0] = __float22half2_rn(make_float2(q.x, q.y)); - q_packed.h2[1] = __float22half2_rn(make_float2(q.z, q.w)); - q_shared_h[i_head][i_embd] = q_packed.i2; + for (int i = tid; i < K_VECS_PER_BLOCK; i += THREADS_PER_BLOCK) { + score_shared[i] = 0.0f; } - // phase 2 - load (and dequantize if needed) K to shared mem - - __shared__ half2 k_shared_h[K_VECS_PER_BLOCK][N_EMBD_PADDED / 4][2]; - - constexpr int n_k = K_VECS_PER_BLOCK * (N_EMBD / 4); - + constexpr int N_K_VEC = K_VECS_PER_BLOCK * (N_EMBD / 4); if constexpr (TYPE_K == GGML_TYPE_F16) { -#pragma unroll - for (int i_k = tid; i_k < n_k; i_k += THREADS_PER_BLOCK) { - const int i_k_vec = i_k / (N_EMBD / 4); - const int i_embd = i_k % (N_EMBD / 4); - const int i_kv = start_kv + i_k_vec; + for (int i = tid; i < N_K_VEC; i += THREADS_PER_BLOCK) { + const int i_k_vec = i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + const int i_kv = start_kv + i_k_vec; if (i_kv < n_kv) { - const int2 * k_base = (const int2 *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); - *(int2*) &k_shared_h[i_k_vec][i_embd] = k_base[i_embd]; + const int2 * k_ptr = (const int2 *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); + *(int2 *) &k_shared[i_k_vec][i_embd4*4] = k_ptr[i_embd4]; } else { - *(int2*) &k_shared_h[i_k_vec][i_embd] = make_int2(0, 0); + *(int2 *) &k_shared[i_k_vec][i_embd4*4] = make_int2(0, 0); } } } else { constexpr dequantize_V_t dequantize_k = get_dequantize_V<TYPE_K, half, 4>(); -#pragma unroll - for (int i_k = tid; i_k < n_k; i_k += THREADS_PER_BLOCK) { - const int i_k_vec = i_k / (N_EMBD / 4); - const int i_embd = i_k % (N_EMBD / 4); - const int i_kv = start_kv + i_k_vec; + for (int i = tid; i < N_K_VEC; i += THREADS_PER_BLOCK) { + const int i_k_vec = i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + const int i_kv = start_kv + i_k_vec; if (i_kv < n_kv) { - const void * k_base = (const void *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); - dequantize_k(k_base, &k_shared_h[i_k_vec][i_embd][0], i_embd * 4); + const void * k_ptr = (const void *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); + dequantize_k(k_ptr, &k_shared[i_k_vec][i_embd4*4], i_embd4*4); } else { - *(int2*) &k_shared_h[i_k_vec][i_embd] = make_int2(0, 0); + *(int2 *) &k_shared[i_k_vec][i_embd4*4] = make_int2(0, 0); } } } - __syncthreads(); - - // phase 3 - calculate lightning indexer scores + constexpr int N_Q_CHUNK_VEC = HEAD_CHUNK * (N_EMBD / 4); + constexpr int N_Q_NEXT = (N_Q_CHUNK_VEC + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; - __shared__ float qk_shared[WARPS_PER_BLOCK][HEADS_PER_INNER_LOOP][K_VECS_PER_BLOCK]; - - // load K fragment - wmma::fragment<wmma::matrix_b, HEADS_PER_INNER_LOOP, K_VECS_PER_BLOCK, K_EMBD_PER_INNER_LOOP, half, wmma::col_major> frag_k; - wmma::load_matrix_sync(frag_k, (half*) &k_shared_h[0][i_warp * K_EMBD_PER_INNER_LOOP / 4], N_EMBD_PADDED); - - float score_k = 0.0f; + for (int i = tid; i < N_Q_CHUNK_VEC; i += THREADS_PER_BLOCK) { + const int i_head = i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + const float4 q = *(const float4 *) (q_base + i_head*nbq1 + i_embd4*sizeof(float4)); + const half2 qh[2] = {make_half2(q.x, q.y), make_half2(q.z, q.w)}; + ggml_cuda_memcpy_1<sizeof(qh)>(&q_shared[0][i_head][i_embd4*4], qh); + } - for (int i_head_0 = 0; i_head_0 < N_HEAD; i_head_0 += HEADS_PER_INNER_LOOP) { - const int i_head_next = i_head_0 + HEADS_PER_INNER_LOOP; + __syncthreads(); - // we don't use accumulator for anything, fill it with zeros - wmma::fragment<wmma::accumulator, HEADS_PER_INNER_LOOP, K_VECS_PER_BLOCK, K_EMBD_PER_INNER_LOOP, float> frag_acc; - wmma::fill_fragment(frag_acc, 0.0f); + constexpr ggml_cuda_mma::data_layout input_layout = ggml_cuda_mma::get_input_data_layout(); - // load Q fragment - wmma::fragment<wmma::matrix_a, HEADS_PER_INNER_LOOP, K_VECS_PER_BLOCK, K_EMBD_PER_INNER_LOOP, half, wmma::row_major> frag_q; - wmma::load_matrix_sync(frag_q, (half*) &q_shared_h[0][i_warp * K_EMBD_PER_INNER_LOOP / 4], N_EMBD_PADDED); + for (int hc = 0; hc < N_HEAD_CHUNKS; ++hc) { + const int buf = hc & 1; + const int nbuf = (hc + 1) & 1; + const bool has_next = (hc + 1) < N_HEAD_CHUNKS; - // preload next Q tile to registers during matrix multiplication float4 q_next[N_Q_NEXT]; - - if (i_head_next < N_HEAD) { + if (has_next) { #pragma unroll - for (int i_q = tid, i_q_next = 0; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { - const int i_head = i_head_next + i_q / (N_EMBD / 4); - const int i_embd = i_q % (N_EMBD / 4); - q_next[i_q_next++] = *(const float4 *) (q_base + i_head*nbq1 + i_embd*sizeof(float4)); + for (int i = tid, j = 0; i < N_Q_CHUNK_VEC; i += THREADS_PER_BLOCK, ++j) { + const int i_head = (hc + 1) * HEAD_CHUNK + i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + q_next[j] = *(const float4 *) (q_base + i_head*nbq1 + i_embd4*sizeof(float4)); } } - // perform matrix multiplication - wmma::mma_sync(frag_acc, frag_q, frag_k, frag_acc); - wmma::store_matrix_sync((float*) &qk_shared[i_warp][0][0], frag_acc, K_VECS_PER_BLOCK, wmma::mem_row_major); - - // make sure all threads finished using q_shared_h so we can store next tile - __syncthreads(); - - // write preloaded Q tile to shared memory - if (i_head_next < N_HEAD) { + for (int kt = i_warp; kt < N_KV_TILES; kt += WARPS_PER_BLOCK) { + tile_qk Dqk; #pragma unroll - for (int i_q = tid, i_q_next = 0; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { - const int i_head = i_q / (N_EMBD / 4); - const int i_embd = i_q % (N_EMBD / 4); - half4 q_packed; - q_packed.h2[0] = __float22half2_rn(make_float2(q_next[i_q_next].x, q_next[i_q_next].y)); - q_packed.h2[1] = __float22half2_rn(make_float2(q_next[i_q_next].z, q_next[i_q_next].w)); - q_shared_h[i_head][i_embd] = q_packed.i2; - ++i_q_next; + for (int et = 0; et < N_EMBD_TILES; ++et) { + ggml_cuda_mma::tile<MMA_DIM, MMA_DIM/2, half2, input_layout> Aq; + ggml_cuda_mma::tile<MMA_DIM, MMA_DIM/2, half2, input_layout> Bk; + ggml_cuda_mma::load_generic(Aq, (const half2 *) &q_shared[buf][0][et*MMA_DIM], N_EMBD_PADDED/2); + ggml_cuda_mma::load_generic(Bk, (const half2 *) &k_shared[kt*MMA_DIM][et*MMA_DIM], N_EMBD_PADDED/2); + ggml_cuda_mma::mma(Dqk, Aq, Bk); } - } - - // accumulate QK multiplication results from all block warps - // (there are 256 threads in block and 256 matmul outputs) - // TODO it will break if WARP_SIZE is not 32 - const int h = tid / K_VECS_PER_BLOCK; - const int k = tid % K_VECS_PER_BLOCK; - const float w_val = w_shared[i_head_0 + h]; - - float sum = 0.0f; #pragma unroll - for (int w = 0; w < WARPS_PER_BLOCK; ++w) { - sum += qk_shared[w][h][k]; + for (int l = 0; l < Dqk.ne; ++l) { + const int head_local = Dqk.get_i(l); + const int kv_local = kt*MMA_DIM + Dqk.get_j(l); + qk_chunk[head_local][kv_local] = Dqk.x[l]; + } } - // ReLU, weight - sum = sum > 0.0f ? sum : 0.0f; - sum *= w_val; - - // wait until qk_shared[0] is no longer used __syncthreads(); - // reuse qk_shared[0] for storing partial results - qk_shared[0][h][k] = sum; - - // wait until all threads write their results - __syncthreads(); + if (has_next) { +#pragma unroll + for (int i = tid, j = 0; i < N_Q_CHUNK_VEC; i += THREADS_PER_BLOCK, ++j) { + const int i_head = i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + const half2 qh[2] = {make_half2(q_next[j].x, q_next[j].y), make_half2(q_next[j].z, q_next[j].w)}; + ggml_cuda_memcpy_1<sizeof(qh)>(&q_shared[nbuf][i_head][i_embd4*4], qh); + } + } - // accumulate result over heads - if (tid < K_VECS_PER_BLOCK) { + for (int kv = tid; kv < K_VECS_PER_BLOCK; kv += THREADS_PER_BLOCK) { + float s = 0.0f; #pragma unroll - for (int i_head = 0; i_head < HEADS_PER_INNER_LOOP; ++i_head) { - score_k += qk_shared[0][i_head][tid]; + for (int h = 0; h < HEAD_CHUNK; ++h) { + const float qk = qk_chunk[h][kv]; + s += (qk > 0.0f ? qk : 0.0f) * w_shared[hc*HEAD_CHUNK + h]; } + score_shared[kv] += s; } - // make sure all threads finished using qk_shared __syncthreads(); } - // phase 4 - store output to VRAM - - if (tid < K_VECS_PER_BLOCK) { - const int i_kv = start_kv + tid; + for (int i_kv_local = tid; i_kv_local < K_VECS_PER_BLOCK; i_kv_local += THREADS_PER_BLOCK) { + const int i_kv = start_kv + i_kv_local; if (i_kv < n_kv) { - const half * m_base = (const half *) ((const char *) M + i_batch*nbm1 + (i_stream%nem3)*nbm3); - float * dst_base = (float *) ((char *) dst + i_batch*nb1 + i_stream*nb3); - dst_base[i_kv] = score_k + __half2float(m_base[i_kv]); + const half * m_base = (const half *) ((const char *) M + i_batch*nbm1 + (i_stream%nem3)*nbm3); + float * dst_base = (float *) ((char *) dst + i_batch*nb1 + i_stream*nb3); + dst_base[i_kv] = score_shared[i_kv_local] + __half2float(m_base[i_kv]); } } } -#else // defined(TURING_MMA_AVAILABLE) - -template <int WARPS_PER_BLOCK, int K_VECS_PER_BLOCK, int64_t N_EMBD, int64_t N_HEAD, ggml_type TYPE_K> -static __global__ void lightning_indexer_kernel_wmma( - const float * Q, const char * K, const float * W, const half * M, float * dst, - int64_t n_stream, int64_t n_batch, int64_t n_kv, - size_t nb1, size_t nb2, size_t nb3, - size_t nbq1, size_t nbq2, size_t nbq3, - size_t nbk1, size_t nbk2, size_t nbk3, - size_t nbw1, size_t nbw2, size_t nbw3, - size_t nbm1, size_t nbm2, size_t nbm3, - int64_t nem3 - ) { - GGML_UNUSED_VARS(Q, K, W, M, dst, - n_stream, n_batch, n_kv, - nb1, nb2, nb3, - nbq1, nbq2, nbq3, - nbk1, nbk2, nbk3, - nbw1, nbw2, nbw3, - nem3); - NO_DEVICE_CODE; -} - -#endif // defined(TURING_MMA_AVAILABLE) -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - -// tokens scored per block by the tile kernel -#define LIGHTNING_INDEXER_TILE_TOKENS 8 - // TODO there is one ugly assumption used in this kernel - that WARP_SIZE is equal to 32 // thanks to that one warp operating on float4 processes whole indexer K/Q vectors // 32 * 4 = 128 (N_EMBD) @@ -406,9 +349,11 @@ static __global__ void lightning_indexer_kernel_tile( constexpr int KEY_LANES = THREADS_PER_BLOCK / TOKENS_PER_BLOCK; constexpr int KEYS_PER_THREAD = K_VECS_PER_BLOCK / KEY_LANES; constexpr int N_EMBD_H2 = N_EMBD / 2; + constexpr int HEADS_PER_PASS = N_HEAD < LIGHTNING_INDEXER_TILE_HEADS_PER_PASS ? N_HEAD : LIGHTNING_INDEXER_TILE_HEADS_PER_PASS; static_assert(THREADS_PER_BLOCK % TOKENS_PER_BLOCK == 0, "threads must cover the token tile"); static_assert(K_VECS_PER_BLOCK % KEY_LANES == 0, "key lanes must cover the key tile"); + static_assert(N_HEAD % HEADS_PER_PASS == 0, "head passes must cover the heads"); const int tid = threadIdx.y * WARP_SIZE + threadIdx.x; const int start_kv = blockIdx.x * K_VECS_PER_BLOCK; @@ -417,7 +362,7 @@ static __global__ void lightning_indexer_kernel_tile( // the row padding keeps the keys of consecutive threads in distinct banks __shared__ half2 k_shared[K_VECS_PER_BLOCK][N_EMBD_H2 + 1]; - __shared__ float2 q_shared[N_HEAD][TOKENS_PER_BLOCK][N_EMBD_H2]; + __shared__ float2 q_shared[HEADS_PER_PASS][TOKENS_PER_BLOCK][N_EMBD_H2]; __shared__ float w_shared[N_HEAD][TOKENS_PER_BLOCK]; // phase 1 - stage the key tile four elements at a time, rows past n_kv are zero @@ -451,22 +396,7 @@ static __global__ void lightning_indexer_kernel_tile( k_shared[r][2*c4 + 1] = hi; } - // phase 2 - stage the queries and weights of every head, tokens past n_batch are zero - -#pragma unroll - for (int i = tid; i < N_HEAD * TOKENS_PER_BLOCK * (N_EMBD / 4); i += THREADS_PER_BLOCK) { - const int h = i / (TOKENS_PER_BLOCK * (N_EMBD / 4)); - const int r = i / (N_EMBD / 4) % TOKENS_PER_BLOCK; - const int c4 = i % (N_EMBD / 4); - - float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f); - if (start_batch + r < n_batch) { - v = *(const float4 *) ((const char *) Q + h*nbq1 + (start_batch + r)*nbq2 + i_stream*nbq3 + c4*sizeof(float4)); - } - - q_shared[h][r][2*c4 + 0] = make_float2(v.x, v.y); - q_shared[h][r][2*c4 + 1] = make_float2(v.z, v.w); - } + // phase 2 - stage the weights of every head, tokens past n_batch are zero if (tid < N_HEAD * TOKENS_PER_BLOCK) { const int h = tid / TOKENS_PER_BLOCK; @@ -475,33 +405,60 @@ static __global__ void lightning_indexer_kernel_tile( ((const float *) ((const char *) W + (start_batch + r)*nbw1 + i_stream*nbw3))[h] : 0.0f; } - __syncthreads(); - - // phase 3 - float products of the widened keys for every head, ReLU, weight - const int kl = tid % KEY_LANES; const int tl = tid / KEY_LANES; float qk[N_HEAD][KEYS_PER_THREAD] = { { 0.0f } }; -#pragma unroll 8 - for (int c = 0; c < N_EMBD_H2; ++c) { - float2 k_val[KEYS_PER_THREAD]; #pragma unroll - for (int j = 0; j < KEYS_PER_THREAD; ++j) { - k_val[j] = __half22float2(k_shared[kl + j*KEY_LANES][c]); + for (int h0 = 0; h0 < N_HEAD; h0 += HEADS_PER_PASS) { + // the previous pass is fully consumed before its queries are replaced + if (h0 > 0) { + __syncthreads(); } + + // phase 3 - stage the queries of the heads of this pass, tokens past n_batch are zero + #pragma unroll - for (int h = 0; h < N_HEAD; ++h) { - const float2 q_val = q_shared[h][tl][c]; + for (int i = tid; i < HEADS_PER_PASS * TOKENS_PER_BLOCK * (N_EMBD / 4); i += THREADS_PER_BLOCK) { + const int h = i / (TOKENS_PER_BLOCK * (N_EMBD / 4)); + const int r = i / (N_EMBD / 4) % TOKENS_PER_BLOCK; + const int c4 = i % (N_EMBD / 4); + + float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f); + if (start_batch + r < n_batch) { + v = *(const float4 *) ((const char *) Q + (h0 + h)*nbq1 + (start_batch + r)*nbq2 + i_stream*nbq3 + c4*sizeof(float4)); + } + + q_shared[h][r][2*c4 + 0] = make_float2(v.x, v.y); + q_shared[h][r][2*c4 + 1] = make_float2(v.z, v.w); + } + + __syncthreads(); + + // phase 4 - float products of the widened keys for the heads of this pass + +#pragma unroll 8 + for (int c = 0; c < N_EMBD_H2; ++c) { + float2 k_val[KEYS_PER_THREAD]; #pragma unroll for (int j = 0; j < KEYS_PER_THREAD; ++j) { - qk[h][j] = fmaf(k_val[j].x, q_val.x, qk[h][j]); - qk[h][j] = fmaf(k_val[j].y, q_val.y, qk[h][j]); + k_val[j] = __half22float2(k_shared[kl + j*KEY_LANES][c]); + } +#pragma unroll + for (int h = 0; h < HEADS_PER_PASS; ++h) { + const float2 q_val = q_shared[h][tl][c]; +#pragma unroll + for (int j = 0; j < KEYS_PER_THREAD; ++j) { + qk[h0 + h][j] = fmaf(k_val[j].x, q_val.x, qk[h0 + h][j]); + qk[h0 + h][j] = fmaf(k_val[j].y, q_val.y, qk[h0 + h][j]); + } } } } + // phase 5 - ReLU, weight, add the mask and write, consecutive threads write consecutive keys + float score[KEYS_PER_THREAD] = { 0.0f }; #pragma unroll @@ -512,8 +469,6 @@ static __global__ void lightning_indexer_kernel_tile( } } - // phase 4 - add the mask and write, consecutive threads write consecutive keys - const int i_batch = start_batch + tl; if (i_batch >= n_batch) { return; @@ -596,27 +551,22 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * const int cc = ggml_cuda_info().devices[device].cc; if (n_embd == 128 && n_head == 64) { -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - if (GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { - // use wmma kernel + if ((turing_mma_available(cc) || amd_mfma_available(cc) || amd_wmma_available(cc)) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { constexpr int K_VECS_PER_BLOCK = 32; - constexpr int WARPS_PER_BLOCK = 8; + constexpr int WARPS_PER_BLOCK = 4; - dim3 block(32, WARPS_PER_BLOCK); + dim3 block(ggml_cuda_info().devices[device].warp_size, WARPS_PER_BLOCK); int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); dim3 grid(num_kv_blocks, n_batch, n_stream); - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_F16) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q4_0) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q4_1) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q5_0) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q5_1) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q8_0) GGML_ABORT("fatal error"); } else { -#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - { -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) // use vector kernel constexpr int K_VECS_PER_WARP = 8; constexpr int WARPS_PER_BLOCK = 8; @@ -637,27 +587,22 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_ABORT("fatal error"); } } else if (n_embd == 128 && n_head == 32) { -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - if (GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { - // use wmma kernel + if ((turing_mma_available(cc) || amd_mfma_available(cc) || amd_wmma_available(cc)) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { constexpr int K_VECS_PER_BLOCK = 32; - constexpr int WARPS_PER_BLOCK = 8; + constexpr int WARPS_PER_BLOCK = 4; - dim3 block(32, WARPS_PER_BLOCK); + dim3 block(ggml_cuda_info().devices[device].warp_size, WARPS_PER_BLOCK); int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); dim3 grid(num_kv_blocks, n_batch, n_stream); - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_F16) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q4_0) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q4_1) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q5_0) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q5_1) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q8_0) GGML_ABORT("fatal error"); } else { -#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - { -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) // use vector kernel constexpr int K_VECS_PER_WARP = 8; constexpr int WARPS_PER_BLOCK = 8; @@ -677,8 +622,6 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_F32) GGML_ABORT("fatal error"); } -#ifndef GGML_USE_MUSA - // MUSA archs 21 and 22 cap static shared memory at 28 KB, below what the tile kernel stages } else if (n_embd == 128 && n_head == 4 && n_batch >= LIGHTNING_INDEXER_TILE_TOKENS) { // too few heads for a wmma tile, the tile kernel shares the keys across the tokens constexpr int WARPS_PER_BLOCK = 8; @@ -698,9 +641,8 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_BF16) LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_F32) GGML_ABORT("fatal error"); -#endif // GGML_USE_MUSA } else if (n_embd == 128 && n_head == 4) { - // a batch smaller than a token tile, or MUSA, use vector kernel + // a batch smaller than a token tile, use vector kernel constexpr int K_VECS_PER_WARP = 8; constexpr int WARPS_PER_BLOCK = 8; constexpr int K_VECS_PER_BLOCK = K_VECS_PER_WARP * WARPS_PER_BLOCK; diff --git a/ggml/src/ggml-cuda/mma.cuh b/ggml/src/ggml-cuda/mma.cuh index 93a51aa0b..524db9d85 100644 --- a/ggml/src/ggml-cuda/mma.cuh +++ b/ggml/src/ggml-cuda/mma.cuh @@ -76,11 +76,11 @@ namespace ggml_cuda_mma { // For the A/C matrices this means I major == row major, J major == column major. // For the B matrix this means I major == column major, J major == row major. // MIRRORED == Each data value is held exactly once per thread subgroup. - DATA_LAYOUT_I_MAJOR = 0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for RDNA4 and CDNA. - DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3. + DATA_LAYOUT_I_MAJOR = 0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for WMMA128 and CDNA. + DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and WMMA128, int and float matrix C for WMMA256. DATA_LAYOUT_I_MAJOR_MIRRORED = 20, // Volta, matrix A&B for RDNA3. DATA_LAYOUT_J_MAJOR_MIRRORED = 30, - DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (RDNA4/CDNA), convert to float to unscramble. + DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (WMMA128/CDNA), convert to float to unscramble. }; // Implemented mma combinations are: // - (I_MAJOR, I_MAJOR) -> I_MAJOR @@ -88,11 +88,11 @@ namespace ggml_cuda_mma { // - (I_MAJOR, J_MAJOR_MIRRORED) -> I_MAJOR static constexpr __device__ data_layout get_input_data_layout() { -#if defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE) +#if defined(AMD_WMMA_256B_AVAILABLE) || defined(VOLTA_MMA_AVAILABLE) return DATA_LAYOUT_I_MAJOR_MIRRORED; #else return DATA_LAYOUT_I_MAJOR; -#endif // defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE) +#endif // defined(AMD_WMMA_256B_AVAILABLE) || defined(VOLTA_MMA_AVAILABLE) } template <int I_, int J_, typename T, data_layout ds_=DATA_LAYOUT_I_MAJOR> @@ -201,7 +201,7 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ int get_j(const int l) { if constexpr (I == 16 && J == 16) { -#if defined(RDNA3) +#if defined(AMD_WMMA_256B_AVAILABLE) if constexpr (std::is_same_v<T, float> || std::is_same_v<T, int>) { // matrix C return 2 * l + (threadIdx.x / 16); @@ -210,11 +210,11 @@ namespace ggml_cuda_mma { return l; } #else - // matrix C is the transposed matrix A&B on RDNA4 + // matrix C is the transposed matrix A&B on WMMA128 return ne * (threadIdx.x / 16) + l; -#endif // defined(RDNA3) +#endif // defined(AMD_WMMA_256B_AVAILABLE) } else if constexpr (I == 16 && J == 8) { - // mmq input for RDNA4 + // mmq input for WMMA128 return ne * (threadIdx.x / 16) + l; } else if constexpr (I == 16 && J == 4) { return ne * (threadIdx.x / 16) + l; @@ -573,7 +573,7 @@ namespace ggml_cuda_mma { static constexpr int I = I_; static constexpr int J = J_; static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED; -#if defined(RDNA3) +#if defined(AMD_WMMA_256B_AVAILABLE) static constexpr int ne = tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::ne; half2 x[ne] = {{0.0f, 0.0f}}; @@ -616,7 +616,7 @@ namespace ggml_cuda_mma { return -1; } } -#endif // defined(RDNA3) +#endif // defined(AMD_WMMA_256B_AVAILABLE) }; template <int I_, int J_> @@ -694,7 +694,7 @@ namespace ggml_cuda_mma { }; static __device__ __forceinline__ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> unscramble(const tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & t) { -#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE) tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> ret; #pragma unroll for (int l0 = 0; l0 < t.ne/2; ++l0) { @@ -705,7 +705,7 @@ namespace ggml_cuda_mma { #else NO_DEVICE_CODE; GGML_UNUSED(t); -#endif // defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE) } #if defined(TURING_MMA_AVAILABLE) @@ -726,7 +726,7 @@ namespace ggml_cuda_mma { return ret; } -#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) +#elif defined(AMD_WMMA_256B_AVAILABLE) static __device__ __forceinline__ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> get_half2( const tile<16, 16, float, DATA_LAYOUT_I_MAJOR> & tile_float) { tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> ret; @@ -949,16 +949,22 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ void load_ldmatrix_swizzled( tile<I, J, T, dl> & t, const T * __restrict__ xs0, const int offset) { #if defined(TURING_MMA_AVAILABLE) - static_assert(I == 16, "bad tile width"); + static_assert(I == 8 || I == 16, "bad tile width"); static_assert(J == 8, "bad tile height"); const int i = threadIdx.x % t.I; - const int j = (threadIdx.x / t.I) * (t.J / 2); + const int j = ((threadIdx.x / t.I) * (t.J / 2)) % t.J; int offset_ij = offset + i * stride + j; offset_ij = swizzle<stride, T>(offset_ij, i); int * xi = (int *) t.x; - asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];" - : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3]) - : "l"(xs0 + offset_ij)); + if constexpr (I == 16) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3]) + : "l"(xs0 + offset_ij)); + } else { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];" + : "=r"(xi[0]), "=r"(xi[1]) + : "l"(xs0 + offset_ij)); + } #elif defined(VOLTA_MMA_AVAILABLE) #pragma unroll for (int o = 0; o < t.ne; o += 4) { @@ -1011,16 +1017,22 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ void load_ldmatrix_trans_swizzled( tile<I, 8, T, dl> & t, const T * __restrict__ xs0, const int offset) { #if defined(TURING_MMA_AVAILABLE) - static_assert(I == 16, "bad tile width"); + static_assert(I == 8 || I == 16, "bad tile width"); static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); - const int i = threadIdx.x % t.I; - const int j = (threadIdx.x / t.I) * (t.J / 2); + const int i = threadIdx.x % 16; + const int j = I == 16 ? (threadIdx.x / 16) * (t.J / 2) : 0; int offset_ij = offset + i * stride + j; offset_ij = swizzle<stride, T>(offset_ij, i); int * xi = (int *) t.x; - asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];" - : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3]) - : "l"(xs0 + offset_ij)); + if constexpr (I == 16) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3]) + : "l"(xs0 + offset_ij)); + } else { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.trans.b16 {%0, %1}, [%2];" + : "=r"(xi[0]), "=r"(xi[1]) + : "l"(xs0 + offset_ij)); + } #elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); if constexpr (I == 32) { @@ -1159,7 +1171,7 @@ namespace ggml_cuda_mma { : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3])); #endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE #elif defined(AMD_WMMA_AVAILABLE) -#if defined(RDNA4) +#if defined(AMD_WMMA_128B_AVAILABLE) using halfx8_t = __attribute__((ext_vector_type(8))) _Float16; halfx8_t& acc_frag = reinterpret_cast<halfx8_t&>(D.x[0]); const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]); @@ -1168,7 +1180,7 @@ namespace ggml_cuda_mma { #else GGML_UNUSED_VARS(D, A, B); NO_DEVICE_CODE; -#endif // defined(RDNA4) +#endif // defined(AMD_WMMA_128B_AVAILABLE) #elif defined(AMD_MFMA_AVAILABLE) // MFMA: FP16 input, FP32 accumulate, convert back to half2. using halfx4_t = __attribute__((ext_vector_type(4))) _Float16; @@ -1206,7 +1218,7 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ void mma( tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR> & A, const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR> & B) { -#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE) tile<16, 8, half2> * D16 = (tile<16, 8, half2> *) &D; const tile<16, 8, half2> * A16 = (const tile<16, 8, half2> *) &A; mma(D16[0], A16[0], B); @@ -1214,7 +1226,7 @@ namespace ggml_cuda_mma { #else GGML_UNUSED_VARS(D, A, B); NO_DEVICE_CODE; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) && defined(RDNA4) +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE) } template <data_layout dl_ab, data_layout dl_d> @@ -1361,14 +1373,14 @@ namespace ggml_cuda_mma { : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3])); #endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE #elif defined(AMD_WMMA_AVAILABLE) -#if defined(RDNA4) +#if defined(AMD_WMMA_128B_AVAILABLE) using halfx8_t = __attribute__((ext_vector_type(8))) _Float16; using floatx8_t = __attribute__((ext_vector_type(8))) float; floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]); const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]); const halfx8_t& b_frag = reinterpret_cast<const halfx8_t&>(B.x[0]); acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag); -#elif defined(RDNA3) +#elif defined(AMD_WMMA_256B_AVAILABLE) using halfx16_t = __attribute__((ext_vector_type(16))) _Float16; using floatx8_t = __attribute__((ext_vector_type(8))) float; floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]); @@ -1378,7 +1390,7 @@ namespace ggml_cuda_mma { #else GGML_UNUSED_VARS(D, A, B); NO_DEVICE_CODE; -#endif // RDNA4 +#endif // AMD_WMMA_128B_AVAILABLE #elif defined(AMD_MFMA_AVAILABLE) using halfx4_t = __attribute__((ext_vector_type(4))) _Float16; using floatx4_t = __attribute__((ext_vector_type(4))) float; @@ -1396,15 +1408,15 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ void mma( tile<16, 16, float, dl_d> & D, const tile<16, 8, nv_bfloat162, dl_ab> & A, const tile<16, 8, nv_bfloat162, dl_ab> & B) { #if defined(AMD_WMMA_AVAILABLE) -#if defined(RDNA4) - using bf16x8_t = __attribute__((ext_vector_type(8))) __bf16; +#if defined(AMD_WMMA_128B_AVAILABLE) + using bf16x8_t = __attribute__((ext_vector_type(8))) short; using floatx8_t = __attribute__((ext_vector_type(8))) float; floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]); const bf16x8_t& a_frag = reinterpret_cast<const bf16x8_t&>(A.x[0]); const bf16x8_t& b_frag = reinterpret_cast<const bf16x8_t&>(B.x[0]); acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12(a_frag, b_frag, acc_frag); -#elif defined(RDNA3) - using bf16x16_t = __attribute__((ext_vector_type(16))) __bf16; +#elif defined(AMD_WMMA_256B_AVAILABLE) + using bf16x16_t = __attribute__((ext_vector_type(16))) short; using floatx8_t = __attribute__((ext_vector_type(8))) float; floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]); const bf16x16_t& a_frag = reinterpret_cast<const bf16x16_t&>(A.x[0]); @@ -1413,7 +1425,7 @@ namespace ggml_cuda_mma { #else GGML_UNUSED_VARS(D, A, B); NO_DEVICE_CODE; -#endif // defined(RDNA4) +#endif // defined(AMD_WMMA_128B_AVAILABLE) #elif defined(AMD_MFMA_AVAILABLE) using floatx4_t = __attribute__((ext_vector_type(4))) float; floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]); @@ -1455,19 +1467,19 @@ namespace ggml_cuda_mma { #elif defined(AMD_WMMA_AVAILABLE) using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int; int32x8_t * acc = (int32x8_t *) D.x; -#if defined(RDNA4) +#if defined(AMD_WMMA_128B_AVAILABLE) using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int; int32x2_t * a_vec = (int32x2_t *) A.x; int32x2_t * b_vec = (int32x2_t *) B.x; acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], true); acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[1], true, b_vec[1], acc[0], true); -#elif defined(RDNA3) +#elif defined(AMD_WMMA_256B_AVAILABLE) using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; int32x4_t * a_vec = (int32x4_t *) A.x; int32x4_t * b_vec = (int32x4_t *) B.x; acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], true); acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[1], true, b_vec[1], acc[0], true); -#endif // RDNA4 +#endif // AMD_WMMA_128B_AVAILABLE #else GGML_UNUSED_VARS(D, A, B); NO_DEVICE_CODE; @@ -1544,7 +1556,7 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ void mma( tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & A, const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) { -#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) +#if defined(AMD_WMMA_256B_AVAILABLE) using halfx16_t = __attribute__((ext_vector_type(16))) _Float16; halfx16_t * xD = (halfx16_t *) D.x; const halfx16_t * xA = (const halfx16_t *) A.x; @@ -1573,17 +1585,17 @@ namespace ggml_cuda_mma { #elif defined(AMD_WMMA_AVAILABLE) using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int; int32x8_t * acc = (int32x8_t *) D.x; -#if defined(RDNA4) +#if defined(AMD_WMMA_128B_AVAILABLE) using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int; int32x2_t * a_vec = (int32x2_t *) A.x; int32x2_t * b_vec = (int32x2_t *) B.x; acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], false); -#elif defined(RDNA3) +#elif defined(AMD_WMMA_256B_AVAILABLE) using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; int32x4_t * a_vec = (int32x4_t *) A.x; int32x4_t * b_vec = (int32x4_t *) B.x; acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], false); -#endif // RDNA4 +#endif // AMD_WMMA_128B_AVAILABLE #else GGML_UNUSED(D); GGML_UNUSED(A); diff --git a/ggml/src/ggml-cuda/mmf.cu b/ggml/src/ggml-cuda/mmf.cu index 646a5899c..040d390e0 100644 --- a/ggml/src/ggml-cuda/mmf.cu +++ b/ggml/src/ggml-cuda/mmf.cu @@ -50,7 +50,8 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr mmf_ids_data * ids_info_ptr = nullptr; ggml_cuda_pool_alloc<int32_t> ids_src_compact_dev; ggml_cuda_pool_alloc<int32_t> ids_dst_compact_dev; - ggml_cuda_pool_alloc<int32_t> expert_bounds_dev; + + ggml_cuda_pool_alloc<ggml_cuda_mm_id_tile_spec> tile_spec_dev; // For MUL_MAT_ID the memory layout is different than for MUL_MAT: const int64_t ncols_dst = ids ? ne2 : ne1; @@ -74,25 +75,28 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr const int64_t n_experts = ne02; const int64_t n_tokens = ne12; const int64_t ne_get_rows = n_tokens * n_expert_used; + const int64_t max_tiles = (ne_get_rows + 16 - 1) / 16 + ne02; ids_src_compact_dev.alloc(ctx.pool(), ne_get_rows); ids_dst_compact_dev.alloc(ctx.pool(), ne_get_rows); - expert_bounds_dev.alloc(ctx.pool(), n_experts + 1); + tile_spec_dev.alloc(ctx.pool(), max_tiles); const int si1 = static_cast<int>(ids_s1); const int sis1 = static_cast<int>(src1->nb[2] / src1->nb[1]); GGML_ASSERT(sis1 > 0); - ggml_cuda_launch_mm_ids_helper(ids_d, ids_src_compact_dev.get(), ids_dst_compact_dev.get(), expert_bounds_dev.get(), - static_cast<int>(n_experts), static_cast<int>(n_tokens), static_cast<int>(n_expert_used), static_cast<int>(ne11), si1, sis1, /*write_inverse =*/ false, ctx.stream()); + ggml_cuda_launch_mm_ids_helper(ids_d, ids_src_compact_dev.get(), ids_dst_compact_dev.get(), tile_spec_dev.get(), + static_cast<int>(n_experts), static_cast<int>(n_tokens), static_cast<int>(n_expert_used), static_cast<int>(ne11), 16, + si1, sis1, /*write_inverse =*/ false, ctx.stream()); CUDA_CHECK(cudaGetLastError()); - ids_info.ids_src_compact = ids_src_compact_dev.get(); - ids_info.ids_dst_compact = ids_dst_compact_dev.get(); - ids_info.expert_bounds_dev = expert_bounds_dev.get(); - ids_info.n_experts = static_cast<int>(n_experts); - ids_info.sis1 = sis1; + ids_info.ids_src_compact = ids_src_compact_dev.get(); + ids_info.ids_dst_compact = ids_dst_compact_dev.get(); + ids_info.tile_spec_dev = tile_spec_dev.get(); + ids_info.n_experts = static_cast<int>(n_experts); + ids_info.max_tiles = static_cast<int>(max_tiles); + ids_info.sis1 = sis1; ids_info_ptr = &ids_info; } diff --git a/ggml/src/ggml-cuda/mmf.cuh b/ggml/src/ggml-cuda/mmf.cuh index 879a86527..c49f6c47a 100644 --- a/ggml/src/ggml-cuda/mmf.cuh +++ b/ggml/src/ggml-cuda/mmf.cuh @@ -1,7 +1,8 @@ #pragma once -#include "mma.cuh" #include "common.cuh" +#include "mma.cuh" +#include "mmid.cuh" #include "convert.cuh" using namespace ggml_cuda_mma; @@ -36,8 +37,9 @@ static constexpr __device__ int mmf_get_padding() { struct mmf_ids_data { const int32_t * ids_src_compact = nullptr; const int32_t * ids_dst_compact = nullptr; - const int32_t * expert_bounds_dev = nullptr; + const ggml_cuda_mm_id_tile_spec * tile_spec_dev = nullptr; int n_experts = 0; + int max_tiles = 0; int sis1 = 0; }; @@ -308,7 +310,7 @@ __launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1) static __global__ void mul_mat_f_ids( const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids_src_compact, const int32_t * __restrict__ ids_dst_compact, - const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, + const ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, float * __restrict__ dst, const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst, const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, @@ -349,21 +351,20 @@ static __global__ void mul_mat_f_ids( constexpr int ntA = rows_per_block / tile_A::I; constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I; - const int row0 = blockIdx.x * rows_per_block; + const int row0 = blockIdx.x * rows_per_block; + const int tile_idx = blockIdx.y; - const int expert_idx = blockIdx.y; - const int expert_start = expert_bounds[expert_idx]; - const int expert_end = expert_bounds[expert_idx + 1]; + const ggml_cuda_mm_id_tile_spec ts = tile_spec[tile_idx]; + + const int expert_idx = ts.expert; + const int expert_start = ts.col_low; + const int expert_end = ts.col_high; const int ncols_expert = expert_end - expert_start; - const int tiles_for_expert = (ncols_expert + cols_per_block - 1) / cols_per_block; - const int tile_idx = blockIdx.z; - if (tile_idx >= tiles_for_expert) { + if (ncols_expert == 0) { return; } - const int col_base = tile_idx * cols_per_block; - GGML_UNUSED(channel_ratio); const int channel_x = expert_idx; @@ -409,10 +410,9 @@ static __global__ void mul_mat_f_ids( #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { const int j = j0 + tile_idx_local*tile_B::I; - const int global_j = col_base + j; float val = 0.0f; - if (j < cols_per_block && global_j < ncols_expert) { - const int src_entry = ids_src_expert[global_j]; + if (j < cols_per_block && j < ncols_expert) { + const int src_entry = ids_src_expert[j]; const uint2 qrm = fast_div_modulo((uint32_t) src_entry, sis1_fd); const int token = (int) qrm.x; const int channel = (int) qrm.y; @@ -462,10 +462,9 @@ static __global__ void mul_mat_f_ids( #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { const int j = j0 + tile_idx_local*tile_B::I; - const int global_j = col_base + j; float2 tmp = make_float2(0.0f, 0.0f); - if (j < cols_per_block && global_j < ncols_expert) { - const int src_entry = ids_src_expert[global_j]; + if (j < cols_per_block && j < ncols_expert) { + const int src_entry = ids_src_expert[j]; const uint2 qrm = fast_div_modulo((uint32_t) src_entry, sis1_fd); const int token = (int) qrm.x; const int channel = (int) qrm.y; @@ -564,9 +563,8 @@ static __global__ void mul_mat_f_ids( } } - const int global_j = col_base + j; - if (j < cols_per_block && global_j < ncols_expert && nchannels_dst > 0) { - const int dst_entry = ids_dst_expert[global_j]; + if (j < cols_per_block && j < ncols_expert && nchannels_dst > 0) { + const int dst_entry = ids_dst_expert[j]; const uint2 qrm = fast_div_modulo((uint32_t) dst_entry, nch_fd); const int token = (int) qrm.x; if (token < ncols_dst_total) { @@ -580,7 +578,7 @@ static __global__ void mul_mat_f_ids( } } #else - GGML_UNUSED_VARS(x, y, ids_src_compact, ids_dst_compact, expert_bounds, dst, + GGML_UNUSED_VARS(x, y, ids_src_compact, ids_dst_compact, tile_spec, dst, ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, sis1_fd, nch_fd); @@ -603,17 +601,16 @@ static inline void mul_mat_f_switch_ids( // Use the compact-ids kernel only for larger tiles; for small ncols_dst (< 16) // we prefer the normal mul_mat_f path with has_ids=true. if (has_ids_data && ncols_dst > 16) { - const int max_tiles = (int) ((ncols_dst + cols_per_block - 1) / cols_per_block); - if (max_tiles == 0) { + if (ids_data->max_tiles == 0) { return; } - dim3 block_nums_ids(block_nums.x, ids_data->n_experts, max_tiles); + dim3 block_nums_ids(block_nums.x, ids_data->max_tiles, 1); const uint3 sis1_fd = ids_data->sis1 > 0 ? init_fastdiv_values((uint32_t) ids_data->sis1) : make_uint3(0, 0, 1); const uint3 nch_fd = init_fastdiv_values((uint32_t) nchannels_dst); mul_mat_f_ids<T, rows_per_block, cols_per_block, nwarps><<<block_nums_ids, block_dims, nbytes_shared_total, stream>>> - (x, y, ids_data->ids_src_compact, ids_data->ids_dst_compact, ids_data->expert_bounds_dev, dst, + (x, y, ids_data->ids_src_compact, ids_data->ids_dst_compact, ids_data->tile_spec_dev, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, @@ -741,9 +738,8 @@ void mul_mat_f_cuda( sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream, ids_data); } break; - default: { + default: GGML_ABORT("fatal error"); - } break; } GGML_UNUSED_VARS(nchannels_y); @@ -845,9 +841,8 @@ static void mul_mat_f_switch_cols_per_block( stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data); } break; - default: { + default: GGML_ABORT("fatal error"); - } break; } } diff --git a/ggml/src/ggml-cuda/mmid.cu b/ggml/src/ggml-cuda/mmid.cu index 12ae14ce9..64de4fb27 100644 --- a/ggml/src/ggml-cuda/mmid.cu +++ b/ggml/src/ggml-cuda/mmid.cu @@ -27,12 +27,16 @@ template <> struct mm_ids_pow2<0> { static constexpr int value = 1; }; // Helper function for mul_mat_id, converts ids to a more convenient format. // ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert. // ids_dst describes the same mapping but for the dst tensor. -// The upper and lower bounds for the ith expert in the compact src1 tensor are stored in expert_bounds[i:i+1]. +// The first (n_tokens*n_expert_used + J - 1) / tile_width elements of tile_spec contain the specication for +// matrix multiplication tiles at token positions which are exactly divided by tile_width. +// Additionally the following n_expert elements of tile_spec may contain additional tiles that fall between these seams. template <int n_expert_used_template> __launch_bounds__(ggml_cuda_get_physical_warp_size(), 1) static __global__ void mm_ids_helper( - const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse) { + const int32_t * __restrict__ ids, + int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, + const int n_tokens, const int n_expert_used_var, const int nchannels_y, const uint3 tile_width, + const int si1, const int sis1, const bool write_inverse) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template; const int expert = blockIdx.x; @@ -118,19 +122,29 @@ static __global__ void mm_ids_helper( return; } - expert_bounds[expert] = nex_prev; + int col_low = nex_prev; + int col_high = nex_prev + min(int(tile_width.z), it_compact); - if (expert < static_cast<int>(gridDim.x) - 1) { - return; + // The while loop covers all tiles at the seams containing the column with % tile_width == 0. + while (col_high > fastdiv(col_low + tile_width.z - 1, tile_width) * tile_width.z) { + tile_spec[fastdiv(col_low + tile_width.z - 1, tile_width)] + = {expert, col_low, col_high, /*padding =*/ 0}; + + col_low = col_high; + col_high = min(col_high + int(tile_width.z), nex_prev + it_compact); } - expert_bounds[gridDim.x] = nex_prev + it_compact; + // Finally, at most one extra tile_spec between the seams per expert. + tile_spec[fastdiv(n_tokens*n_expert_used + tile_width.z - 1, tile_width) + expert] + = {expert, col_low, col_high, /*padding =*/ 0}; } template <int n_expert_used_template> static void launch_mm_ids_helper( - const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { + const int32_t * __restrict__ ids, + int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, + const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int tile_width, + const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { GGML_ASSERT(n_tokens < (1 << 22) && "too few bits in mm_ids_helper_store"); GGML_ASSERT(n_expert_used_var < (1 << 10) && "too few bits in mm_ids_helper_store"); @@ -139,41 +153,53 @@ static void launch_mm_ids_helper( const size_t smpbo = ggml_cuda_info().devices[id].smpbo; CUDA_SET_SHARED_MEMORY_LIMIT(mm_ids_helper<n_expert_used_template>, smpbo); + const uint3 tile_width_fd = init_fastdiv_values(tile_width); + const dim3 num_blocks(n_experts, 1, 1); const dim3 block_size(warp_size, 1, 1); const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store); GGML_ASSERT(nbytes_shared <= smpbo); mm_ids_helper<n_expert_used_template><<<num_blocks, block_size, nbytes_shared, stream>>> - (ids, ids_src1, ids_dst, expert_bounds, n_tokens, n_expert_used_var, nchannels_y, si1, sis1, write_inverse); + (ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse); } void ggml_cuda_launch_mm_ids_helper( - const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { + const int32_t * __restrict__ ids, + int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, + const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int tile_width, + const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { switch (n_expert_used) { case 2: - launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 4: - launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 6: - launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 8: - launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 10: - launch_mm_ids_helper<10>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper<10>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 16: - launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 32: - launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; default: - launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; } } diff --git a/ggml/src/ggml-cuda/mmid.cuh b/ggml/src/ggml-cuda/mmid.cuh index 74c2db433..dc878a71c 100644 --- a/ggml/src/ggml-cuda/mmid.cuh +++ b/ggml/src/ggml-cuda/mmid.cuh @@ -1,5 +1,16 @@ #pragma once +struct ggml_cuda_mm_id_tile_spec { + int32_t expert; + int32_t col_low; + int32_t col_high; + + int32_t padding; +}; +static_assert(sizeof(ggml_cuda_mm_id_tile_spec) == 16, "bad ggml_cuda_mm_id_tile_spec size"); + void ggml_cuda_launch_mm_ids_helper( - const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds, - int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream); + const int32_t * __restrict__ ids, + int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, + const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int tile_width, + const int si1, const int sis1, const bool write_inverse, cudaStream_t stream); diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh index 180b2d937..ead3825d2 100644 --- a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +++ b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh @@ -26,16 +26,16 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); @@ -78,15 +78,15 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); // --------------------------------------------------------------------------------------------- @@ -115,42 +115,42 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); // --------------------------------------------------------------------------------------------- diff --git a/ggml/src/ggml-cuda/mmq-few.cu b/ggml/src/ggml-cuda/mmq-few.cu new file mode 100644 index 000000000..2c2a2745e --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-few.cu @@ -0,0 +1,876 @@ +#include "mmq-few.cuh" +#include "cp-async.cuh" +#include "mma.cuh" +#include "quantize.cuh" +#include "vecdotq.cuh" + +#include <type_traits> + +// NVIDIA only: the kernel uses the NVIDIA int8 MMA fragment layout and cp.async. +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +using namespace ggml_cuda_mma; + +// Few-row MMQ: int8 MMA mat-mul for 2-24 src1 columns. A CUDA block has 16 src0 rows (32 for 2 column groups on large matrices). +// Its warps split K in units of 256 values and copy raw src0 rows into their own cp.async ring, src1 is quantized to the MMA fragment layout. + +static constexpr int MMQ_FEW_MAX_NCG = 3; // max. number of column groups +static constexpr int MMQ_FEW_NROWS = 16; // src0 rows of one MMA tile +static constexpr int MMQ_FEW_NWARPS = 4; + +template <ggml_type type> +static constexpr __host__ __device__ int mmq_few_unit_size_of() { + return QK_K/ggml_cuda_type_traits<type>::qk*ggml_cuda_type_traits<type>::bs; +} + +// src0 bytes per 256 values, 0 for the types without a few-row decoder. +static constexpr __host__ __device__ int mmq_few_unit_size(const ggml_type type) { + switch (type) { + case GGML_TYPE_Q8_0: return mmq_few_unit_size_of<GGML_TYPE_Q8_0>(); + case GGML_TYPE_Q3_K: return mmq_few_unit_size_of<GGML_TYPE_Q3_K>(); + case GGML_TYPE_Q4_K: return mmq_few_unit_size_of<GGML_TYPE_Q4_K>(); + case GGML_TYPE_Q5_K: return mmq_few_unit_size_of<GGML_TYPE_Q5_K>(); + case GGML_TYPE_Q6_K: return mmq_few_unit_size_of<GGML_TYPE_Q6_K>(); + case GGML_TYPE_IQ3_S: return mmq_few_unit_size_of<GGML_TYPE_IQ3_S>(); + case GGML_TYPE_IQ4_NL: return mmq_few_unit_size_of<GGML_TYPE_IQ4_NL>(); + case GGML_TYPE_IQ4_XS: return mmq_few_unit_size_of<GGML_TYPE_IQ4_XS>(); + default: return 0; + } +} + +// Q4_K and Q5_K also need the src1 sums for their mins. +static constexpr __host__ __device__ mmq_q8_1_ds_layout mmq_few_ds_layout(const ggml_type type) { + return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K ? MMQ_Q8_1_DS_LAYOUT_DS4 : MMQ_Q8_1_DS_LAYOUT_D4; +} + +// Units of 256 values per ring stage: 1 for Q6_K, 3 column groups (fewer free registers) and 32-row tiles (2 blocks per SM), else 2. +static constexpr __host__ __device__ int mmq_few_nunits_stage(const ggml_type type, const int ncg, const int nmt) { + return type == GGML_TYPE_Q6_K || ncg > 2 || nmt > 1 ? 1 : 2; +} + +// A third ring stage only makes Q6_K faster, the other types get slower on large matrices. +static constexpr __host__ __device__ int mmq_few_nstages(const ggml_type type) { + return type == GGML_TYPE_Q6_K ? 3 : 2; +} + +// The src0 data of a ring stage does not fill whole 16 byte chunks, it is copied from the preceding 16 byte boundary. +static constexpr __host__ __device__ bool mmq_few_unaligned(const ggml_type type, const int nunits_stage) { + return (nunits_stage*mmq_few_unit_size(type)) % 16 != 0; +} + +// for unaligned data the max. over the 8 possible start offsets +static constexpr __host__ __device__ int mmq_few_nchunks(const ggml_type type, const int nunits_stage) { + const int nbytes = nunits_stage*mmq_few_unit_size(type); + return mmq_few_unaligned(type, nunits_stage) ? (14 + nbytes + 15)/16 : nbytes/16; +} + +// Unaligned rows are padded to an odd number of 16 byte chunks against bank conflicts, for aligned rows the padding costs more than it saves. +static constexpr __host__ __device__ int mmq_few_sram_stride(const ggml_type type, const int nunits_stage) { + const int nchunks = mmq_few_nchunks(type, nunits_stage); + return mmq_few_unaligned(type, nunits_stage) && nchunks % 2 == 0 ? 16*(nchunks + 1) : 16*nchunks; +} + +static constexpr __host__ __device__ int mmq_few_nbytes_shared(const ggml_type type, const int ncg, const int nmt) { + return MMQ_FEW_NWARPS*mmq_few_nstages(type)*nmt*MMQ_FEW_NROWS*mmq_few_sram_stride(type, mmq_few_nunits_stage(type, ncg, nmt)); +} + +// With Q6_K and Q8_0 two CUDA blocks with 32-row tiles would not fit into an SM. +static constexpr __host__ __device__ bool mmq_few_tall_supported(const ggml_type type) { + return type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || + type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL || type == GGML_TYPE_IQ4_XS; +} + +#if defined(CP_ASYNC_AVAILABLE) + +// The src1 fragments of a lane for one unit of 256 values, see quantize.cuh. +struct mmq_few_y_frag { + int4 qs[4]; + int4 ds[4]; +}; + +static __device__ __forceinline__ void mmq_few_load_y(mmq_few_y_frag & yf, const int4 * y) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + yf.qs[j] = y[j*WARP_SIZE + threadIdx.x]; + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + yf.ds[j] = y[4*WARP_SIZE + 4*(threadIdx.x % 4) + j]; + } +} + +static __device__ __forceinline__ int2 mmq_few_get_b(const mmq_few_y_frag & yf, const int s) { + const int4 q = yf.qs[s/2]; + return s % 2 == 0 ? make_int2(q.x, q.y) : make_int2(q.z, q.w); +} + +template <mmq_q8_1_ds_layout ds_layout> +static __device__ __forceinline__ float2 mmq_few_get_d(const mmq_few_y_frag & yf, const int s) { + if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + const float * d = (const float *) &yf.ds[s/2]; + return make_float2(d[2*(s % 2) + 0], d[2*(s % 2) + 1]); + } else { + const half2 * ds = (const half2 *) &yf.ds[s/2]; + return make_float2(__low2float(ds[2*(s % 2) + 0]), __low2float(ds[2*(s % 2) + 1])); + } +} + +static __device__ __forceinline__ float2 mmq_few_get_sum(const mmq_few_y_frag & yf, const int s) { + const half2 * ds = (const half2 *) &yf.ds[s/2]; + return make_float2(__high2float(ds[2*(s % 2) + 0]), __high2float(ds[2*(s % 2) + 1])); +} + +// C.x[0], C.x[1]: row g, columns 2t and 2t+1. C.x[2], C.x[3]: row g+8. +static __device__ __forceinline__ tile<16, 8, int> mmq_few_mma(const int (&lo)[2], const int (&hi)[2], const int2 b) { + tile<16, 8, int> A; + A.x[0] = lo[0]; + A.x[1] = lo[1]; + A.x[2] = hi[0]; + A.x[3] = hi[1]; + tile<8, 8, int> B; + B.x[0] = b.x; + B.x[1] = b.y; + tile<16, 8, int> C; + mma(C, A, B); + return C; +} + +static __device__ __forceinline__ tile<16, 8, int> mmq_few_mma_k16(const int (&a)[2], const int b) { + tile<16, 4, int> A; + A.x[0] = a[0]; + A.x[1] = a[1]; + tile<8, 4, int> B; + B.x[0] = b; + tile<16, 8, int> C; + mma(C, A, B); + return C; +} + +// sum_d += d * (C * ls) of sub-block s for all column groups, ls are the src0 row scales +template <ggml_type type, int ncg, typename scale_t> +static __device__ __forceinline__ void mmq_few_mma_add( + const int (&lo)[2], const int (&hi)[2], const scale_t (&ls)[2], const mmq_few_y_frag (&yf)[ncg], const int s, float (&sum_d)[ncg][4]) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const tile<16, 8, int> C = mmq_few_mma(lo, hi, mmq_few_get_b(yf[c], s)); + + const float2 d = mmq_few_get_d<mmq_few_ds_layout(type)>(yf[c], s); + sum_d[c][0] += d.x * float(C.x[0]*ls[0]); + sum_d[c][1] += d.y * float(C.x[1]*ls[0]); + sum_d[c][2] += d.x * float(C.x[2]*ls[1]); + sum_d[c][3] += d.y * float(C.x[3]*ls[1]); + } +} + +// Same with one k=16 MMA per half of sub-block s, the int8 scales of the halves are bytes 0 and 1 of sc[r]. +template <ggml_type type, int ncg> +static __device__ __forceinline__ void mmq_few_mma_k16_add( + const int (&lo)[2], const int (&hi)[2], const int (&sc)[2], const mmq_few_y_frag (&yf)[ncg], const int s, float (&sum_d)[ncg][4]) { + const int sc_lo[2] = {(int8_t) sc[0], (int8_t) sc[1]}; + const int sc_hi[2] = {(int8_t) (sc[0] >> 8), (int8_t) (sc[1] >> 8)}; +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const int2 b = mmq_few_get_b(yf[c], s); + const tile<16, 8, int> C_lo = mmq_few_mma_k16(lo, b.x); + const tile<16, 8, int> C_hi = mmq_few_mma_k16(hi, b.y); + + const float2 d = mmq_few_get_d<mmq_few_ds_layout(type)>(yf[c], s); + sum_d[c][0] += d.x * float(C_lo.x[0]*sc_lo[0] + C_hi.x[0]*sc_hi[0]); + sum_d[c][1] += d.y * float(C_lo.x[1]*sc_lo[0] + C_hi.x[1]*sc_hi[0]); + sum_d[c][2] += d.x * float(C_lo.x[2]*sc_lo[1] + C_hi.x[2]*sc_hi[1]); + sum_d[c][3] += d.y * float(C_lo.x[3]*sc_lo[1] + C_hi.x[3]*sc_hi[1]); + } +} + +template <int ncg> +static __device__ __forceinline__ void mmq_few_add_scaled(float (&sum)[ncg][4], const float (&d)[2], const float (&sum_d)[ncg][4]) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + sum[c][l] += d[l/2]*sum_d[c][l]; + } + } +} + +// vec_dot: one unit of 256 values of the 16 rows of a warp, x[0] and x[1] are the rows g and g+8 of lane (g, t). +// The lane decodes the values [4t, 4t + 4) and [16 + 4t, 16 + 4t + 4) of each sub-block, its A fragments. + +template <ggml_type type, int ncg> +static __device__ __forceinline__ void mmq_few_vec_dot_q45_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + using block_t = std::conditional_t<type == GGML_TYPE_Q4_K, block_q4_K, block_q5_K>; + const int t = threadIdx.x % 4; + + int qs[2][4][2]; + [[maybe_unused]] int qh[2][2]; // Q5_K: bit s holds the high bit of sub-block s + int sc[2][2]; + int mn[2][2]; + float d[2]; + float dmin[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_t * bx = (const block_t *) x[r]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + qs[r][j][0] = get_int_b4(bx->qs + 32*j, t); + qs[r][j][1] = get_int_b4(bx->qs + 32*j + 16, t); + } + if constexpr (type == GGML_TYPE_Q5_K) { + qh[r][0] = get_int_b4(bx->qh, t); + qh[r][1] = get_int_b4(bx->qh + 16, t); + } + const int4 hdr = *(const int4 *) bx; // dm and the 12 scale bytes + const int scales[3] = {hdr.y, hdr.z, hdr.w}; + sc[r][0] = unpack_scales_q45_K(scales, 0); + sc[r][1] = unpack_scales_q45_K(scales, 1); + mn[r][0] = unpack_scales_q45_K(scales, 2); + mn[r][1] = unpack_scales_q45_K(scales, 3); + const float2 dm = __half22float2(bx->dm); + d[r] = dm.x; + dmin[r] = dm.y; + } + + float sum_d[ncg][4] = {{0.0f}}; + float sum_m[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + const int j = s/2; + const int shift = 4*(s % 2); + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = (qs[r][j][0] >> shift) & 0x0F0F0F0F; + hi[r] = (qs[r][j][1] >> shift) & 0x0F0F0F0F; + if constexpr (type == GGML_TYPE_Q5_K) { + lo[r] |= ((qh[r][0] >> s) & 0x01010101) << 4; + hi[r] |= ((qh[r][1] >> s) & 0x01010101) << 4; + } + } + const int ls[2] = {(sc[0][s/4] >> (8*(s % 4))) & 0xFF, (sc[1][s/4] >> (8*(s % 4))) & 0xFF}; + const float m0 = (mn[0][s/4] >> (8*(s % 4))) & 0xFF; + const float m1 = (mn[1][s/4] >> (8*(s % 4))) & 0xFF; + mmq_few_mma_add<type>(lo, hi, ls, yf, s, sum_d); +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const float2 sm = mmq_few_get_sum(yf[c], s); + sum_m[c][0] += m0 * sm.x; + sum_m[c][1] += m0 * sm.y; + sum_m[c][2] += m1 * sm.x; + sum_m[c][3] += m1 * sm.y; + } + } +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + sum[c][l] += d[l/2]*sum_d[c][l] - dmin[l/2]*sum_m[c][l]; + } + } +} + +// Q6_K and Q3_K have a scale per 16 values: one k=16 MMA per half of a sub-block. +template <int ncg> +static __device__ __forceinline__ void mmq_few_vec_dot_q6_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + d[r] = __half2float(((const block_q6_K *) x[r])->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int n = 0; n < 2; ++n) { + int ql[2][2][2]; // [row][ql offset 0 or 32][lo or hi] + int qh[2][2]; + int sc[2][2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q6_K * bx = (const block_q6_K *) x[r]; +#pragma unroll + for (int k = 0; k < 2; ++k) { + ql[r][k][0] = get_int_b2(bx->ql + 64*n + 32*k, t); + ql[r][k][1] = get_int_b2(bx->ql + 64*n + 32*k + 16, t); + } + qh[r][0] = get_int_b2(bx->qh + 32*n, t); + qh[r][1] = get_int_b2(bx->qh + 32*n + 16, t); + sc[r][0] = get_int_b2(bx->scales + 8*n, 0); + sc[r][1] = get_int_b2(bx->scales + 8*n, 1); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int s = 4*n + j; + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = __vsub4(((ql[r][j % 2][0] >> (4*(j/2))) & 0x0F0F0F0F) | (((qh[r][0] >> (2*j)) & 0x03030303) << 4), 0x20202020); + hi[r] = __vsub4(((ql[r][j % 2][1] >> (4*(j/2))) & 0x0F0F0F0F) | (((qh[r][1] >> (2*j)) & 0x03030303) << 4), 0x20202020); + } + const int sc_s[2] = {sc[0][j/2] >> (16*(j % 2)), sc[1][j/2] >> (16*(j % 2))}; // scales 2s and 2s+1 + mmq_few_mma_k16_add<GGML_TYPE_Q6_K>(lo, hi, sc_s, yf, s, sum_d); + } + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template <int ncg> +static __device__ __forceinline__ void mmq_few_vec_dot_q3_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + int hm[2][2]; + int sc[2][4]; // the 16 scales of a row as int8, minus 32 + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q3_K * bx = (const block_q3_K *) x[r]; + hm[r][0] = ~get_int_b2(bx->hmask, t); // a set bit means that 4 is subtracted + hm[r][1] = ~get_int_b2(bx->hmask + 16, t); +#pragma unroll + for (int ksc = 0; ksc < 4; ++ksc) { + sc[r][ksc] = unpack_scales_q3_K(bx->scales, ksc); + } + d[r] = __half2float(bx->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int n = 0; n < 2; ++n) { + int qs[2][2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q3_K * bx = (const block_q3_K *) x[r]; + qs[r][0] = get_int_b2(bx->qs + 32*n, t); + qs[r][1] = get_int_b2(bx->qs + 32*n + 16, t); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int s = 4*n + j; + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = __vsub4((qs[r][0] >> (2*j)) & 0x03030303, ((hm[r][0] >> s) & 0x01010101) << 2); + hi[r] = __vsub4((qs[r][1] >> (2*j)) & 0x03030303, ((hm[r][1] >> s) & 0x01010101) << 2); + } + const int sc_s[2] = {sc[0][s/2] >> (16*(s % 2)), sc[1][s/2] >> (16*(s % 2))}; // scales 2s and 2s+1 + mmq_few_mma_k16_add<GGML_TYPE_Q3_K>(lo, hi, sc_s, yf, s, sum_d); + } + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template <int ncg> +static __device__ __forceinline__ void mmq_few_vec_dot_iq4_xs(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; + int scales_h[2]; + int scales_l[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq4_xs * bx = (const block_iq4_xs *) x[r]; + d[r] = __half2float(bx->d); + scales_h[r] = bx->scales_h; + scales_l[r] = get_int_b4(bx->scales_l, 0); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + int ls[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq4_xs * bx = (const block_iq4_xs *) x[r]; + const int2 v = get_int_from_table_16(get_int_b4(bx->qs + 16*s, t), kvalues_iq4nl); + lo[r] = v.x; + hi[r] = v.y; + ls[r] = (((scales_l[r] >> (4*s)) & 0x0F) | (((scales_h[r] >> (2*s)) & 0x03) << 4)) - 32; + } + mmq_few_mma_add<GGML_TYPE_IQ4_XS>(lo, hi, ls, yf, s, sum_d); + } + mmq_few_add_scaled(sum, d, sum_d); +} + +// Q8_0 and IQ4_NL have a block with its own scale per 32 values. +template <ggml_type type, int ncg> +static __device__ __forceinline__ void mmq_few_vec_dot_q8_0_iq4_nl(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + if constexpr (type == GGML_TYPE_Q8_0) { + const block_q8_0 * bx = (const block_q8_0 *) x[r] + s; + lo[r] = get_int_b2(bx->qs, t); + hi[r] = get_int_b2(bx->qs + 16, t); + d[r] = __half2float(bx->d); + } else { + const block_iq4_nl * bx = (const block_iq4_nl *) x[r] + s; + const int2 v = get_int_from_table_16(get_int_b2(bx->qs, t), kvalues_iq4nl); + lo[r] = v.x; + hi[r] = v.y; + d[r] = __half2float(bx->d); + } + } + mmq_few_mma_add<type>(lo, hi, d, yf, s, sum); + } +} + +template <int ncg> +static __device__ __forceinline__ void mmq_few_vec_dot_iq3_s(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + d[r] = __half2float(((const block_iq3_s *) x[r])->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + int ls[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq3_s * bx = (const block_iq3_s *) x[r]; + const int qh = bx->qh[s]; + // the values [4k, 4k + 4) of a sub-block come from grid entry k, with the signs in bits 4*(k%2) of byte k/2 + const int k_lo = t; + const int k_hi = 4 + t; + const int grid_lo = iq3s_grid[bx->qs[8*s + k_lo] | (((qh >> k_lo) & 1) << 8)]; + const int grid_hi = iq3s_grid[bx->qs[8*s + k_hi] | (((qh >> k_hi) & 1) << 8)]; + const int sign_lo = (bx->signs[4*s + k_lo/2] >> (4*(k_lo % 2))) & 0x0F; + const int sign_hi = (bx->signs[4*s + k_hi/2] >> (4*(k_hi % 2))) & 0x0F; + const int mask_lo = __vcmpne4(((sign_lo & 0x03) << 7) | ((sign_lo & 0x0C) << 21), 0x00000000); + const int mask_hi = __vcmpne4(((sign_hi & 0x03) << 7) | ((sign_hi & 0x0C) << 21), 0x00000000); + lo[r] = __vsub4(grid_lo ^ mask_lo, mask_lo); + hi[r] = __vsub4(grid_hi ^ mask_hi, mask_hi); + ls[r] = 1 + 2*((bx->scales[s/2] >> (4*(s % 2))) & 0x0F); + } + mmq_few_mma_add<GGML_TYPE_IQ3_S>(lo, hi, ls, yf, s, sum_d); + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template <ggml_type type, int ncg> +static __device__ __forceinline__ void mmq_few_vec_dot(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + if constexpr (type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL) { + mmq_few_vec_dot_q8_0_iq4_nl<type, ncg>(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q3_K) { + mmq_few_vec_dot_q3_K<ncg>(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + mmq_few_vec_dot_q45_K<type, ncg>(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q6_K) { + mmq_few_vec_dot_q6_K<ncg>(x, yf, sum); + } else if constexpr (type == GGML_TYPE_IQ3_S) { + mmq_few_vec_dot_iq3_s<ncg>(x, yf, sum); + } else if constexpr (type == GGML_TYPE_IQ4_XS) { + mmq_few_vec_dot_iq4_xs<ncg>(x, yf, sum); + } else { + static_assert(type == GGML_TYPE_COUNT, "unsupported type"); + } +} + +#endif // defined(CP_ASYNC_AVAILABLE) + +template <ggml_type type, int ncg, int nmt> +__launch_bounds__(MMQ_FEW_NWARPS*ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_q_few( + const char * x, const int4 * y_ptr, float * dst, + const int nunits, const int nrows_x, const int ncols_dst, const int64_t stride_row_x, const int stride_col_dst, + const int channel_ratio, const int64_t stride_channel_x, const int stride_channel_dst, + const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant) { +#if defined(CP_ASYNC_AVAILABLE) + constexpr int nwarps = MMQ_FEW_NWARPS; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + static_assert(warp_size == WARP_SIZE, "unsupported warp size"); + + constexpr int unit_size = mmq_few_unit_size(type); + constexpr int nrows = nmt*MMQ_FEW_NROWS; + constexpr int nunits_stage = mmq_few_nunits_stage(type, ncg, nmt); + constexpr int nstages = mmq_few_nstages(type); + constexpr bool unaligned = mmq_few_unaligned(type, nunits_stage); + constexpr int nchunks = mmq_few_nchunks(type, nunits_stage); + constexpr int sram_stride = mmq_few_sram_stride(type, nunits_stage); + constexpr int nbytes_stage = nrows*sram_stride; + constexpr int ncopies = nrows*nchunks; + static_assert(unit_size > 0, "unsupported type"); + + extern __shared__ char data_mmq_few[]; + char * ring = data_mmq_few + threadIdx.y*nstages*nbytes_stage; + + const int row0 = blockIdx.x*nrows; + const int channel_dst = blockIdx.y; + const int sample_dst = blockIdx.z; + const int nchannels = gridDim.y; + + x += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x; + dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst; + const int4 * GGML_CUDA_RESTRICT y = y_ptr + (int64_t(sample_dst)*nchannels + channel_dst)*nunits*ncg*MMQ_FEW_Q8_1_INT4; + + // No bounds check for the units, the gate makes nunits a multiple of nunits_stage. + // The extra bytes of unaligned data are inside the allocation, which ggml aligns and pads to 128 bytes. + auto load_stage = [&](const int stage, const int iu0) { + const unsigned int dst_stage = ggml_cuda_cvta_generic_to_shared(ring + stage*nbytes_stage); +#pragma unroll + for (int c0 = 0; c0 < ncopies; c0 += warp_size) { + const int c = c0 + threadIdx.x; + const int i = c / nchunks; + const int ic = c % nchunks; + if (ncopies % warp_size == 0 || c < ncopies) { + const int row = min(row0 + i, nrows_x - 1); + const char * src = x + int64_t(row)*stride_row_x + int64_t(iu0)*unit_size; + if constexpr (unaligned) { + const int off = (uintptr_t) src % 16; + if (16*ic < off + nunits_stage*unit_size) { + cp_async_cg_16<0>(dst_stage + i*sram_stride + 16*ic, src - off + 16*ic); + } + } else { + cp_async_cg_16<0>(dst_stage + i*sram_stride + 16*ic, src + 16*ic); + } + } + } + }; + + // offsets of the rows of this lane from their 16 byte boundary + int off_row[2*nmt] = {0}; + if constexpr (unaligned) { +#pragma unroll + for (int r = 0; r < 2*nmt; ++r) { + const int row = min(row0 + int(threadIdx.x)/4 + 8*r, nrows_x - 1); + off_row[r] = (uintptr_t) (x + int64_t(row)*stride_row_x) % 16; + } + } + + // Only weights, which no preceding kernel writes, are read before the PDL wait. + if (!x_is_constant) { + ggml_cuda_pdl_sync(); + } + +#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + // DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order. + { + const int tid = threadIdx.y*warp_size + threadIdx.x; + const int nrows_blk = min(nrows, nrows_x - row0); + const int64_t row_bytes = int64_t(nunits)*unit_size; + const bool contig = stride_row_x == row_bytes; + for (int ir = 0; ir < (contig ? 1 : nrows_blk); ++ir) { + const uintptr_t p0 = (uintptr_t) (x + int64_t(row0 + ir)*stride_row_x); + const uintptr_t p1 = p0 + (contig ? nrows_blk*row_bytes : row_bytes); + for (uintptr_t p = (p0 & ~uintptr_t(127)) + 128*tid; p < p1; p += 128*nwarps*warp_size) { + ggml_cuda_prefetch_l2((const void *) p); + } + } + } +#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + + const int iu_step = nunits_stage*nwarps; + +#pragma unroll + for (int stage = 0; stage < nstages - 1; ++stage) { + const int iu0 = nunits_stage*threadIdx.y + stage*iu_step; + if (iu0 < nunits) { + load_stage(stage, iu0); + } + cp_async_commit_group(); + } + if (x_is_constant) { + ggml_cuda_pdl_sync(); + } + + auto load_y = [&](mmq_few_y_frag (&yf)[ncg], const int iu) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { + mmq_few_load_y(yf[c], y + (int64_t(iu)*ncg + c)*MMQ_FEW_Q8_1_INT4); + } + }; + + // The next src1 fragments are loaded right after the MMAs that use the current ones, an earlier load is slower. + mmq_few_y_frag yf[nunits_stage][ncg]; + if (nunits_stage*threadIdx.y < nunits) { +#pragma unroll + for (int k = 0; k < nunits_stage; ++k) { + load_y(yf[k], nunits_stage*threadIdx.y + k); + } + } + + float sum[nmt][ncg][4] = {{{0.0f}}}; + + for (int iter = 0, iu0 = nunits_stage*threadIdx.y; iu0 < nunits; ++iter, iu0 += iu_step) { + const int iu_load = iu0 + (nstages - 1)*iu_step; + if (iu_load < nunits) { + load_stage((iter + nstages - 1) % nstages, iu_load); + } + cp_async_commit_group(); + + const bool has_next = iu0 + iu_step < nunits; + + cp_async_wait_group<nstages - 1>(); + ggml_cuda_syncwarp(); + + const char * xs = ring + (iter % nstages)*nbytes_stage; + const int off_stage = unaligned ? (iu0*unit_size) % 16 : 0; +#pragma unroll + for (int k = 0; k < nunits_stage; ++k) { +#pragma unroll + for (int m = 0; m < nmt; ++m) { + const char * const xr[2] = { + xs + (MMQ_FEW_NROWS*m + threadIdx.x/4 + 0)*sram_stride + (off_row[2*m + 0] + off_stage) % 16 + k*unit_size, + xs + (MMQ_FEW_NROWS*m + threadIdx.x/4 + 8)*sram_stride + (off_row[2*m + 1] + off_stage) % 16 + k*unit_size, + }; + mmq_few_vec_dot<type, ncg>(xr, yf[k], sum[m]); + } + if (has_next) { + load_y(yf[k], iu0 + iu_step + k); + } + } + + ggml_cuda_syncwarp(); // the stage is overwritten in the next iteration + } + cp_async_wait_all(); + + ggml_cuda_pdl_lc(); + + // reduce the sums over the warps as one flat array + constexpr int nsum = nmt*ncg*4; + float * sum_flat = &sum[0][0][0]; + + float * buf_sum = (float *) data_mmq_few; + __syncthreads(); + if (threadIdx.y > 0) { +#pragma unroll + for (int i = 0; i < nsum; ++i) { + buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i]; + } + } + __syncthreads(); + if (threadIdx.y > 0) { + return; + } +#pragma unroll + for (int w = 0; w < nwarps - 1; ++w) { +#pragma unroll + for (int i = 0; i < nsum; ++i) { + sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x]; + } + } + +#pragma unroll + for (int m = 0; m < nmt; ++m) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + const int i = row0 + MMQ_FEW_NROWS*m + tile<16, 8, int>::get_i(l); + const int j = c*MMQ_FEW_NCOLS + tile<16, 8, int>::get_j(l); + if (i < nrows_x && j < ncols_dst) { + dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l]; + } + } + } + } +#else + GGML_UNUSED_VARS(x, y_ptr, dst, nunits, nrows_x, ncols_dst, stride_row_x, stride_col_dst, + channel_ratio, stride_channel_x, stride_channel_dst, + sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant); + NO_DEVICE_CODE; +#endif // defined(CP_ASYNC_AVAILABLE) +} + +// ggml_is_constant (static in ggml.c) on the tensor that a view points to +static bool mmq_few_is_constant(const ggml_tensor * src0) { + const ggml_tensor * x_root = src0->view_src ? src0->view_src : src0; + return x_root->buffer && ggml_backend_buffer_get_usage(x_root->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + !(x_root->flags & GGML_TENSOR_FLAG_PARAM); +} + +template <ggml_type type, int ncg, int nmt> +static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, cudaStream_t stream) { + GGML_TENSOR_BINARY_OP_LOCALS; + + const int nunits = ne00 / QK_K; + GGML_ASSERT(nunits % mmq_few_nunits_stage(type, ncg, nmt) == 0); + + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; + const int nbytes_shared = mmq_few_nbytes_shared(type, ncg, nmt); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few<type, ncg, nmt>), nbytes_shared); + + const bool x_is_constant = mmq_few_is_constant(src0); + + const dim3 block_nums((ne01 + nmt*MMQ_FEW_NROWS - 1) / (nmt*MMQ_FEW_NROWS), ne12, ne13); + const dim3 block_dims(warp_size, MMQ_FEW_NWARPS, 1); + const ggml_cuda_kernel_launch_params params(block_nums, block_dims, nbytes_shared, stream); + ggml_cuda_kernel_launch(mul_mat_q_few<type, ncg, nmt>, params, + (const char *) src0->data, src1_q, (float *) dst->data, + nunits, (int) ne01, (int) ne1, (int64_t) nb01, (int) (nb1/sizeof(float)), + (int) (ne12/ne02), (int64_t) nb02, (int) (nb2/sizeof(float)), + (int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant); +} + +static int mmq_few_ncg(const int64_t ncols) { + return (ncols + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS; +} + +// With 2 column groups, 32-row tiles halve the src1 loads per src0 byte. With 3 column groups they spill registers. +static int mmq_few_nmt(const ggml_type type, const int ncg, const int64_t nrows_x) { + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + return mmq_few_tall_supported(type) && ncg == 2 && nrows_x >= int64_t(nsm)*2*MMQ_FEW_NROWS ? 2 : 1; +} + +template <ggml_type type> +static void launch_mul_mat_q_few_tall(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, cudaStream_t stream) { + if constexpr (mmq_few_tall_supported(type)) { + if (mmq_few_nmt(type, 2, src0->ne[1]) == 2) { + launch_mul_mat_q_few<type, 2, 2>(src0, src1, dst, src1_q, stream); + return; + } + } + launch_mul_mat_q_few<type, 2, 1>(src0, src1, dst, src1_q, stream); +} + +template <ggml_type type> +static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, const int ncg, cudaStream_t stream) { + switch (ncg) { + case 1: + launch_mul_mat_q_few<type, 1, 1>(src0, src1, dst, src1_q, stream); + break; + case 2: + launch_mul_mat_q_few_tall<type>(src0, src1, dst, src1_q, stream); + break; + case 3: + launch_mul_mat_q_few<type, 3, 1>(src0, src1, dst, src1_q, stream); + break; + default: + GGML_ABORT("fatal error"); + } +} + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; + + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + GGML_ASSERT(nb00 == ggml_type_size(src0->type)); + GGML_ASSERT(nb10 == sizeof(float)); + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS); + GGML_ASSERT(ne00 % QK_K == 0); + GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0); + + cudaStream_t stream = ctx.stream(); + + const int nunits = ne00 / QK_K; + const int ncg = mmq_few_ncg(ne11); + + ggml_cuda_pool_alloc<int4> src1_q(ctx.pool(), int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4); + quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float), + ne11, ne12, ne13, stream); + + switch (src0->type) { + case GGML_TYPE_Q8_0: + launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q8_0>(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q3_K: + launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q3_K>(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q4_K: + launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q4_K>(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q5_K: + launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q5_K>(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q6_K: + launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q6_K>(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ3_S: + launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ3_S>(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ4_NL: + launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_NL>(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ4_XS: + launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_XS>(src0, src1, dst, src1_q.get(), ncg, stream); + break; + default: + GGML_ABORT("unsupported type for few-row MMQ"); + } +} + +// Min. number of src1 columns for the few-row MMQ, 0 if it is not used for this type, GPU and row length. +static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00) { +#ifdef GGML_CUDA_FORCE_CUBLAS + return 0; +#endif // GGML_CUDA_FORCE_CUBLAS + + // NVIDIA Ampere to Blackwell + if (!ampere_mma_available(cc) || cc >= GGML_CUDA_CC_RUBIN) { + return 0; + } + // on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code + if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) { + return 0; + } + // with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there + if (mmq_few_unit_size(type) == 0 || ne00 < 8*QK_K) { + return 0; + } + if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10 + return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2; + } + switch (type) { // all GPUs except DGX Spark, tuned on RTX 5090 + case GGML_TYPE_Q3_K: return 3; + case GGML_TYPE_Q5_K: return 3; + case GGML_TYPE_Q4_K: return 4; + case GGML_TYPE_IQ4_NL: return 4; + case GGML_TYPE_Q6_K: return 4; + case GGML_TYPE_Q8_0: return 5; + case GGML_TYPE_IQ3_S: return 5; + case GGML_TYPE_IQ4_XS: return 6; + default: return 0; + } +} + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) { + const int min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0]); + if (min_ncols == 0 || src1->ne[1] < min_ncols || src1->ne[1] > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) { + return false; + } + // the row must hold a whole number of ring stages of the launched kernel + const int ncg = mmq_few_ncg(src1->ne[1]); + const int nmt = mmq_few_nmt(src0->type, ncg, src0->ne[1]); + const int nunits_stage = mmq_few_nunits_stage(src0->type, ncg, nmt); + if (src0->ne[0] % (nunits_stage*QK_K) != 0) { + return false; + } + // unaligned data needs the alignment of the decoder loads + const int x_alignment = mmq_few_unaligned(src0->type, nunits_stage) ? (ggml_type_size(src0->type) % 4 == 0 ? 4 : 2) : 16; + return ggml_cuda_is_aligned(src0, x_alignment) && ggml_cuda_is_aligned(src1, 16); +} + +#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_UNUSED_VARS(ctx, src0, src1, dst); + GGML_ABORT("few-row MMQ is not supported on HIP or MUSA"); +} + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) { + GGML_UNUSED_VARS(src0, src1, cc); + return false; +} + +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) diff --git a/ggml/src/ggml-cuda/mmq-few.cuh b/ggml/src/ggml-cuda/mmq-few.cuh new file mode 100644 index 000000000..016834998 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-few.cuh @@ -0,0 +1,7 @@ +#pragma once + +#include "common.cuh" + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, int cc); + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index d5654868b..9a5a6a405 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -5,13 +5,15 @@ #include "mmq.cuh" template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q1_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); #else @@ -46,6 +48,25 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ for (int j = 0; j < 2; ++j) { const int q = qxi[j]; +#if defined(GGML_USE_HIP) + const uint32_t q_bits = (uint16_t) q; + + // Expand each group of four Q1_0 bits into byte selectors for v_perm_b32. + const uint32_t q0_bits = q_bits >> 0; + const uint32_t q1_bits = q_bits >> 4; + const uint32_t q2_bits = q_bits >> 8; + const uint32_t q3_bits = q_bits >> 12; + + const uint32_t q0_indices = (q0_bits & 0x01) | ((q0_bits & 0x02) << 7) | ((q0_bits & 0x04) << 14) | ((q0_bits & 0x08) << 21); + const uint32_t q1_indices = (q1_bits & 0x01) | ((q1_bits & 0x02) << 7) | ((q1_bits & 0x04) << 14) | ((q1_bits & 0x08) << 21); + const uint32_t q2_indices = (q2_bits & 0x01) | ((q2_bits & 0x02) << 7) | ((q2_bits & 0x04) << 14) | ((q2_bits & 0x08) << 21); + const uint32_t q3_indices = (q3_bits & 0x01) | ((q3_bits & 0x02) << 7) | ((q3_bits & 0x04) << 14) | ((q3_bits & 0x08) << 21); + + const int v0 = ggml_hip_perm(0x000001FF, 0x000001FF, q0_indices); + const int v1 = ggml_hip_perm(0x000001FF, 0x000001FF, q1_indices); + const int v2 = ggml_hip_perm(0x000001FF, 0x000001FF, q2_indices); + const int v3 = ggml_hip_perm(0x000001FF, 0x000001FF, q3_indices); +#else // unpack crumbs into nibble indices const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13] const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15] @@ -59,6 +80,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ const int v1 = __byte_perm(s0, s1, 0x7632); const int v2 = __byte_perm(s2, s3, 0x5410); const int v3 = __byte_perm(s2, s3, 0x7632); +#endif // defined(GGML_USE_HIP) #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) x_qs[i*sram_stride + dst_offset + j*4+0] = v0; @@ -96,13 +118,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); #else @@ -142,8 +166,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18); const uint32_t qy_bits = q >> 8; const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18); - const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices); - const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices); + const int qx = ggml_hip_perm(0x020100FF, 0x020100FF, qx_indices); + const int qy = ggml_hip_perm(0x020100FF, 0x020100FF, qy_indices); #else // unpack even and odd crumbs into byte values const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); @@ -185,13 +209,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); #else @@ -248,13 +274,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_1( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); #else @@ -311,13 +339,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -391,13 +421,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_1( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); #else @@ -468,14 +500,14 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } } -template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { +template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K); #else @@ -499,7 +531,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx; + const block_q8_0 * bxi = ggml_cuda_mmq_x_row<block_q8_0, has_fusion>(x, x_gate, kbx0, i, stride) + kbx; #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) x_qs[i*sram_stride + 0 + txi] = get_int_b2(bxi[0].qs, kqsx); @@ -522,7 +554,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd; + const block_q8_0 * bxi = ggml_cuda_mmq_x_row<block_q8_0, has_fusion>(x, x_gate, kbx0, i, stride) + kbxd; #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) x_df[i*sram_stride + kbxd] = bxi->d; @@ -535,13 +567,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ // --------------------------------------------------------------------------------------------- template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); #else @@ -595,14 +629,23 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } } +// The 6-bit scales 4*ksc .. 4*ksc + 3 of a q3_K block, minus 32, as int8. +static __device__ __forceinline__ int unpack_scales_q3_K(const uint8_t * scales, const int ksc) { + const int sc_low = (get_int_b2(scales, ksc % 2) >> (4 * (ksc / 2))) & 0x0F0F0F0F; + const int sc_high = ((get_int_b2(scales, 2) >> (2 * ksc)) << 4) & 0x30303030; + return __vsub4(sc_low | sc_high, 0x20202020); +} + template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -659,15 +702,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ const int ksc = threadIdx.x % 4; - const int ksc_low = ksc % (QI3_K/8); - const int shift_low = 4 * (ksc / (QI3_K/8)); - const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F; - - const int ksc_high = QI3_K/8; - const int shift_high = 2 * ksc; - const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030; - - const int sc = __vsub4(sc_low | sc_high, 0x20202020); + const int sc = unpack_scales_q3_K(bxi->scales, ksc); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) const int8_t * sc8 = (const int8_t *) ≻ @@ -708,14 +743,14 @@ static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, co ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits } -template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { +template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); #else @@ -737,7 +772,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; + const block_q4_K * bxi = ggml_cuda_mmq_x_row<block_q4_K, has_fusion>(x, x_gate, kbx0, i, stride); const int qs0 = get_int_b4(bxi->qs, txi); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) @@ -766,7 +801,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; + const block_q4_K * bxi = ggml_cuda_mmq_x_row<block_q4_K, has_fusion>(x, x_gate, kbx0, i, stride); const int * scales = (const int *) bxi->scales; const int ksc = threadIdx.x % 2; @@ -780,7 +815,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f); #pragma unroll - for (int l = 0; l < sizeof(int); ++l) { + for (int l = 0; l < int(sizeof(int)); ++l) { x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]); } } @@ -794,7 +829,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; + const block_q4_K * bxi = ggml_cuda_mmq_x_row<block_q4_K, has_fusion>(x, x_gate, kbx0, i, stride); x_dm[i] = bxi->dm; } @@ -807,7 +842,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8); + const block_q4_K * bxi = ggml_cuda_mmq_x_row<block_q4_K, has_fusion>(x, x_gate, kbx0, i, stride) + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8); const int * scales = (const int *) bxi->scales; @@ -819,14 +854,14 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ #endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) } -template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { +template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -848,7 +883,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; + const block_q5_K * bxi = ggml_cuda_mmq_x_row<block_q5_K, has_fusion>(x, x_gate, kbx0, i, stride); const int ky = QR5_K*txi; const int ql = get_int_b4(bxi->qs, txi); @@ -889,7 +924,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; + const block_q5_K * bxi = ggml_cuda_mmq_x_row<block_q5_K, has_fusion>(x, x_gate, kbx0, i, stride); const int * scales = (const int *) bxi->scales; const int ksc = threadIdx.x % 2; @@ -917,7 +952,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; + const block_q5_K * bxi = ggml_cuda_mmq_x_row<block_q5_K, has_fusion>(x, x_gate, kbx0, i, stride); x_dm[i] = bxi->dm; } @@ -931,7 +966,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; + const block_q5_K * bxi = ggml_cuda_mmq_x_row<block_q5_K, has_fusion>(x, x_gate, kbx0, i, stride); const int * scales = (const int *) bxi->scales; @@ -943,14 +978,14 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ #endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) } -template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { +template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K); @@ -973,7 +1008,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride; + const block_q6_K * bxi = ggml_cuda_mmq_x_row<block_q6_K, has_fusion>(x, x_gate, kbx0, i, stride); const int ql = get_int_b2(bxi->ql, txi); const int ql0 = (ql >> 0) & 0x0F0F0F0F; @@ -1003,7 +1038,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride; + const block_q6_K * bxi = ggml_cuda_mmq_x_row<block_q6_K, has_fusion>(x, x_gate, kbx0, i, stride); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) x_df[i*sram_stride] = bxi->d; @@ -1021,7 +1056,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4; + const block_q6_K * bxi = ggml_cuda_mmq_x_row<block_q6_K, has_fusion>(x, x_gate, kbx0, i, stride) + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4; #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8)); @@ -1034,13 +1069,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ // --------------------------------------------------------------------------------------------- template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_s( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1096,13 +1133,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1160,13 +1199,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xs( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1225,13 +1266,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_s( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1293,13 +1336,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_xxs( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1357,13 +1402,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_s( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1425,14 +1472,14 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } } -template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { +template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1453,7 +1500,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride; + const block_iq4_xs * bxi = ggml_cuda_mmq_x_row<block_iq4_xs, has_fusion>(x, x_gate, kbx0, i, stride); const int aux_q4 = get_int_b4(bxi->qs, kqsx); const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); @@ -1477,7 +1524,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride; + const block_iq4_xs * bxi = ggml_cuda_mmq_x_row<block_iq4_xs, has_fusion>(x, x_gate, kbx0, i, stride); const float d = __half2float(bxi->d); @@ -1493,13 +1540,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_nl( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1562,13 +1611,15 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ // --------------------------------------------------------------------------------------------- template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1629,7 +1680,9 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4_fp4( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4); @@ -1670,14 +1723,14 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } } -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4( - const char * __restrict__ x, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) { +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); int * x_qs = (int *) x_tile; float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); #else @@ -1699,7 +1752,7 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_ i = min(i, i_max); } - const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx; + const block_nvfp4 * bxi = ggml_cuda_mmq_x_row<block_nvfp4, has_fusion>(x, x_gate, kb0, i, stride) + kbx; const uint32_t * __restrict__ src_qs = reinterpret_cast<const uint32_t *>(bxi->qs); const int kqs = 16 * kbx; const int ksc = 4 * kbx; @@ -1726,8 +1779,8 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_ } } -template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { +template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4); @@ -1742,7 +1795,6 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ const int kbx = txi % threads_per_row; const int row_in_warp = txi / threads_per_row; - const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx; uint32_t * x_u32_scale = x_u32 + 64 + kbx; #pragma unroll @@ -1753,7 +1805,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ i = min(i, i_max); } - const block_nvfp4 * bxi = bxi_base + i * stride; + const block_nvfp4 * bxi = ggml_cuda_mmq_x_row<block_nvfp4, has_fusion>(x, x_gate, kbx0, i, stride) + kbx; const uint32_t * src_qs = reinterpret_cast<const uint32_t *>(bxi->qs); diff --git a/ggml/src/ggml-cuda/mmq-vec-dot.cuh b/ggml/src/ggml-cuda/mmq-vec-dot.cuh index b39e4d579..a75dd046e 100644 --- a/ggml/src/ggml-cuda/mmq-vec-dot.cuh +++ b/ggml/src/ggml-cuda/mmq-vec-dot.cuh @@ -10,8 +10,8 @@ using namespace ggml_cuda_mma; template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I); const int * x_qs = (const int *) x; @@ -60,8 +60,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I); const int * x_qs = (const int *) x; @@ -110,8 +110,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); const int * x_qs = (const int *) x; @@ -148,8 +148,8 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( typedef tile<16, 8, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -203,8 +203,8 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( typedef tile< 8, 8, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -281,8 +281,8 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I); const int * x_qs = (const int *) x; @@ -318,8 +318,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ typedef tile<16, 8, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -368,8 +368,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ typedef tile< 8, 8, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -442,8 +442,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, I); const int * x_qs = (const int *) x; @@ -474,7 +474,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ } // Used for Q3_K, IQ2_S, and IQ2_XS: -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma( +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) constexpr data_layout input_layout = get_input_data_layout(); @@ -483,7 +483,7 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -533,7 +533,7 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_ typedef tile<16, 8, int> tile_C; constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -610,8 +610,8 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I); const int * x_qs = (const int *) x; @@ -680,8 +680,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ typedef tile<16, 4, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -749,8 +749,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ typedef tile< 8, 4, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -870,8 +870,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I); const int * x_qs = (const int *) x; @@ -905,8 +905,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I); const int * x_qs = (const int *) x; @@ -940,8 +940,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I); const int * x_qs = (const int *) x; @@ -975,8 +975,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I); const int * x_qs = (const int *) x; @@ -1015,8 +1015,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ typedef tile<16, 4, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -1066,8 +1066,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ typedef tile< 8, 4, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -1181,7 +1181,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ typedef tile<16, 8, float> tile_C; constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q4); constexpr int ntx = rows_per_warp / tile_C::I; constexpr int nfrags = MMQ_TILE_NE_K / tile_A::J; diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 43603bb8a..2e6199766 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -8,66 +8,66 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream, const ggml_prec prec_src1) { switch (args.type_x) { case GGML_TYPE_Q1_0: - mul_mat_q_case<GGML_TYPE_Q1_0>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q1_0, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q2_0: - mul_mat_q_case<GGML_TYPE_Q2_0>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q2_0, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q4_0: - mul_mat_q_case<GGML_TYPE_Q4_0>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q4_0, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q4_1: - mul_mat_q_case<GGML_TYPE_Q4_1>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q4_1, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q5_0: - mul_mat_q_case<GGML_TYPE_Q5_0>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q5_0, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q5_1: - mul_mat_q_case<GGML_TYPE_Q5_1>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q5_1, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q8_0: - mul_mat_q_case<GGML_TYPE_Q8_0>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q8_0, GGML_PREC_Q8>(ctx, args, stream); break; // ----------------------------------------------------------------------- case GGML_TYPE_Q2_K: - mul_mat_q_case<GGML_TYPE_Q2_K>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q2_K, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q3_K: - mul_mat_q_case<GGML_TYPE_Q3_K>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q3_K, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q4_K: - mul_mat_q_case<GGML_TYPE_Q4_K>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q4_K, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q5_K: - mul_mat_q_case<GGML_TYPE_Q5_K>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q5_K, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_Q6_K: - mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_Q6_K, GGML_PREC_Q8>(ctx, args, stream); break; // ----------------------------------------------------------------------- case GGML_TYPE_IQ1_S: - mul_mat_q_case<GGML_TYPE_IQ1_S>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_IQ2_XXS: - mul_mat_q_case<GGML_TYPE_IQ2_XXS>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_IQ2_XXS, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_IQ2_XS: - mul_mat_q_case<GGML_TYPE_IQ2_XS>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_IQ2_XS, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_IQ2_S: - mul_mat_q_case<GGML_TYPE_IQ2_S>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_IQ2_S, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_IQ3_XXS: - mul_mat_q_case<GGML_TYPE_IQ3_XXS>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_IQ3_XXS, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_IQ3_S: - mul_mat_q_case<GGML_TYPE_IQ3_S>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_IQ3_S, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_IQ4_XS: - mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_IQ4_XS, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_IQ4_NL: - mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_IQ4_NL, GGML_PREC_Q8>(ctx, args, stream); break; // ----------------------------------------------------------------------- case GGML_TYPE_MXFP4: @@ -76,18 +76,17 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con mul_mat_q_case<GGML_TYPE_MXFP4, GGML_PREC_Q4>(ctx, args, stream); break; } - mul_mat_q_case<GGML_TYPE_MXFP4>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_MXFP4, GGML_PREC_Q8>(ctx, args, stream); break; case GGML_TYPE_NVFP4: if (prec_src1 == GGML_PREC_Q4) { mul_mat_q_case<GGML_TYPE_NVFP4, GGML_PREC_Q4>(ctx, args, stream); break; } - mul_mat_q_case<GGML_TYPE_NVFP4>(ctx, args, stream); + mul_mat_q_case<GGML_TYPE_NVFP4, GGML_PREC_Q8>(ctx, args, stream); break; default: GGML_ABORT("fatal error"); - break; } } @@ -115,33 +114,91 @@ static ggml_prec ggml_cuda_mmq_get_prec_env() { // src1 is quantized to Q8_1 unless the FP4 types can use 4-bit activations, in which case they // default to the native W4A4 instructions on Blackwell. -static ggml_prec ggml_cuda_mmq_get_prec_src1(const ggml_tensor * src0, const ggml_tensor * dst, const int cc) { +static ggml_prec ggml_cuda_mmq_get_prec_src1(const ggml_type type, const ggml_prec prec_requested, const int cc) { static const ggml_prec prec_env = ggml_cuda_mmq_get_prec_env(); ggml_prec prec = prec_env; if (prec == GGML_PREC_UNDEFINED) { - prec = (ggml_prec) ggml_get_op_params_i32(dst, 3); + prec = prec_requested; } // Q4 only for the FP4 types on Blackwell GGML_ASSERT(prec == GGML_PREC_UNDEFINED || prec == GGML_PREC_Q8 || prec == GGML_PREC_Q4); - const bool can_use_q4 = (src0->type == GGML_TYPE_NVFP4 || src0->type == GGML_TYPE_MXFP4) && blackwell_mma_available(cc); + const bool can_use_q4 = (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4) && blackwell_mma_available(cc); if (prec == GGML_PREC_Q8 || !can_use_q4) { return GGML_PREC_Q8; } return GGML_PREC_Q4; } +// Tile width in ne11 (dense) / ne12 (MoE) direction with the fewest tiles that fits into shared memory, 0 if there is none. +static int ggml_cuda_mmq_get_J_best( + const ggml_type type, const int64_t ncols, const bool fallback, const int cc, const size_t smpbo, const ggml_prec prec_src1, + int * nthreads_best = nullptr) { + int J_best = 0; + int ntiles_J_best = INT_MAX; + + for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) { + const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1); + if (config.type == GGML_TYPE_COUNT) { + continue; + } + + if (mmq_get_nbytes_shared(config, cc) > smpbo) { + continue; + } + + const int ntiles_x = (ncols + config.J - 1) / config.J; + + if (ntiles_x < ntiles_J_best) { + J_best = J; + if (nthreads_best) { + *nthreads_best = config.nthreads; + } + ntiles_J_best = ntiles_x; + } + } + return J_best; +} + void ggml_cuda_mul_mat_q( - ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, + const ggml_cuda_mm_fusion_args_host * fusion) { GGML_ASSERT( src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID. + // Fused gate/up/GLU: src0 is the up weight, dst the GLU output. + // Fused scale (no gate): dst is the MUL that applies the weight scale to the MUL_MAT. + const ggml_tensor * gate = fusion ? fusion->gate : nullptr; + ggml_cuda_mm_fusion_args_device fusion_device = {}; + if (fusion) { + GGML_ASSERT(!fusion->x_bias && !fusion->gate_bias); + // NVFP4 weight scales (only applied for NVFP4): one value per tensor, or one per expert with ids. + for (const ggml_tensor * scale : { fusion->x_scale, fusion->gate_scale }) { + GGML_ASSERT(!scale || (src0->type == GGML_TYPE_NVFP4 && scale->type == GGML_TYPE_F32 && ggml_is_contiguous(scale) && + ggml_nelements(scale) == (ids ? src0->ne[2] : 1))); + } + if (gate) { + GGML_ASSERT(!fusion->x_scale == !fusion->gate_scale); + GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0)); + fusion_device.gate = gate->data; + fusion_device.gate_scale = fusion->gate_scale ? fusion->gate_scale->data : nullptr; + fusion_device.glu_op = fusion->glu_op; + fusion_device.glu_limit = fusion->glu_limit; + } else { + GGML_ASSERT(!fusion->gate_scale && fusion->x_scale); + } + fusion_device.x_scale = fusion->x_scale ? fusion->x_scale->data : nullptr; + } + GGML_TENSOR_BINARY_OP_LOCALS; cudaStream_t stream = ctx.stream(); - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + const size_t smpbo = ggml_cuda_info().devices[id].smpbo; const size_t ts_src0 = ggml_type_size(src0->type); const size_t ts_src1 = ggml_type_size(src1->type); @@ -156,14 +213,16 @@ void ggml_cuda_mul_mat_q( const float * src1_d = (const float *) src1->data; float * dst_d = (float *) dst->data; - // If src0 is a temporary compute buffer, clear any potential padding. - if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { - const size_t size_data = ggml_nbytes(src0); - const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0); - if (size_alloc > size_data) { - GGML_ASSERT(ggml_is_contiguously_allocated(src0)); - GGML_ASSERT(!src0->view_src); - CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream)); + // If src0 (or the fused gate weight) is a temporary compute buffer, clear any potential padding. + for (const ggml_tensor * t : { src0, gate }) { + if (t && ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + const size_t size_data = ggml_nbytes(t); + const size_t size_alloc = ggml_backend_buffer_get_alloc_size(t->buffer, t); + if (size_alloc > size_data) { + GGML_ASSERT(ggml_is_contiguously_allocated(t)); + GGML_ASSERT(!t->view_src); + CUDA_CHECK(cudaMemsetAsync((char *) t->data + size_data, 0, size_alloc - size_data, stream)); + } } } @@ -176,17 +235,41 @@ void ggml_cuda_mul_mat_q( const int64_t s03 = src0->nb[3] / ts_src0; const int64_t s3 = dst->nb[3] / ts_dst; - const bool fallback = ne01 % 128 != 0; + const bool fallback = ggml_cuda_mmq_needs_fallback(ne01); - const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc); + // With fusion dst is the GLU output or the scale node, not a MUL_MAT; the src1 precision requested by the fused MUL_MATs is in the fusion args. + const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0->type, fusion ? fusion->prec_src1 : ggml_prec_get_src(dst, 1), cc); const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4; const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq); const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ; + int J_best = 0; + int nthreads_best = 0; + { + int64_t ncols_opt = ne11; + if (ids) { + const int64_t n_expert_used = ids->ne[0]; + ncols_opt = ne12; + + // Each expert only sees ne12*n_expert_used/ne02 tokens on average. + ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02; + } + + J_best = ggml_cuda_mmq_get_J_best(src0->type, ncols_opt, fallback, cc, smpbo, prec_src1, &nthreads_best); + } + GGML_ASSERT(J_best > 0); + + // A tile of size J can read in at most J - 1 extra columns. + // For simplicity, round up the padding of a full tile to a multiple of the number of bytes that nthreads can load in parallel. + const size_t src1_load_chunk_size = nthreads_best * sizeof(int); + const size_t src1_q8_1_padding = ((J_best * sizeof(block_q8_1_mmq) + src1_load_chunk_size - 1) / src1_load_chunk_size) + * src1_load_chunk_size; + if (!ids) { - const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * y_block_size/y_values_per_block + - ggml_cuda_mmq_get_J_max(src0->type, fallback, cc, ne11) * sizeof(block_q8_1_mmq); + const int ntj = (ne1 + J_best - 1) / J_best; + + const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding; ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1); ggml_cuda_pool_alloc<float> src1_scale(ctx.pool()); if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) { @@ -223,7 +306,7 @@ void ggml_cuda_mul_mat_q( ne00, ne01, ne1, s01, ne11, s1, ne02, ne12, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne1, ne1}; + ntj, J_best, fusion_device}; ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1); return; } @@ -237,8 +320,10 @@ void ggml_cuda_mul_mat_q( GGML_ASSERT(ne1 == n_expert_used); ggml_cuda_pool_alloc<int32_t> ids_src1(ctx.pool(), ne_get_rows); - ggml_cuda_pool_alloc<int32_t> ids_dst(ctx.pool(), ne_get_rows); - ggml_cuda_pool_alloc<int32_t> expert_bounds(ctx.pool(), ne02 + 1); + ggml_cuda_pool_alloc<int32_t> ids_dst(ctx.pool(), ne_get_rows + J_best-1); // Needs to be padded for unconditional memory access. + + const int ntj = (ne_get_rows + J_best - 1) / J_best + ne02; + ggml_cuda_pool_alloc<ggml_cuda_mm_id_tile_spec> tile_spec(ctx.pool(), ntj); // gate/up activations are broadcast across experts (ne11 == 1): quantize each token once and // scatter to its slots. ids_src1 then holds the inverse map (token slot -> compact row). @@ -249,13 +334,12 @@ void ggml_cuda_mul_mat_q( const int si1 = ids->nb[1] / ggml_element_size(ids); const int sis1 = nb12 / nb11; - ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(), - ne02, ne12, n_expert_used, ne11, si1, sis1, /*write_inverse =*/ dedup_bcast, stream); + ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), tile_spec.get(), + ne02, ne12, n_expert_used, ne11, J_best, si1, sis1, /*write_inverse =*/ dedup_bcast, stream); CUDA_CHECK(cudaGetLastError()); } - const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + - ggml_cuda_mmq_get_J_max(src0->type, fallback, cc, ne12) * sizeof(block_q8_1_mmq); + const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding; ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1); ggml_cuda_pool_alloc<float> src1_scale(ctx.pool()); if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) { @@ -296,21 +380,14 @@ void ggml_cuda_mul_mat_q( ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); const int64_t s13 = ne12*s12; - // Each expert only sees ne12*n_expert_used/ne02 tokens on average. - // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12. - int64_t ncols_opt = ne12; - if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) { - ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02; - } - // Note that ne02 is used instead of ne12 because the number of y channels determines the z dimension of the CUDA grid. const mmq_args args = { - src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), dst_d, + src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), tile_spec.get(), dst_d, src1_scale.ptr, ne00, ne01, ne_get_rows, s01, ne_get_rows, s1, - ne02, ne02, s02, s12, s2, + ne02, 1, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne12, ncols_opt}; + ntj, J_best, fusion_device}; ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1); } @@ -406,7 +483,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t } if (amd_wmma_available(cc)) { - if (GGML_CUDA_CC_IS_RDNA3(cc)) { + if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) { // High expert counts are almost always better on MMQ due to // the synchronization overhead in the cuBLAS/hipBLAS path: // https://github.com/ggml-org/llama.cpp/pull/18202 @@ -423,7 +500,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256); case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S: - return GGML_CUDA_CC_IS_RDNA3_5(cc) || ne11 <= 128; + return GGML_CUDA_CC_IS_RDNA3_5(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || ne11 <= 128; default: return true; } @@ -448,3 +525,24 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE; } + +bool ggml_cuda_mmq_fusion_is_efficient( + const ggml_type type, const ggml_prec prec_requested, const int64_t nrows_x, const int64_t ncols, const int64_t nchannels, const int device) { + const int cc = ggml_cuda_info().devices[device].cc; + const int nsm = ggml_cuda_info().devices[device].nsm; + const size_t smpbo = ggml_cuda_info().devices[device].smpbo; + + // Same tile width as ggml_cuda_mul_mat_q; the fused kernel has no fallback and writes I/2 output rows per tile. + const bool fallback = false; + const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(type, prec_requested, cc); + const int J_best = ggml_cuda_mmq_get_J_best(type, ncols, fallback, cc, smpbo, prec_src1); + if (J_best == 0) { + return false; + } + const int64_t nrows_tile = ggml_cuda_mmq_get_config(type, J_best, fallback, cc, prec_src1).I / 2; + const int64_t ntiles = (ncols + J_best - 1)/J_best * ((nrows_x + nrows_tile - 1)/nrows_tile) * nchannels; + // Same condition as launch_mul_mat_q uses to run the stream-k kernel with one block per tile. + const int64_t tiles_nwaves = (ntiles + nsm - 1) / nsm; + const int64_t tiles_efficiency_percent = 100 * ntiles / (nsm*tiles_nwaves); + return tiles_efficiency_percent >= 90; +} diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 0cd31d916..d645a8aac 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1,6 +1,8 @@ #pragma once #include "common.cuh" +#include "mmid.cuh" +#include "unary.cuh" #include <climits> #include <cstdint> @@ -10,10 +12,12 @@ #define MMQ_ITER_K_FP4 512 #define MMQ_NWARPS 8 -typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride); +typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, const char * __restrict__ x_gate, int * x_tile, + const int kbx0, const int i_max, const int stride); typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00); -typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted, - float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max); +typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, + float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_sup, + const ggml_glu_op glu_op, const float glu_limit, const float * __restrict__ x_scale, const float * __restrict__ gate_scale); enum mmq_q8_1_ds_layout { MMQ_Q8_1_DS_LAYOUT_D4, @@ -96,7 +100,6 @@ static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) { return MMQ_Q8_1_DS_LAYOUT_D4; default: GGML_ABORT("fatal error"); - break; } } @@ -208,7 +211,7 @@ struct ggml_cuda_mmq_config { static_assert((nthreads_) % 32 == 0 && (nthreads_) <= 512, "bad nthreads"); \ static_assert( (occupancy_) <= 8, "bad occupancy"); \ static_assert((I_) % 32 == 0, "bad I"); \ - static_assert((J_) % 8 == 0, "bad J"); \ + static_assert((J_) % 8 == 0 && (J_) <= 128, "bad J"); \ static_assert((K_vram_) % 256 == 0, "bad K_vram"); \ return ggml_cuda_mmq_config((type_), (nthreads_), (occupancy_), (I_), (J_), (sram_layout_), (K_vram_), (stream_k_), (fallback_)); \ } \ @@ -227,7 +230,7 @@ struct ggml_cuda_mmq_config { #undef CASE -static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1 = GGML_PREC_Q8) { +static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { if (GGML_CUDA_CC_IS_AMD(cc)) { if (GGML_CUDA_CC_IS_GCN(cc)) { return ggml_cuda_mmq_get_config_gcn(type, J, fallback); @@ -238,6 +241,9 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty if (GGML_CUDA_CC_IS_RDNA4(cc)) { return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); } + if (GGML_CUDA_CC_IS_RDNA4_M(cc)) { + return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); + } if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); } @@ -262,7 +268,7 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback); } -static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { #ifdef GGML_USE_HIP #ifdef GCN return ggml_cuda_mmq_get_config_gcn(type, J, fallback); @@ -270,6 +276,8 @@ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_t return ggml_cuda_mmq_get_config_cdna(type, J, fallback); #elif defined(RDNA4) return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); +#elif defined(RDNA4_M) + return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); #elif defined(RDNA3_5) return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); #elif defined(RDNA3) @@ -295,93 +303,86 @@ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_t GGML_UNUSED_VARS(type, J, fallback, prec_src1); } -static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).type; +static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).type; } -static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type; } -static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).nthreads; } -static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).occupancy; } -static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).I; +static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).I; } -static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).I; } -static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).J; +static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).J; } -static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).J; } -static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).sram_layout; +static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).sram_layout; } -static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).sram_layout; } -static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).K_vram; +static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).K_vram; } -static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).K_vram; } -static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).stream_k; +static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).stream_k; } -static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).stream_k; } -static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).fallback; +static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).fallback; } -static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).fallback; } // --------------------------------------------------------------------------------------------- -static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc)); +static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc, prec_src1)); } -static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, prec_src1)); } -static __host__ int ggml_cuda_mmq_get_J_max(const ggml_type type, const bool fallback, const int cc, const int64_t ne11) { - int ret = std::min(ne11, int64_t(512)); - ret -= ret % 8; - for (;ret > 0; ret -= 8) { - if (ggml_cuda_mmq_get_config(type, ret, fallback, cc).type != GGML_TYPE_COUNT) { - return ret; - } - } - return ret; +static __host__ bool ggml_cuda_mmq_needs_fallback(const int64_t nrows_x) { + return nrows_x % 128 != 0; } -static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback) { - return ggml_cuda_mmq_get_config(type, J, fallback).rows_per_warp(); +static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).rows_per_warp(); } #define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_0 + I/QI4_0, 0} @@ -434,23 +435,53 @@ static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config // ------------------------------------------------------------ +// Weight types with a fused gate/up/GLU path (see ggml_cuda_mmq_get_util_funcs). +static constexpr __host__ __device__ bool ggml_cuda_mmq_fusion_supported(const ggml_type type) { + return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_XS || + type == GGML_TYPE_NVFP4; +} + +// Fused gate/up/GLU: the weight tile alternates chunks of this many up and gate rows, so rows r and r + 8 of an MMA tile (same thread) pair up. +static constexpr int GGML_CUDA_MMQ_FUSION_ROWS = 8; + +// Source of weight tile row i; with has_fusion the up or gate row that the interleaved tile holds there. +template <typename block_t, bool has_fusion> +static __device__ __forceinline__ const block_t * ggml_cuda_mmq_x_row( + const char * __restrict__ x, const char * __restrict__ x_gate, const int kbx0, const int i, const int stride) { + if constexpr (has_fusion) { + constexpr int R = GGML_CUDA_MMQ_FUSION_ROWS; + const char * x_src = (i / R) % 2 == 0 ? x : x_gate; + return (const block_t *) x_src + kbx0 + ((i / (2*R))*R + i % R)*stride; + } else { + GGML_UNUSED(x_gate); + return (const block_t *) x + kbx0 + i*stride; + } +} + #include "mmq-load-tiles.cuh" #include "mmq-vec-dot.cuh" -template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst, - const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) { + const float * __restrict__ y_scale, const int stride, const int i_max, const int j_sup, + const ggml_glu_op glu_op, const float glu_limit, const float * __restrict__ x_scale, const float * __restrict__ gate_scale) { + GGML_UNUSED_VARS(glu_op, glu_limit, gate_scale); + + // NVFP4 weight scale of a fused MUL (per tensor, or of the expert of this tile), 1 without. + const float scale = type == GGML_TYPE_NVFP4 && x_scale ? *x_scale : 1.0f; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); + static_assert(I % warp_size == 0, "DP4A MMQ tile height must be a multiple of the physical warp size"); - const bool y_scale_used = y_scale != nullptr; + const bool y_scale_used = type == GGML_TYPE_NVFP4 && y_scale != nullptr; #pragma unroll for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; - if (j > j_max) { + if (j >= j_sup) { return; } @@ -462,69 +493,93 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_ continue; } + float value = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; if constexpr (type == GGML_TYPE_NVFP4) { if (y_scale_used) { - dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; - } else { - dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + value *= y_scale[j]; } } else { - dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; GGML_UNUSED(y_scale_used); } + dst[ids_dst[j]*stride + i] = value * scale; } } } -template<ggml_type type, int J, bool fallback> +// has_fusion: accumulators l and l + ne/2 hold the up and gate values of one output element, the GLU of the two is written. +// x_scale/gate_scale: optional NVFP4 scales of the up and gate weights (per tensor, or of the expert of this tile). +template<ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst, - const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) { + const float * __restrict__ y_scale, const int stride, const int i_max, const int j_sup, + const ggml_glu_op glu_op, const float glu_limit, const float * __restrict__ x_scale, const float * __restrict__ gate_scale) { #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; #else typedef tile<16, 8, int> tile_C; #endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + static_assert(!has_fusion || tile_C::I == 2*GGML_CUDA_MMQ_FUSION_ROWS, "each C tile must hold one chunk of up rows and one of gate rows"); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I); - const bool y_scale_used = y_scale != nullptr; + const bool y_scale_used = type == GGML_TYPE_NVFP4 && y_scale != nullptr; + + // NVFP4: load the weight scales once, 1 without scales. Without fusion x_scale is the scale of a fused MUL. + constexpr bool fused_nvfp4 = has_fusion && type == GGML_TYPE_NVFP4; + const float scale_up = type == GGML_TYPE_NVFP4 && x_scale ? *x_scale : 1.0f; + const float scale_gate = fused_nvfp4 && gate_scale ? *gate_scale : 1.0f; #pragma unroll for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { #pragma unroll for (int n = 0; n < ntx; ++n) { #pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { + for (int l = 0; l < (has_fusion ? tile_C::ne/2 : tile_C::ne); ++l) { const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l); - if (j > j_max) { + if (j >= j_sup) { continue; } - const int i = i0 + n*tile_C::I + tile_C::get_i(l); + const int i = (i0 + n*tile_C::I)/(has_fusion ? 2 : 1) + tile_C::get_i(l); if (fallback && i > i_max) { continue; } - if constexpr (type == GGML_TYPE_NVFP4) { - if (y_scale_used) { - dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]; - } else { - dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + if (has_fusion) { + const int sum_index = (j0/tile_C::J + n)*tile_C::ne + l; + float up = sum[sum_index]; + float gate = sum[sum_index + tile_C::ne/2]; + if constexpr (type == GGML_TYPE_NVFP4) { + if (y_scale_used) { + up *= y_scale[j]; + gate *= y_scale[j]; + } + up *= scale_up; + gate *= scale_gate; } + dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, gate, up, glu_limit); } else { - dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; - GGML_UNUSED(y_scale_used); + float value = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + if constexpr (type == GGML_TYPE_NVFP4) { + if (y_scale_used) { + value *= y_scale[j]; + } + } else { + GGML_UNUSED(y_scale_used); + } + dst[ids_dst[j]*stride + i] = value * scale_up; } } } } + + GGML_UNUSED_VARS(y_scale_used, glu_op, glu_limit, x_scale, gate_scale, scale_up, scale_gate); } // ------------------------------------------------------------------------------------------------------------------------------------- @@ -541,8 +596,9 @@ struct ggml_cuda_mmq_util_funcs { vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {} }; -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion> static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() { + static_assert(!has_fusion || ggml_cuda_mmq_fusion_supported(type), "no fused loader for this type"); if (!ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) { switch (type) { case GGML_TYPE_Q1_0: @@ -550,136 +606,136 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func VDR_Q1_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q2_0: return ggml_cuda_mmq_util_funcs( VDR_Q2_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q4_0: return ggml_cuda_mmq_util_funcs( VDR_Q4_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>, ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q4_1: return ggml_cuda_mmq_util_funcs( VDR_Q4_1_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>, ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q5_0: return ggml_cuda_mmq_util_funcs( VDR_Q5_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q5_1: return ggml_cuda_mmq_util_funcs( VDR_Q5_1_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q8_0: return ggml_cuda_mmq_util_funcs( VDR_Q8_0_Q8_1_MMQ, - ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>, + ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback, false>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); // --------------------------------------------------------------------------------------------- case GGML_TYPE_Q2_K: return ggml_cuda_mmq_util_funcs( VDR_Q2_K_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>, ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q3_K: return ggml_cuda_mmq_util_funcs( VDR_Q3_K_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>, ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q4_K: return ggml_cuda_mmq_util_funcs( VDR_Q4_K_Q8_1_MMQ, - ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>, + ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback, false>, ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q5_K: return ggml_cuda_mmq_util_funcs( VDR_Q5_K_Q8_1_MMQ, - ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>, + ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback, false>, ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_Q6_K: return ggml_cuda_mmq_util_funcs( VDR_Q6_K_Q8_1_MMQ, - ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>, + ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback, false>, ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); // --------------------------------------------------------------------------------------------- case GGML_TYPE_IQ1_S: return ggml_cuda_mmq_util_funcs( VDR_IQ1_S_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_IQ2_XXS: return ggml_cuda_mmq_util_funcs( VDR_IQ2_XXS_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_IQ2_XS: return ggml_cuda_mmq_util_funcs( VDR_IQ2_XS_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_IQ2_S: return ggml_cuda_mmq_util_funcs( VDR_IQ2_S_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_IQ3_XXS: return ggml_cuda_mmq_util_funcs( VDR_IQ3_XXS_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_IQ3_S: return ggml_cuda_mmq_util_funcs( VDR_IQ3_S_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_IQ4_XS: return ggml_cuda_mmq_util_funcs( VDR_IQ4_XS_Q8_1_MMQ, - ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>, + ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback, false>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_IQ4_NL: return ggml_cuda_mmq_util_funcs( VDR_IQ4_NL_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); // --------------------------------------------------------------------------------------------- case GGML_TYPE_MXFP4: return ggml_cuda_mmq_util_funcs( VDR_MXFP4_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); case GGML_TYPE_NVFP4: return ggml_cuda_mmq_util_funcs( VDR_NVFP4_Q8_1_MMQ, - ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback>, + ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback, prec_src1, false>, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>, - ggml_cuda_mmq_write_back_dp4a<type, J, fallback>); + ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>); default: return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); } @@ -695,16 +751,16 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func -1, ggml_cuda_mmq_load_tiles_mxfp4_fp4<type, J, fallback>, ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); } break; case GGML_TYPE_NVFP4: if (prec_src1 == GGML_PREC_Q4) { return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_nvfp4_nvfp4<type, J, fallback>, + ggml_cuda_mmq_load_tiles_nvfp4_nvfp4<type, J, fallback, has_fusion>, ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>); } break; default: @@ -720,178 +776,182 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func -1, ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_Q2_0: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_Q4_0: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_DS4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_Q4_1: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_Q5_0: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_Q5_1: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_Q8_0: return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>, + ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback, has_fusion>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>); // --------------------------------------------------------------------------------------------- case GGML_TYPE_Q2_K: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>, ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_Q3_K: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>, - ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>, + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_Q4_K: return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>, + ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback, has_fusion>, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>); case GGML_TYPE_Q5_K: return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>, + ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback, has_fusion>, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>); case GGML_TYPE_Q6_K: return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>, + ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback, has_fusion>, ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>); // --------------------------------------------------------------------------------------------- case GGML_TYPE_IQ1_S: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_IQ2_XXS: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_IQ2_XS: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>, - ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>, + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_IQ2_S: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>, - ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>, + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_IQ3_XXS: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_IQ3_S: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_IQ4_XS: return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>, + ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback, has_fusion>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>); case GGML_TYPE_IQ4_NL: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); // --------------------------------------------------------------------------------------------- case GGML_TYPE_MXFP4: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>); case GGML_TYPE_NVFP4: return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback, prec_src1>, + ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback, prec_src1, has_fusion>, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>, - ggml_cuda_mmq_write_back_mma<type, J, fallback>); + ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>); default: return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); } } -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static constexpr __device__ int ggml_cuda_mmq_get_vdr() { - return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().vdr; + return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1, false>().vdr; } -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion> static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() { - return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().load_tiles; + return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1, has_fusion>().load_tiles; } -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() { - return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().vec_dot; + return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1, false>().vec_dot; } -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion> static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() { - return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().write_back; + return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1, has_fusion>().write_back; } // --------------------------------------------------------------------------------------------- -template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1 = GGML_PREC_Q8> +// has_fusion: x and x_gate are the up and gate weights. +template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1, bool has_fusion> static __device__ __forceinline__ void mul_mat_q_process_tile( - const char * __restrict__ x, const int offset_x, const int * __restrict__ y, + const char * __restrict__ x, const char * __restrict__ x_gate, const int offset_x, const int * __restrict__ y, const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup, - const float * __restrict__ y_scale, + const float * __restrict__ y_scale, const ggml_glu_op glu_op, const float glu_limit, + const float * __restrict__ x_scale, const float * __restrict__ gate_scale, const int stride_row_x, const int ncols_y, const int stride_col_dst, - const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) { + const int tile_x_i_max, const int tile_y_j_sup, const int kb0_start, const int kb0_stop) { + static_assert(!(fixup && has_fusion), "the fused gate/up path does not split K"); + static_assert(!(fallback && has_fusion), "the fused gate/up path needs nrows_x % 128 == 0"); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; constexpr int qk = ggml_cuda_type_traits<type>::qk; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); - constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles<type, J, fallback, prec_src1>(); + constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles<type, J, fallback, prec_src1, has_fusion>(); constexpr ggml_cuda_mmq_vec_dot_t vec_dot = ggml_cuda_mmq_get_vec_dot<type, J, fallback, prec_src1>(); - constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back<type, J, fallback, prec_src1>(); + constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back<type, J, fallback, prec_src1, has_fusion>(); extern __shared__ int data_mul_mat_q[]; int * tile_y = data_mul_mat_q + J; @@ -914,7 +974,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int); for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) { - load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x); + load_tiles(x, x_gate, tile_x, offset_x + kb0, tile_x_i_max, stride_row_x); { const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz; #pragma unroll @@ -949,28 +1009,41 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( } if (fixup) { - write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J); + write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J, glu_op, glu_limit, x_scale, gate_scale); } else { - write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j); + write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_i_max, tile_y_j_sup, glu_op, glu_limit, x_scale, gate_scale); } } // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598 -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion> __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1)) static __global__ void mul_mat_q( - const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, - const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup, - const float * __restrict__ y_scale, + const char * __restrict__ x, const char * __restrict__ x_gate, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, + const ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, float * __restrict__ dst, float * __restrict__ tmp_fixup, + const float * __restrict__ y_scale, const ggml_glu_op glu_op, const float glu_limit, + const float * __restrict__ x_scale, const float * __restrict__ gate_scale, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst, const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, const uint3 sample_ratio, const uint3 nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, - const uint3 ntx) { + const uint3 ntj) { + +#ifdef TURING_MMA_AVAILABLE + constexpr bool fusion_available = true; +#else + constexpr bool fusion_available = false; +#endif // TURING_MMA_AVAILABLE +#ifdef BLACKWELL_MMA_AVAILABLE + constexpr bool native_fp4_available = true; +#else + constexpr bool native_fp4_available = false; +#endif // BLACKWELL_MMA_AVAILABLE - // Skip unused template specializations for faster compilation: - if (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type == GGML_TYPE_COUNT) { + // Skip unused template specializations for faster compilation (the host only uses 4-bit src1 with native FP4): + if (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type == GGML_TYPE_COUNT || + (has_fusion && (!fusion_available || (prec_src1 == GGML_PREC_Q4 && !native_fp4_available)))) { NO_DEVICE_CODE; return; } @@ -980,7 +1053,7 @@ static __global__ void mul_mat_q( constexpr int qk = ggml_cuda_type_traits<type>::qk; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); - const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y + const uint32_t nti = (nrows_x + I - 1) / I; // Number of tiles in I direction // Initialize the ids for writing back data with just the index. // For regular matrix multiplications this is never changed. @@ -998,38 +1071,35 @@ static __global__ void mul_mat_q( } __syncthreads(); - if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) { + // The fused kernel needs the whole K range of a tile in one block, so it always uses tiling. + if constexpr (has_fusion || !ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) { + // The fused kernel writes the GLU of I/2 up rows and I/2 gate rows per tile. + constexpr int I_dst = has_fusion ? I/2 : I; + const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); const int wt = tmp2.x; - const int zt = tmp2.y; - const int jt = blockIdx.y; - const int it = blockIdx.x; + int zt = tmp2.y; + const int jt = blockIdx.x; + const int it = blockIdx.y; // Defaults for regular matrix multiplication: - int col_low = 0; - int col_high = ncols_dst; - int col_diff = ncols_dst; + int col_low = jt*J; + int col_high = min(col_low + J, ncols_dst); int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; - int offset_y_scale; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; - } else { - GGML_UNUSED(offset_y_scale); - } if (ids_dst) { - col_low = expert_bounds[zt + 0]; - col_high = expert_bounds[zt + 1]; - col_diff = col_high - col_low; + const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt]; + zt = tsj.expert; + col_low = tsj.col_low; + col_high = tsj.col_high; - offset_y = 0; - offset_dst = 0; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = 0; - } + offset_y = 0; + offset_y_scale = 0; + offset_dst = 0; - if (jt*J >= col_diff) { + if (col_low == col_high) { return; } @@ -1042,29 +1112,30 @@ static __global__ void mul_mat_q( break; } - ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + ids_dst_shared[j] = ids_dst[col_low + j]; } __syncthreads(); } - offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int)); - offset_dst += it*I; - const float * y_scale_tile = nullptr; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale += col_low + jt*J; - y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; - } + offset_y += col_low * (sizeof(block_q8_1_mmq)/sizeof(int)); + offset_y_scale += col_low; + offset_dst += it*I_dst; - const int tile_x_max_i = nrows_x - it*I - 1; - const int tile_y_max_j = col_diff - jt*J - 1; + const float * y_scale_tile = type == GGML_TYPE_NVFP4 && y_scale ? y_scale + offset_y_scale : nullptr; - const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; + const int tile_x_i_max = nrows_x - it*I_dst - 1; + + const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I_dst*stride_row_x; + + // With ids the NVFP4 weight scales are per expert. + const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale; + const float * gate_scale_tile = gate_scale && ids_dst ? gate_scale + zt : gate_scale; constexpr bool fixup = false; - mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1> - (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, - stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z); + mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, has_fusion> + (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + glu_op, glu_limit, x_scale_tile, gate_scale_tile, stride_row_x, ncols_y, stride_col_dst, + tile_x_i_max, col_high - col_low, 0, blocks_per_ne00.z); return; } @@ -1072,8 +1143,8 @@ static __global__ void mul_mat_q( constexpr int blocks_per_iter = ITER_K / qk; // kbc == k block continuous, current index in continuous ijk space. - int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; - int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; + int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter; kbc_stop -= fastmodulo(kbc_stop, blocks_per_ne00) % blocks_per_iter; @@ -1083,41 +1154,34 @@ static __global__ void mul_mat_q( int kb0_stop = min(blocks_per_ne00.z, uint32_t(kb0_start + kbc_stop - kbc)); while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) { int tmp = fastdiv(kbc, blocks_per_ne00); - uint2 tmp2 = fast_div_modulo(tmp, ntx); + uint2 tmp2 = fast_div_modulo(tmp, ntj); const int jt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nchannels_y); - const int zt = tmp2.y; + int zt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nsamples_y); const int wt = tmp2.y; const int it = tmp2.x; // Defaults for regular matrix multiplication: - int col_low = 0; - int col_high = ncols_dst; - int col_diff = ncols_dst; + int col_low = jt*J; + int col_high = min(col_low + J, ncols_dst); int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; - int offset_y_scale; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; - } else { - GGML_UNUSED(offset_y_scale); - } if (ids_dst) { - col_low = expert_bounds[zt + 0]; - col_high = expert_bounds[zt + 1]; - col_diff = col_high - col_low; + const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt]; + zt = tsj.expert; + col_low = tsj.col_low; + col_high = tsj.col_high; - offset_y = 0; - offset_dst = 0; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = 0; - } + offset_y = 0; + offset_y_scale = 0; + offset_dst = 0; - if (jt*J >= col_diff) { + if (col_low == col_high) { kbc += blocks_per_ne00.z; kbc -= fastmodulo(kbc, blocks_per_ne00); @@ -1136,29 +1200,28 @@ static __global__ void mul_mat_q( break; } - ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + ids_dst_shared[j] = ids_dst[col_low + j]; } __syncthreads(); } - offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); - offset_dst += it*I; - const float * y_scale_tile = nullptr; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale += col_low + jt * J; - y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; - } + offset_y += col_low * (sizeof(block_q8_1_mmq) / sizeof(int)); + offset_y_scale += col_low; + offset_dst += it*I; - const int tile_x_max_i = nrows_x - it*I - 1; - const int tile_y_max_j = col_diff - jt*J - 1; + const float * y_scale_tile = type == GGML_TYPE_NVFP4 && y_scale ? y_scale + offset_y_scale : nullptr; + + const int tile_x_i_max = nrows_x - it*I - 1; const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; + const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale; + constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. - mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1> - (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, - stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); + mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, false> + (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + glu_op, glu_limit, x_scale_tile, gate_scale, stride_row_x, ncols_y, stride_col_dst, + tile_x_i_max, col_high - col_low, kb0_start, kb0_stop); kbc += blocks_per_ne00.z; kbc -= fastmodulo(kbc, blocks_per_ne00); @@ -1172,41 +1235,34 @@ static __global__ void mul_mat_q( } int tmp = fastdiv(kbc, blocks_per_ne00); - uint2 tmp2 = fast_div_modulo(tmp, ntx); + uint2 tmp2 = fast_div_modulo(tmp, ntj); const int jt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nchannels_y); - const int zt = tmp2.y; + int zt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nsamples_y); const int wt = tmp2.y; const int it = tmp2.x; // Defaults for regular matrix multiplication: - int col_low = 0; - int col_high = ncols_dst; - int col_diff = ncols_dst; + int col_low = jt*J; + int col_high = min(col_low + J, ncols_dst); int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; - int offset_y_scale; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; - } else { - GGML_UNUSED(offset_y_scale); - } if (ids_dst) { - col_low = expert_bounds[zt + 0]; - col_high = expert_bounds[zt + 1]; - col_diff = col_high - col_low; + const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt]; + zt = tsj.expert; + col_low = tsj.col_low; + col_high = tsj.col_high; - offset_y = 0; - offset_dst = 0; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = 0; - } + offset_y = 0; + offset_y_scale = 0; + offset_dst = 0; - if (jt*J >= col_diff) { + if (col_low == col_high) { return; } @@ -1225,33 +1281,32 @@ static __global__ void mul_mat_q( __syncthreads(); } - offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); - offset_dst += it*I; - const float * y_scale_tile = nullptr; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale += col_low + jt * J; - y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; - } + offset_y += col_low * (sizeof(block_q8_1_mmq) / sizeof(int)); + offset_y_scale += col_low; + offset_dst += it*I; - const int tile_x_max_i = nrows_x - it*I - 1; - const int tile_y_max_j = col_diff - jt*J - 1; + const float * y_scale_tile = type == GGML_TYPE_NVFP4 && y_scale ? y_scale + offset_y_scale : nullptr; + + const int tile_x_i_max = nrows_x - it*I - 1; const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; + const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale; + constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. - mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1> - (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, - stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); + mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, false> + (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + glu_op, glu_limit, x_scale_tile, gate_scale, stride_row_x, ncols_y, stride_col_dst, + tile_x_i_max, col_high - col_low, kb0_start, kb0_stop); } -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1)/2, 1) static __global__ void mul_mat_q_stream_k_fixup( - const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, + const int32_t * __restrict__ ids_dst, const ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, float * __restrict__ dst, float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y, - const int stride_sample_dst, const uint3 ntx) { + const int stride_sample_dst, const uint3 ntj) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / 2) / warp_size; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); @@ -1262,13 +1317,13 @@ static __global__ void mul_mat_q_stream_k_fixup( float sum[J / nwarps] = {0.0f}; const int i = blockIdx.y*warp_size + threadIdx.x; - const int nty = (nrows_x + I - 1) / I; + const int nti = (nrows_x + I - 1) / I; const int bidx0 = blockIdx.x; // kbc == k block continuous, current index in continuous ijk space. - int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; - int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; + int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; kbc0 -= fastmodulo(kbc0, blocks_per_ne00) % blocks_per_iter; kbc0_stop -= fastmodulo(kbc0_stop, blocks_per_ne00) % blocks_per_iter; @@ -1287,7 +1342,7 @@ static __global__ void mul_mat_q_stream_k_fixup( int bidx = bidx0 - 1; int kbc_stop = kbc0; while(true) { - int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter; if (kbc == kbc_stop) { // Did not have any data. @@ -1319,11 +1374,11 @@ static __global__ void mul_mat_q_stream_k_fixup( } int tmp = fastdiv(kbc0, blocks_per_ne00); - uint2 tmp2 = fast_div_modulo(tmp, ntx); + uint2 tmp2 = fast_div_modulo(tmp, ntj); const int jt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nchannels_y); - const int zt = tmp2.y; + int zt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nsamples_y); const int wt = tmp2.y; @@ -1334,7 +1389,7 @@ static __global__ void mul_mat_q_stream_k_fixup( dst += offset_dst; const int i_max = nrows_x - it*I - 1; - const int j_max = ncols_dst - jt*J - 1; + const int j_sup = ncols_dst - jt*J; if (fallback && i > i_max) { return; } @@ -1343,7 +1398,7 @@ static __global__ void mul_mat_q_stream_k_fixup( for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; - if (j > j_max) { + if (j >= j_sup) { return; } @@ -1353,12 +1408,18 @@ static __global__ void mul_mat_q_stream_k_fixup( } __shared__ int ids_dst_shared[J]; - const int col_low = expert_bounds[zt + 0]; - const int col_high = expert_bounds[zt + 1]; - const int col_diff = col_high - col_low; + const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt]; + zt = tsj.expert; + const int col_low = tsj.col_low; + const int col_high = tsj.col_high; + const int j_sup = col_high - col_low; + + if (j_sup <= 0) { + return; + } for (int j = threadIdx.y*warp_size + threadIdx.x; j < J; j += nwarps*warp_size) { - ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + ids_dst_shared[j] = ids_dst[col_low + j]; } __syncthreads(); @@ -1366,7 +1427,6 @@ static __global__ void mul_mat_q_stream_k_fixup( dst += offset_dst; const int i_max = nrows_x - it*I - 1; - const int j_max = col_diff - jt*J - 1; if (fallback && i > i_max) { return; } @@ -1375,7 +1435,7 @@ static __global__ void mul_mat_q_stream_k_fixup( for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; - if (j > j_max) { + if (j >= j_sup) { return; } @@ -1384,13 +1444,14 @@ static __global__ void mul_mat_q_stream_k_fixup( } struct mmq_args { - const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst; - const float * y_scale; + const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; + const ggml_cuda_mm_id_tile_spec * tile_spec; float * dst; const float * y_scale; int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst; int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst; int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst; - int64_t ncols_max; - int64_t ncols_opt; // value to optimize the tile size against, launch grid still uses ncols_max + int64_t ntj; + int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance. + ggml_cuda_mm_fusion_args_device fusion = {}; // gate != nullptr selects the fused gate/up/GLU path }; static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) { @@ -1400,7 +1461,7 @@ static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const i return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int)); } -template <ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion> static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { const int id = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[id].cc; @@ -1414,45 +1475,52 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const dim3 block_dims(warp_size, nwarps, 1); - CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1>), nbytes_shared); - CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1>), nbytes_shared); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, /*fallback =*/ false, prec_src1, has_fusion>), nbytes_shared); + if constexpr (!has_fusion) { + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, /*fallback =*/ true, prec_src1, false>), nbytes_shared); + } - const int nty = (args.nrows_x + config.I - 1) / config.I; - const int ntx = (args.ncols_max + config.J - 1) / config.J; - const int ntzw = args.nchannels_y * args.nsamples_y; - const dim3 block_nums_xy_tiling(nty, ntx, ntzw); + const int I_dst = has_fusion ? config.I/2 : config.I; // Rows of dst per tile, see mul_mat_q. + const int nti = (args.nrows_x + I_dst - 1) / I_dst; + const int ntzw = args.nchannels_y * args.nsamples_y; + // Column tiles innermost so that the concurrently running blocks share the same tile of x and read it from L2. + const dim3 block_nums_xy_tiling(args.ntj, nti, ntzw); - GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0); - GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0); - const int channel_ratio = args.nchannels_y / args.nchannels_x; - const int sample_ratio = args.nsamples_y / args.nsamples_x; + GGML_ASSERT(args.ids_dst || args.nchannels_y % args.nchannels_x == 0); + GGML_ASSERT( args.nsamples_y % args.nsamples_x == 0); + const int channel_ratio = args.ids_dst ? 1 : args.nchannels_y / args.nchannels_x; + const int sample_ratio = args.nsamples_y / args.nsamples_x; const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk); - const uint3 ntx_fd = init_fastdiv_values(ntx); + const uint3 ntj_fd = init_fastdiv_values(args.ntj); const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y); const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y); const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); - if (!config.stream_k) { - mul_mat_q<type, J, fallback, prec_src1><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale, + if (has_fusion || !config.stream_k) { + GGML_ASSERT(nti <= 65535); // gridDim.y limit + mul_mat_q<type, J, fallback, prec_src1, has_fusion><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>> + (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, nullptr, + args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - ntx_fd); + ntj_fd); return; } // For the stream-k kernel it is possible to run it with tiling by setting the number of CUDA blocks equal to the number of tiles. // This is worthwhile if the efficiency of tiling is high and skipping the fixup kernel is more important. - const int ntiles_dst = ntx * nty * ntzw; + const int ntiles_dst = args.ntj * nti * ntzw; const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm; const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves); - const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1); GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow. + const int nwork = ntiles_dst * ((args.ncols_x + config.K_vram - 1) / config.K_vram); // Maximum number of CUDA blocks that can do useful work. + const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : std::min(nsm, nwork), 1, 1); + const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0; ggml_cuda_pool & pool = ctx.pool(id); @@ -1464,12 +1532,13 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1); const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z); - mul_mat_q<type, J, fallback, prec_src1><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale, + mul_mat_q<type, J, fallback, prec_src1, has_fusion><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>> + (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, tmp_fixup.ptr, + args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - ntx_fd); + ntj_fd); if (!fixup_needed) { return; @@ -1477,107 +1546,93 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a CUDA_CHECK(cudaGetLastError()); mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>> - (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, + (args.ids_dst, args.tile_spec, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, - ntx_fd); + ntj_fd); } -template <ggml_type type, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, bool fallback, ggml_prec prec_src1, bool has_fusion> void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { - const int id = ggml_cuda_get_device(); - const int cc = ggml_cuda_info().devices[id].cc; - const size_t smpbo = ggml_cuda_info().devices[id].smpbo; - - int J_best = 0; - int ntiles_J_best = INT_MAX; - - for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) { - const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1); - if (config.type == GGML_TYPE_COUNT) { - continue; - } - - if (mmq_get_nbytes_shared(config, cc) > smpbo) { - continue; - } - - const int ntiles_x = (args.ncols_opt + config.J - 1) / config.J; - - if (ntiles_x < ntiles_J_best) { - J_best = J; - ntiles_J_best = ntiles_x; - } - } - - switch (J_best) { + switch (args.J_best) { case 8: - launch_mul_mat_q<type, 8, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 8, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 16: - launch_mul_mat_q<type, 16, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 16, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 24: - launch_mul_mat_q<type, 24, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 24, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 32: - launch_mul_mat_q<type, 32, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 32, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 40: - launch_mul_mat_q<type, 40, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 40, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 48: - launch_mul_mat_q<type, 48, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 48, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 56: - launch_mul_mat_q<type, 56, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 56, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 64: - launch_mul_mat_q<type, 64, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 64, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 72: - launch_mul_mat_q<type, 72, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 72, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 80: - launch_mul_mat_q<type, 80, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 80, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 88: - launch_mul_mat_q<type, 88, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 88, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 96: - launch_mul_mat_q<type, 96, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 96, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 104: - launch_mul_mat_q<type, 104, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 104, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 112: - launch_mul_mat_q<type, 112, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 112, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 120: - launch_mul_mat_q<type, 120, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 120, fallback, prec_src1, has_fusion>(ctx, args, stream); break; case 128: - launch_mul_mat_q<type, 128, fallback, prec_src1>(ctx, args, stream); + launch_mul_mat_q<type, 128, fallback, prec_src1, has_fusion>(ctx, args, stream); break; default: - fprintf(stderr, "J_best=%d\n", J_best); + fprintf(stderr, "J_best=%d\n", args.J_best); GGML_ABORT("fatal error"); - break; } } -template <ggml_type type, ggml_prec prec_src1 = GGML_PREC_Q8> +template <ggml_type type, bool fallback, ggml_prec prec_src1> +static void mul_mat_q_switch_fusion(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + if constexpr (!fallback && ggml_cuda_mmq_fusion_supported(type)) { + if (args.fusion.gate) { + mul_mat_q_switch_J<type, fallback, prec_src1, true>(ctx, args, stream); + return; + } + } + GGML_ASSERT(!args.fusion.gate); + mul_mat_q_switch_J<type, fallback, prec_src1, false>(ctx, args, stream); +} + +template <ggml_type type, ggml_prec prec_src1> void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { - if (args.nrows_x % 128 == 0) { - constexpr bool fallback = false; - mul_mat_q_switch_J<type, fallback, prec_src1>(ctx, args, stream); - } else { + if (ggml_cuda_mmq_needs_fallback(args.nrows_x)) { constexpr bool fallback = true; - mul_mat_q_switch_J<type, fallback, prec_src1>(ctx, args, stream); + mul_mat_q_switch_fusion<type, fallback, prec_src1>(ctx, args, stream); + } else { + constexpr bool fallback = false; + mul_mat_q_switch_fusion<type, fallback, prec_src1>(ctx, args, stream); } } #define DECL_MMQ_CASE(type) \ - template void mul_mat_q_case<type>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ + template void mul_mat_q_case<type, GGML_PREC_Q8>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ // FP4 variant: uses native FP4 MMA instead of keeping src1 at Q8_1. #define DECL_MMQ_CASE_W4A4(type) \ @@ -1614,6 +1669,10 @@ extern DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4); // ------------------------------------------------------------------------------------------------------------------------- void ggml_cuda_mul_mat_q( - ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst); + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, + const ggml_cuda_mm_fusion_args_host * fusion = nullptr); bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); + +// prec_requested: the src1 precision requested by the fused MUL_MATs. +bool ggml_cuda_mmq_fusion_is_efficient(ggml_type type, ggml_prec prec_requested, int64_t nrows_x, int64_t ncols, int64_t nchannels, int device); diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu index d0a572e15..154a621fa 100644 --- a/ggml/src/ggml-cuda/mmvf.cu +++ b/ggml/src/ggml-cuda/mmvf.cu @@ -5,6 +5,14 @@ #include "mmf.cuh" #include "convert.cuh" +static __device__ __forceinline__ float2 mmvf_load_fp8x2(const ggml_fp8_e4m3_t * x, const int col2) { +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + return ggml_cuda_cast<float2>(reinterpret_cast<const __nv_fp8x2_e4m3 *>(x)[col2]); +#else + return make_float2(ggml_cuda_cast<float>(x[2*col2]), ggml_cuda_cast<float>(x[2*col2 + 1])); +#endif +} + template <typename T, typename type_acc, int ncols_dst, int block_size, bool has_fusion = false, bool is_multi_token_id = false> static __global__ void mul_mat_vec_f( const T * x_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, @@ -56,16 +64,24 @@ static __global__ void mul_mat_vec_f( bool use_gate = false; bool use_bias = false; bool use_gate_bias = false; + bool use_scale = false; + bool use_gate_scale = false; ggml_glu_op glu_op = ggml_glu_op::GGML_GLU_OP_SWIGLU; float glu_limit = 0.0f; const T * gate_x = nullptr; const float * x_bias = nullptr; const float * gate_bias = nullptr; + const float * x_scale = nullptr; + const float * gate_scale = nullptr; if constexpr (has_fusion) { use_gate = fusion.gate != nullptr; use_bias = fusion.x_bias != nullptr; use_gate_bias = fusion.gate_bias != nullptr; + if constexpr (std::is_same_v<T, ggml_fp8_e4m3_t>) { + use_scale = fusion.x_scale != nullptr; + use_gate_scale = fusion.gate_scale != nullptr && use_gate; + } glu_op = fusion.glu_op; glu_limit = fusion.glu_limit; @@ -81,6 +97,12 @@ static __global__ void mul_mat_vec_f( } else { use_gate_bias = false; } + if (use_scale) { + x_scale = static_cast<const float *>(fusion.x_scale); + } + if (use_gate_scale) { + gate_scale = static_cast<const float *>(fusion.gate_scale); + } } if (use_gate) { @@ -97,8 +119,6 @@ static __global__ void mul_mat_vec_f( } } - const float2 * y2 = (const float2 *) y; - extern __shared__ char data_mmv[]; float * buf_iw = (float *) data_mmv; [[maybe_unused]] float * buf_iw_gate = nullptr; @@ -128,6 +148,7 @@ static __global__ void mul_mat_vec_f( } if constexpr (std::is_same_v<T, float>) { + const float2 * y2 = (const float2 *) y; const float2 * x2 = (const float2 *) x; [[maybe_unused]] const float2 * gate_x2 = nullptr; if constexpr (has_fusion) { @@ -160,6 +181,7 @@ static __global__ void mul_mat_vec_f( } } } else if constexpr (std::is_same_v<T, half>) { + const float2 * y2 = (const float2 *) y; const half2 * x2 = (const half2 *) x; [[maybe_unused]] const half2 * gate_x2 = nullptr; if constexpr (has_fusion) { @@ -235,6 +257,7 @@ static __global__ void mul_mat_vec_f( #endif // FP16_AVAILABLE } } else if constexpr (std::is_same_v<T, nv_bfloat16>) { + const float2 * y2 = (const float2 *) y; //TODO: add support for ggml_cuda_mad for hip_bfloat162 #if defined(GGML_USE_HIP) const int * x2 = (const int *) x; @@ -301,6 +324,28 @@ static __global__ void mul_mat_vec_f( } } #endif + } else if constexpr (std::is_same_v<T, ggml_fp8_e4m3_t>) { + const float2 * y2 = (const float2 *) y; + for (int col2 = tid; col2 < ncols2; col2 += block_size) { + const float2 tmpx = mmvf_load_fp8x2(x, col2); + float2 tmpx_gate = {}; + if constexpr (has_fusion) { + if (use_gate) { + tmpx_gate = mmvf_load_fp8x2(gate_x, col2); + } + } +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const float2 tmpy = y2[j*stride_col_y2 + col2]; + ggml_cuda_mad(sumf[j], tmpx, tmpy); + + if constexpr (has_fusion) { + if (use_gate) { + ggml_cuda_mad(sumf_gate[j], tmpx_gate, tmpy); + } + } + } + } } else { static_assert(std::is_same_v<T, void>, "unsupported type"); } @@ -348,12 +393,18 @@ static __global__ void mul_mat_vec_f( float value = sumf[tid]; if constexpr (has_fusion) { + if (use_scale) { + value *= x_scale[ids ? channel_x : 0]; + } if (use_bias) { value += x_bias[tid*stride_col_dst + row]; } if (use_gate) { float gate_value = sumf_gate[tid]; + if (use_gate_scale) { + gate_value *= gate_scale[ids ? channel_x : 0]; + } if (use_gate_bias) { gate_value += gate_bias[tid*stride_col_dst + row]; } @@ -380,7 +431,11 @@ static __global__ void mul_mat_vec_f( dst[tid*stride_col_dst + row] = value; if constexpr (!has_fusion) { - GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, glu_limit, gate_x, x_bias, gate_bias, sumf_gate); + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, glu_op, glu_limit, gate_x, x_bias, gate_bias, + x_scale, gate_scale, sumf_gate); + } + if constexpr (!std::is_same_v<T, ggml_fp8_e4m3_t>) { + GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale); } } @@ -395,7 +450,8 @@ static void mul_mat_vec_f_switch_fusion( const ggml_cuda_kernel_launch_params launch_params = {block_nums, block_dims, nbytes_shared, stream}; - const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; + const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr || + fusion.x_scale != nullptr || fusion.gate_scale != nullptr; if constexpr (ncols_dst == 1) { if (has_fusion) { ggml_cuda_kernel_launch(mul_mat_vec_f<T, type_acc, ncols_dst, block_size, true, is_multi_token_id>, launch_params, @@ -504,9 +560,8 @@ void launch_mul_mat_vec_f_cuda( channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; - default: { + default: GGML_ABORT("fatal error"); - } break; } } @@ -603,7 +658,6 @@ static void mul_mat_vec_f_cuda_switch_ncols_dst( break; default: GGML_ABORT("fatal error"); - break; } } @@ -618,7 +672,7 @@ static void mul_mat_vec_f_cuda( const int64_t ids_stride, enum ggml_prec prec, cudaStream_t stream) { if constexpr(std::is_same_v<T, half>) { - if (prec == GGML_PREC_DEFAULT) { + if (prec == GGML_PREC_UNDEFINED) { mul_mat_vec_f_cuda_switch_ncols_dst<T, half> (x, y, ids, fusion, dst, ncols, nrows, ncols_dst, stride_row, stride_col_y, stride_col_dst, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, @@ -651,9 +705,11 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor GGML_ASSERT( nb10 == ts_src1); GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type)); GGML_ASSERT( nb0 == ts_dst); + GGML_ASSERT(src0->type != GGML_TYPE_F8_E4M3 || (uintptr_t) src0->data % alignof(uint16_t) == 0); const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32; + // the fused GLU path gets the GLU node as dst, which carries no matmul precision, so it accumulates in F32 + const enum ggml_prec prec = fast_fp16_available(cc) && !(fusion && fusion->gate) ? ggml_prec_get_acc(dst) : GGML_PREC_F32; const float * src1_d = (const float *) src1->data; const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr; @@ -664,6 +720,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor if (fusion) { GGML_ASSERT( !ids || dst->ne[2] == 1); GGML_ASSERT( ids || dst->ne[1] == 1); + GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_F8_E4M3); if (fusion->x_bias) { GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32); GGML_ASSERT(fusion->x_bias->ne[0] == dst->ne[0]); @@ -672,6 +729,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor } if (fusion->gate) { GGML_ASSERT(fusion->gate->type == src0->type && ggml_are_same_stride(fusion->gate, src0)); + GGML_ASSERT(src0->type != GGML_TYPE_F8_E4M3 || (uintptr_t) fusion->gate->data % alignof(uint16_t) == 0); fusion_local.gate = fusion->gate->data; } if (fusion->gate_bias) { @@ -680,6 +738,18 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]); fusion_local.gate_bias = fusion->gate_bias->data; } + if (fusion->x_scale) { + GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(fusion->x_scale)); + GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1)); + fusion_local.x_scale = fusion->x_scale->data; + } + if (fusion->gate_scale) { + GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale)); + GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1)); + fusion_local.gate_scale = fusion->gate_scale->data; + } fusion_local.glu_op = fusion->glu_op; fusion_local.glu_limit = fusion->glu_limit; } @@ -724,6 +794,12 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream()); } break; + case GGML_TYPE_F8_E4M3: { + const ggml_fp8_e4m3_t * src0_d = (const ggml_fp8_e4m3_t *) src0->data; + mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, + ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, + ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream()); + } break; default: GGML_ABORT("unsupported type: %s", ggml_type_name(src0->type)); } @@ -745,7 +821,7 @@ void ggml_cuda_op_mul_mat_vec_f( const int id = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[id].cc; - const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32; + const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec_get_acc(dst) : GGML_PREC_F32; // ggml_cuda_op provides single, contiguous matrices const int64_t stride_row = ne00; @@ -790,7 +866,11 @@ void ggml_cuda_op_mul_mat_vec_f( GGML_UNUSED_VARS(ctx, src1, dst, src1_ddq_i, src1_ncols, src1_padded_row_size); } -bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) { +bool ggml_cuda_should_use_mmvf(const ggml_tensor * src0, int cc, int warp_size, int64_t ne11) { + const ggml_type type = src0->type; + const int64_t * src0_ne = src0->ne; + const size_t * src0_nb = src0->nb; + if (src0_ne[0] % 2 != 0) { return false; } @@ -800,11 +880,8 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const return false; } - // Pointers not aligned to the size of half2/nv_bfloat162/float2 would result in a crash: - for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { - if (src0_nb[i] % (2*ts) != 0) { - return false; - } + if (!ggml_cuda_is_aligned(src0, 2*ts)) { + return false; } switch (type) { @@ -846,7 +923,7 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const return ne11 <= MMVF_MAX_BATCH_SIZE; } if (fp16_mma_hardware_available(cc)) { - if (GGML_CUDA_CC_IS_RDNA3(cc)) { + if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) { return ne11 <= 3; } if (GGML_CUDA_CC_IS_RDNA4(cc)) { @@ -864,6 +941,18 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) { return ne11 <= MMVF_MAX_BATCH_SIZE; } + // MMF tiles 32 rows per block, so a matrix of few rows leaves most SMs idle while MMVF runs a block + // per row: with a long enough K, MMVF wins for the small batches of speculative decoding as long as + // its work (rows x batch) stays small relative to the SM count, with a budget that grows with K + if (src0_small && src0_ne[0] >= 1024) { + const bool blackwell = cc >= GGML_CUDA_CC_BLACKWELL; + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + const int64_t max_ncols = blackwell ? MMVF_MAX_BATCH_SIZE : 5; + const int64_t max_k = blackwell ? 6144 : 4096; + if (ne11 <= max_ncols && src0_ne[1]*ne11*256 <= nsm*std::min(src0_ne[0], max_k)) { + return true; + } + } if (ampere_mma_available(cc)) { return src0_small && ne11 == 1; } @@ -884,6 +973,8 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const return ne11 <= MMVF_MAX_BATCH_SIZE; } return ne11 <= MMVF_MAX_BATCH_SIZE; + case GGML_TYPE_F8_E4M3: + return ne11 <= MMVF_MAX_BATCH_SIZE; default: return false; } diff --git a/ggml/src/ggml-cuda/mmvf.cuh b/ggml/src/ggml-cuda/mmvf.cuh index 350b5cc06..5a2477b07 100644 --- a/ggml/src/ggml-cuda/mmvf.cuh +++ b/ggml/src/ggml-cuda/mmvf.cuh @@ -11,4 +11,4 @@ void ggml_cuda_op_mul_mat_vec_f( const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream); -bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11); +bool ggml_cuda_should_use_mmvf(const ggml_tensor * src0, int cc, int warp_size, int64_t ne11); diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 251577d2f..c6d895db4 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -29,10 +29,6 @@ static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) { return false; } } - -static __device__ __forceinline__ void mmvq_prefetch_l2(const void * p) { - asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); -} #endif typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs); @@ -108,7 +104,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { return MMVQ_PARAMETERS_RDNA4; #elif defined(RDNA3_0) return MMVQ_PARAMETERS_RDNA3_0; -#elif defined(RDNA2) || defined(RDNA3_5) +#elif defined(RDNA2) || defined(RDNA3_5) || defined(RDNA4_M) return MMVQ_PARAMETERS_RDNA2; #elif defined(GCN) || defined(CDNA) return MMVQ_PARAMETERS_GCN; @@ -128,7 +124,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) { if (GGML_CUDA_CC_IS_RDNA3_0(cc)) { return MMVQ_PARAMETERS_RDNA3_0; } - if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) { + if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) { return MMVQ_PARAMETERS_RDNA2; } if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) { @@ -283,9 +279,10 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type // Host function: returns the max batch size for the current arch+type at runtime. int get_mmvq_mmid_max_batch(ggml_type type, int cc) { - // NVIDIA: Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID. + // NVIDIA: P100, Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID. if (GGML_CUDA_CC_IS_NVIDIA(cc)) { - if (cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) { + cc = ggml_cuda_highest_compiled_arch(cc); + if (cc == GGML_CUDA_CC_PASCAL || cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) { return MMVQ_MAX_BATCH_SIZE; } if (cc >= GGML_CUDA_CC_TURING) { @@ -299,7 +296,7 @@ int get_mmvq_mmid_max_batch(ggml_type type, int cc) { if (GGML_CUDA_CC_IS_RDNA4(cc)) { return get_mmvq_mmid_max_batch_rdna4(type); } - if (GGML_CUDA_CC_IS_RDNA3(cc)) { + if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) { return get_mmvq_mmid_max_batch_rdna3(type); } if (GGML_CUDA_CC_IS_RDNA1(cc) || GGML_CUDA_CC_IS_RDNA2(cc)) { @@ -432,7 +429,7 @@ template <ggml_type type> static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() { #if defined(RDNA4) return get_mmvq_mmid_max_batch_rdna4(type); -#elif defined(RDNA3) +#elif defined(RDNA3) || defined(RDNA4_M) return get_mmvq_mmid_max_batch_rdna3(type); #elif defined(RDNA2) || defined(RDNA1) return get_mmvq_mmid_max_batch_rdna1_rdna2(type); @@ -440,7 +437,7 @@ static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() { return get_mmvq_mmid_max_batch_cdna(type); #elif defined(GCN) return get_mmvq_mmid_max_batch_gcn(type); -#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE) +#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_PASCAL || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE) return MMVQ_MAX_BATCH_SIZE; #elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING return get_mmvq_mmid_max_batch_turing_plus(type); @@ -468,10 +465,12 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d } else if (table_id == MMVQ_PARAMETERS_GCN) { switch (ncols_dst) { case 1: + return 2; case 2: case 3: + return type == GGML_TYPE_Q3_K ? 1 : 2; case 4: - return 2; + return type == GGML_TYPE_Q6_K? 1 : 2; case 5: case 6: case 7: @@ -576,11 +575,31 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d return 1; } -static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) { +// Low bit types load few weight bytes per thread, a second row per block shares the q8_1 loads and adds memory parallelism. +static constexpr __host__ __device__ bool mmvq_two_rows_ncols1(ggml_type type) { + switch (type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + return true; + default: + return false; + } +} + +static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) { if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) { switch (ncols_dst) { case 1: - return small_k ? nwarps : 1; + if (small_k) { + return nwarps; + } + if (table_id == MMVQ_PARAMETERS_GB10) { + return type == GGML_TYPE_IQ2_S || type == GGML_TYPE_IQ3_S ? 2 : 1; + } + return table_id == MMVQ_PARAMETERS_GENERIC && mmvq_two_rows_ncols1(type) ? 2 : 1; case 2: case 3: case 4: @@ -615,7 +634,7 @@ static __global__ void mul_mat_vec_q( constexpr int vdr = get_vdr_mmvq(type); constexpr mmvq_parameter_table_id table_id = get_device_table_id(); constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters); - constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps); + constexpr int rows_per_cuda_block = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); @@ -727,16 +746,17 @@ static __global__ void mul_mat_vec_q( #if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK // start the next iterations' weight loads early if constexpr (mmvq_should_prefetch(type)) { - constexpr int pf_dist = 2; // loop iterations, not blocks + // loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1 + constexpr int pf_dist = type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q6_K ? 1 : 2; const int kbx_pf = kbx + pf_dist*blocks_per_iter; if (kbx_pf < blocks_per_row_x) { #pragma unroll for (int i = 0; i < rows_per_cuda_block; ++i) { const size_t off = (size_t)(kbx_offset + i*stride_row_x + kbx_pf) * ggml_cuda_type_traits<type>::bs; - mmvq_prefetch_l2((const char *) vx + off); + ggml_cuda_prefetch_l2((const char *) vx + off); if constexpr (has_fusion) { if (use_gate) { - mmvq_prefetch_l2((const char *) vgate + off); + ggml_cuda_prefetch_l2((const char *) vgate + off); } } } @@ -1013,7 +1033,7 @@ static std::pair<dim3, dim3> calc_launch_params( const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens, const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false, const bool halve_iters = false) { const int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters); - const int rpb = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps); + const int rpb = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps); const int64_t nblocks = (nrows_x + rpb - 1) / rpb; const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens); const dim3 block_dims(warp_size, nwarps, 1); @@ -1275,7 +1295,6 @@ static void mul_mat_vec_q_switch_ncols_dst( } break; default: GGML_ABORT("fatal error"); - break; } } static void mul_mat_vec_q_switch_type( @@ -1427,7 +1446,6 @@ static void mul_mat_vec_q_switch_type( break; default: GGML_ABORT("fatal error"); - break; } } @@ -1533,7 +1551,7 @@ void ggml_cuda_mul_mat_vec_q( const int64_t s11 = src1->nb[1] / ts_src1; const int64_t s12 = src1->nb[2] / ts_src1; const int64_t s13 = src1->nb[3] / ts_src1; - quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + quantize_row_q8_1_cuda(src1_d, src1_q8_1.get(), ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); } const int64_t s01 = src0->nb[1] / ts_src0; diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index 5543307be..3f0ea6f2a 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -3,38 +3,46 @@ template <int block_size> static __global__ void norm_f32( - const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, - const int64_t stride_sample, const float eps) { - const int nrows = gridDim.x; - const int nchannels = gridDim.y; + const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) { + const int nrows = gridDim.x; + const int row = blockIdx.x; + const int tid = threadIdx.x; - const int row = blockIdx.x; - const int channel = blockIdx.y; - const int sample = blockIdx.z; - const int tid = threadIdx.x; + extern __shared__ float2 s_sum2[]; - x += sample*stride_sample + channel*stride_channel + row*stride_row; - dst += ((sample*nchannels + channel)*nrows + row)*ncols; + ggml_cuda_pdl_sync(); - float2 mean_var = make_float2(0.0f, 0.0f); + // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples + for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + const float * xc = x + sample*stride_sample + channel*stride_channel + row*stride_row; + float * dstc = dst + ((sample*nchannels + channel)*nrows + row)*ncols; - ggml_cuda_pdl_sync(); - for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; - mean_var.x += xi; - mean_var.y += xi * xi; - } + float2 mean_var = make_float2(0.0f, 0.0f); - // sum up partial sums - extern __shared__ float2 s_sum2[]; - mean_var = block_reduce<block_reduce_method::SUM, block_size>(mean_var, s_sum2); + for (int col = tid; col < ncols; col += block_size) { + const float xi = xc[col]; + mean_var.x += xi; + mean_var.y += xi * xi; + } - const float mean = mean_var.x / ncols; - const float var = mean_var.y / ncols - mean * mean; - const float inv_std = rsqrtf(var + eps); + // sum up partial sums + mean_var = block_reduce<block_reduce_method::SUM, block_size>(mean_var, s_sum2); - for (int col = tid; col < ncols; col += block_size) { - dst[col] = (x[col] - mean) * inv_std; + const float mean = mean_var.x / ncols; + const float var = mean_var.y / ncols - mean * mean; + const float inv_std = rsqrtf(var + eps); + + for (int col = tid; col < ncols; col += block_size) { + dstc[col] = (xc[col] - mean) * inv_std; + } + + if constexpr (block_size > WARP_SIZE) { + // sync is needed as we reuse s_sum2 across block_reduce invocations, see #26385 + __syncthreads(); + } + } } } @@ -73,10 +81,12 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr } } -template <int block_size, bool do_multiply = false, bool do_add = false, bool do_scale = false> +template <int block_size, int vpt, bool do_vec, bool do_multiply = false, bool do_add = false, bool do_scale = false> static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, + const int nchannels, + const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, @@ -99,61 +109,122 @@ static __global__ void rms_norm_f32(const float * x, const uint3 add_nsamples_packed = make_uint3(0, 0, 0), const float scale_out = 1.0f) { ggml_cuda_pdl_lc(); - const int nrows = gridDim.x; - const int nchannels = gridDim.y; - - const int row = blockIdx.x; - const int channel = blockIdx.y; - const int sample = blockIdx.z; - const int tid = threadIdx.x; + const int nrows = gridDim.x; + const int row = blockIdx.x; + const int tid = threadIdx.x; static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying"); static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying"); - x += sample*stride_sample + channel*stride_channel + row*stride_row; - dst += ((sample*nchannels + channel)*nrows + row)*ncols; - - if constexpr (do_multiply) { - const uint32_t mul_row = fastmodulo(row, mul_nrows_packed); - const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed); - const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed); - mul += mul_sample * mul_stride_sample + mul_channel * mul_stride_channel + mul_row * mul_stride_row; - } - - if constexpr (do_add) { - const int add_row = fastmodulo(row, add_nrows_packed); - const int add_channel = fastmodulo(channel, add_nchannels_packed); - const int add_sample = fastmodulo(sample, add_nsamples_packed); - add += add_sample * add_stride_sample + add_channel * add_stride_channel + add_row * add_stride_row; - } - - float tmp = 0.0f; // partial sum for thread in warp - - ggml_cuda_pdl_sync(); - for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; - tmp += xi * xi; - } - - // sum up partial sums extern __shared__ float s_sum[]; - tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum); - const float mean = tmp / ncols; - const float scale = rsqrtf(mean + eps); + ggml_cuda_pdl_sync(); - for (int col = tid; col < ncols; col += block_size) { - if constexpr (do_multiply && do_add) { - const int mul_col = fastmodulo(col, mul_ncols_packed); - const int add_col = fastmodulo(col, add_ncols_packed); - dst[col] = scale * x[col] * mul[mul_col] + add[add_col]; - } else if constexpr (do_multiply) { - const int mul_col = fastmodulo(col, mul_ncols_packed); - dst[col] = scale * x[col] * mul[mul_col]; - } else if constexpr (do_scale) { - dst[col] = scale_out * (scale * x[col]); - } else { - dst[col] = scale * x[col]; + // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples + for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + const float * xc = x + sample*stride_sample + channel*stride_channel + row*stride_row; + float * dstc = dst + ((sample*nchannels + channel)*nrows + row)*ncols; + + [[maybe_unused]] const float * mulc = nullptr; + if constexpr (do_multiply) { + const uint32_t mul_row = fastmodulo(row, mul_nrows_packed); + const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed); + const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed); + mulc = mul + mul_sample * mul_stride_sample + mul_channel * mul_stride_channel + mul_row * mul_stride_row; + } + + [[maybe_unused]] const float * addc = nullptr; + if constexpr (do_add) { + const int add_row = fastmodulo(row, add_nrows_packed); + const int add_channel = fastmodulo(channel, add_nchannels_packed); + const int add_sample = fastmodulo(sample, add_nsamples_packed); + addc = add + add_sample * add_stride_sample + add_channel * add_stride_channel + add_row * add_stride_row; + } + + if constexpr (do_vec) { + const int ncols4 = ncols / 4; + const float4 * x4 = (const float4 *) xc; + const float4 * mul4 = (const float4 *) mulc; + const float4 * add4 = (const float4 *) addc; + float4 * dst4 = (float4 *) dstc; + + float4 xs[vpt]; + float tmp = 0.0f; + +#pragma unroll + for (int k = 0; k < vpt; k++) { + const int i = tid + k*block_size; + if (i < ncols4) { + xs[k] = x4[i]; + tmp += xs[k].x*xs[k].x + xs[k].y*xs[k].y + xs[k].z*xs[k].z + xs[k].w*xs[k].w; + } + } + tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum); + const float scale = rsqrtf(tmp / ncols + eps); + +#pragma unroll + for (int k = 0; k < vpt; k++) { + const int i = tid + k*block_size; + if (i < ncols4) { + float4 r = make_float4(scale*xs[k].x, scale*xs[k].y, scale*xs[k].z, scale*xs[k].w); + if constexpr (do_multiply) { + const float4 m = mul4[i]; + r.x *= m.x; + r.y *= m.y; + r.z *= m.z; + r.w *= m.w; + } + if constexpr (do_add) { + const float4 a = add4[i]; + r.x += a.x; + r.y += a.y; + r.z += a.z; + r.w += a.w; + } + if constexpr (do_scale) { + r.x *= scale_out; + r.y *= scale_out; + r.z *= scale_out; + r.w *= scale_out; + } + dst4[i] = r; + } + } + } else { + float tmp = 0.0f; // partial sum for thread in warp + + for (int col = tid; col < ncols; col += block_size) { + const float xi = xc[col]; + tmp += xi * xi; + } + + // sum up partial sums + tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum); + + const float mean = tmp / ncols; + const float scale = rsqrtf(mean + eps); + + for (int col = tid; col < ncols; col += block_size) { + if constexpr (do_multiply && do_add) { + const int mul_col = fastmodulo(col, mul_ncols_packed); + const int add_col = fastmodulo(col, add_ncols_packed); + dstc[col] = scale * xc[col] * mulc[mul_col] + addc[add_col]; + } else if constexpr (do_multiply) { + const int mul_col = fastmodulo(col, mul_ncols_packed); + dstc[col] = scale * xc[col] * mulc[mul_col]; + } else if constexpr (do_scale) { + dstc[col] = scale_out * (scale * xc[col]); + } else { + dstc[col] = scale * xc[col]; + } + } + } + + if constexpr (block_size > WARP_SIZE) { + // sync is needed as we reuse s_sum across block_reduce invocations, see #26385 + __syncthreads(); + } } } } @@ -247,50 +318,57 @@ static __global__ void rms_norm_back_f32( template <int block_size> static __global__ void l2_norm_f32( - const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, - const int64_t stride_sample, const float eps) { - const int nrows = gridDim.x; - const int nchannels = gridDim.y; + const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) { + const int nrows = gridDim.x; + const int row = blockIdx.x; + const int tid = threadIdx.x; - const int row = blockIdx.x; - const int channel = blockIdx.y; - const int sample = blockIdx.z; - const int tid = threadIdx.x; + extern __shared__ float s_sum[]; - x += sample*stride_sample + channel*stride_channel + row*stride_row; - dst += ((sample*nchannels + channel)*nrows + row)*ncols; + ggml_cuda_pdl_sync(); - float tmp = 0.0f; // partial sum for thread in warp + // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples + for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + const float * xc = x + sample*stride_sample + channel*stride_channel + row*stride_row; + float * dstc = dst + ((sample*nchannels + channel)*nrows + row)*ncols; - ggml_cuda_pdl_sync(); - for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; - tmp += xi * xi; - } + float tmp = 0.0f; // partial sum for thread in warp - // sum up partial sums - extern __shared__ float s_sum[]; - tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum); - ggml_cuda_pdl_lc(); + for (int col = tid; col < ncols; col += block_size) { + const float xi = xc[col]; + tmp += xi * xi; + } - // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html - const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + // sum up partial sums + tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum); - for (int col = tid; col < ncols; col += block_size) { - dst[col] = scale * x[col]; + // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html + const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + + for (int col = tid; col < ncols; col += block_size) { + dstc[col] = scale * xc[col]; + } + + if constexpr (block_size > WARP_SIZE) { + // sync is needed as we reuse s_sum across block_reduce invocations, see #26385 + __syncthreads(); + } + } } } static void norm_f32_cuda( const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { - const dim3 blocks_num(nrows, nchannels, nsamples); + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); if (ncols < 1024) { const dim3 block_dims(WARP_SIZE, 1, 1); - norm_f32<WARP_SIZE><<<blocks_num, block_dims, 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + norm_f32<WARP_SIZE><<<blocks_num, block_dims, 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps); } else { const dim3 block_dims(1024, 1, 1); - norm_f32<1024><<<blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float2): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + norm_f32<1024><<<blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float2): 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps); } } @@ -305,34 +383,89 @@ static void group_norm_f32_cuda( } } +#define RMS_NORM_LAUNCH(BLOCK_SIZE, VPT, VEC) \ + do { \ + const dim3 block_dims(BLOCK_SIZE, 1, 1); \ + const ggml_cuda_kernel_launch_params launch_params{ blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream }; \ + ggml_cuda_kernel_launch(rms_norm_f32<BLOCK_SIZE, VPT, VEC, do_multiply, do_add, do_scale>, launch_params, \ + x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, \ + mul, mul_stride_row, mul_stride_channel, mul_stride_sample, \ + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, \ + add, add_stride_row, add_stride_channel, add_stride_sample, \ + add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out); \ + } while (0) + +template <bool do_multiply, bool do_add, bool do_scale> +static void rms_norm_f32_dispatch( + const dim3 blocks_num, + const ggml_tensor * x_t, float * dst, const int ncols, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, + const ggml_tensor * mul_t, const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample, + const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed, + const ggml_tensor * add_t, const int64_t add_stride_row, const int64_t add_stride_channel, const int64_t add_stride_sample, + const uint3 add_ncols_packed, const uint3 add_nrows_packed, const uint3 add_nchannels_packed, const uint3 add_nsamples_packed, + cudaStream_t stream, const float scale_out = 1.0f) { + static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying"); + static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying"); + + const float * x = (const float *) x_t->data; + const float * mul = mul_t ? (const float *) mul_t->data : nullptr; + const float * add = add_t ? (const float *) add_t->data : nullptr; + + // dst is written at row * ncols, i.e. always at a multiple of ncols, so only its base needs checking here + bool vec = ncols % 4 == 0 && ncols > 0 && ((uintptr_t) dst & 15) == 0; + vec = vec && ggml_cuda_is_aligned(x_t, 16); + if constexpr (do_multiply) { + vec = vec && mul_ncols_packed.z == (uint32_t) ncols + && ggml_cuda_is_aligned(mul_t, 16); + } + if constexpr (do_add) { + vec = vec && add_ncols_packed.z == (uint32_t) ncols + && ggml_cuda_is_aligned(add_t, 16); + } + + const int n4 = ncols / 4; + if (vec) { + if (n4 <= 32) { + RMS_NORM_LAUNCH(32, 1, true); + } else if (n4 <= 128) { + RMS_NORM_LAUNCH(128, 1, true); + } else if (n4 <= 256) { + RMS_NORM_LAUNCH(256, 1, true); + } else if (n4 <= 512) { + RMS_NORM_LAUNCH(512, 1, true); + } else if (n4 <= 1024) { + RMS_NORM_LAUNCH(1024, 1, true); + } else if (n4 <= 4096) { + RMS_NORM_LAUNCH(1024, 4, true); + } else { + RMS_NORM_LAUNCH(1024, 1, false); + } + } else if (ncols < 1024) { + RMS_NORM_LAUNCH(256, 1, false); + } else { + RMS_NORM_LAUNCH(1024, 1, false); + } +} +#undef RMS_NORM_LAUNCH + template <bool do_scale = false> static void rms_norm_f32_cuda( - const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, + const ggml_tensor * x_t, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream, const float scale_out = 1.0f) { - const dim3 blocks_num(nrows, nchannels, nsamples); - if (ncols < 1024) { - const dim3 block_dims(256, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; - ggml_cuda_kernel_launch(rms_norm_f32<256, false, false, do_scale>, launch_params, - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, - // underlying cudaLaunchKernelEx does not support default params + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); + rms_norm_f32_dispatch</*do_multiply*/false, /*do_add*/false, do_scale>( + blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), - nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), scale_out); - } else { - const dim3 block_dims(1024, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; - ggml_cuda_kernel_launch(rms_norm_f32<1024, false, false, do_scale>, launch_params, x, dst, ncols, stride_row, stride_channel, stride_sample, eps, - // underlying cudaLaunchKernelEx does not support default params nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), - nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), scale_out); - } + stream, scale_out); } -static void rms_norm_mul_f32_cuda(const float * x, - const float * mul, - const float * add, - float * dst, +static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t, + const ggml_tensor * mul_t, + const ggml_tensor * add_t, + float * dst, const int ncols, const int nrows, const int nchannels, @@ -356,60 +489,34 @@ static void rms_norm_mul_f32_cuda(const float * x, const uint32_t add_nsamples, const float eps, cudaStream_t stream) { - const dim3 blocks_num(nrows, nchannels, nsamples); - if (mul == nullptr) { - rms_norm_f32_cuda(x, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream); + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); + if (mul_t == nullptr) { + rms_norm_f32_cuda(x_t, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream); return; } - if (add == nullptr) { - const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols); - const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); - const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); - const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); - if (ncols < 1024) { - const dim3 block_dims(256, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; - ggml_cuda_kernel_launch(rms_norm_f32<256, true>, launch_params, - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, - mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, - // underlying cudaLaunchKernelEx does not support default params - nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), 1.0f); - } else { - const dim3 block_dims(1024, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; - ggml_cuda_kernel_launch(rms_norm_f32<1024, true>, launch_params, - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, - mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, - // underlying cudaLaunchKernelEx does not support default params - nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), 1.0f); - } + const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols); + const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); + const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); + const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); + const uint3 zero_packed = make_uint3(0, 0, 0); + + if (add_t == nullptr) { + rms_norm_f32_dispatch</*do_multiply*/true, /*do_add*/false, /*do_scale*/false>( + blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, + mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + nullptr, 0, 0, 0, zero_packed, zero_packed, zero_packed, zero_packed, stream); } else { - const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols); - const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); - const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); - const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); - const uint3 add_ncols_packed = init_fastdiv_values(add_ncols); const uint3 add_nrows_packed = init_fastdiv_values(add_nrows); const uint3 add_nchannels_packed = init_fastdiv_values(add_nchannels); const uint3 add_nsamples_packed = init_fastdiv_values(add_nsamples); - if (ncols < 1024) { - const dim3 block_dims(256, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims,block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; - ggml_cuda_kernel_launch(rms_norm_f32<256, true, true>, launch_params, - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, - mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add, - add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed, - add_nchannels_packed, add_nsamples_packed, 1.0f); - } else { - const dim3 block_dims(1024, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; - ggml_cuda_kernel_launch(rms_norm_f32<1024, true, true>, launch_params, - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, - mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add, - add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed, - add_nchannels_packed, add_nsamples_packed, 1.0f); - } + rms_norm_f32_dispatch</*do_multiply*/true, /*do_add*/true, /*do_scale*/false>( + blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, + mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + add_t, add_stride_row, add_stride_channel, add_stride_sample, + add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream); } } @@ -426,15 +533,15 @@ static void rms_norm_back_f32_cuda(const float * grad, const float * xf, float * static void l2_norm_f32_cuda( const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { - const dim3 blocks_num(nrows, nchannels, nsamples); + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); if (ncols < 1024) { const dim3 block_dims(WARP_SIZE, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, 0, stream}; - ggml_cuda_kernel_launch(l2_norm_f32<WARP_SIZE>, launch_params, x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + ggml_cuda_kernel_launch(l2_norm_f32<WARP_SIZE>, launch_params, x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps); } else { const dim3 block_dims(1024, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; - ggml_cuda_kernel_launch(l2_norm_f32<1024>, launch_params, x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + ggml_cuda_kernel_launch(l2_norm_f32<1024>, launch_params, x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps); } } @@ -483,7 +590,6 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; - const float * src0_d = (const float *) src0->data; float * dst_d = (float *) dst->data; cudaStream_t stream = ctx.stream(); @@ -502,12 +608,11 @@ void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int64_t s02 = nb02 / ts0; const int64_t s03 = nb03 / ts0; - rms_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream); + rms_norm_f32_cuda(src0, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream); } void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * scale_tensor) { const ggml_tensor * src0 = dst->src[0]; - const float * src0_d = (const float *) src0->data; float * dst_d = (float *) scale_tensor->data; cudaStream_t stream = ctx.stream(); @@ -529,7 +634,7 @@ void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_ten const int64_t s02 = nb02 / ts0; const int64_t s03 = nb03 / ts0; - rms_norm_f32_cuda<true>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, scale); + rms_norm_f32_cuda<true>(src0, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, scale); } void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor) { @@ -538,15 +643,11 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * memcpy(&eps, dst->op_params, sizeof(float)); - const float * src0_d = (const float *) rms_norm_src->data; - const float * mul_d = nullptr; const ggml_tensor * mul_src = nullptr; if (mul_tensor->src[0] == dst) { - mul_d = (float *) mul_tensor->src[1]->data; mul_src = mul_tensor->src[1]; } else if(mul_tensor->src[1] == dst) { - mul_d = (float *) mul_tensor->src[0]->data; mul_src = mul_tensor->src[0]; } else { GGML_ASSERT(false); @@ -582,7 +683,7 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * const int mul_nchannels = mul_src->ne[2]; const int mul_nsamples = mul_src->ne[3]; - rms_norm_mul_f32_cuda(src0_d, mul_d, nullptr, dst_d, + rms_norm_mul_f32_cuda(rms_norm_src, mul_src, nullptr, dst_d, ne00, ne01, ne02, ne03, /*s00*/ s01, s02, s03, /*mul_s00*/ mul_s01, mul_s02, mul_s03, @@ -601,28 +702,21 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, memcpy(&eps, dst->op_params, sizeof(float)); - const float * src0_d = (const float *) rms_norm_src->data; - const float * mul_d = nullptr; const ggml_tensor * mul_src = nullptr; if (mul_tensor->src[0] == dst) { - mul_d = (float *) mul_tensor->src[1]->data; mul_src = mul_tensor->src[1]; } else if (mul_tensor->src[1] == dst) { - mul_d = (float *) mul_tensor->src[0]->data; mul_src = mul_tensor->src[0]; } else { GGML_ASSERT(false); } - const float * add_d = nullptr; const ggml_tensor * add_src = nullptr; if (add_tensor->src[0] == mul_tensor) { - add_d = (float *) add_tensor->src[1]->data; add_src = add_tensor->src[1]; } else if (add_tensor->src[1] == mul_tensor) { - add_d = (float *) add_tensor->src[0]->data; add_src = add_tensor->src[0]; } else { GGML_ASSERT(false); @@ -670,7 +764,7 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, const int add_nchannels = add_src->ne[2]; const int add_nsamples = add_src->ne[3]; - rms_norm_mul_f32_cuda(src0_d, mul_d,add_d,dst_d, + rms_norm_mul_f32_cuda(rms_norm_src, mul_src,add_src,dst_d, ne00,ne01, ne02, ne03, /*s00*/ s01, s02, s03, /*mul_s00*/ mul_s01, mul_s02, mul_s03, diff --git a/ggml/src/ggml-cuda/pad.cu b/ggml/src/ggml-cuda/pad.cu index 31cd00f77..e39a6bf04 100644 --- a/ggml/src/ggml-cuda/pad.cu +++ b/ggml/src/ggml-cuda/pad.cu @@ -15,49 +15,52 @@ static __global__ void pad_f32(const float * src, size_t s00, size_t s01, size_t // blockIdx.z: i3*ne2+i2 // blockIdx.y: i1 // blockIDx.x: i0 / CUDA_PAD_BLOCK_SIZE - // gridDim.y: ne1 + // gridDim.y and gridDim.z are capped at 65535, blocks stride over larger ne1 and ne2*ne3 int i0 = threadIdx.x + blockIdx.x * blockDim.x; - int i1 = blockIdx.y; - int i2 = blockIdx.z % ne2; - int i3 = blockIdx.z / ne2; - - if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) { + if (i0 >= ne0) { return; } - const int64_t dst_idx = i3 * (ne0 * ne1 * ne2) + i2 * (ne0 * ne1) + i1 * ne0 + i0; - - if (!circular) { - if ((i0 >= lp0 && i0 < ne0 - rp0) && (i1 >= lp1 && i1 < ne1 - rp1) && (i2 >= lp2 && i2 < ne2 - rp2) && - (i3 >= lp3 && i3 < ne3 - rp3)) { - const int64_t i00 = i0 - lp0; - const int64_t i01 = i1 - lp1; - const int64_t i02 = i2 - lp2; - const int64_t i03 = i3 - lp3; - - const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00; - - dst[dst_idx] = src[src_idx]; - } else { - dst[dst_idx] = 0.0f; + for (int i1 = blockIdx.y; i1 < ne1; i1 += gridDim.y) { + for (int i23 = blockIdx.z; i23 < ne2 * ne3; i23 += gridDim.z) { + int i2 = i23 % ne2; + int i3 = i23 / ne2; + + const int64_t dst_idx = i3 * (ne0 * ne1 * ne2) + i2 * (ne0 * ne1) + i1 * ne0 + i0; + + if (!circular) { + if ((i0 >= lp0 && i0 < ne0 - rp0) && (i1 >= lp1 && i1 < ne1 - rp1) && (i2 >= lp2 && i2 < ne2 - rp2) && + (i3 >= lp3 && i3 < ne3 - rp3)) { + const int64_t i00 = i0 - lp0; + const int64_t i01 = i1 - lp1; + const int64_t i02 = i2 - lp2; + const int64_t i03 = i3 - lp3; + + const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00; + + dst[dst_idx] = src[src_idx]; + } else { + dst[dst_idx] = 0.0f; + } + } + // circular means on a torus, so x and y wrap around + else { + const int64_t ne00 = ne0 - lp0 - rp0; + const int64_t ne01 = ne1 - lp1 - rp1; + const int64_t ne02 = ne2 - lp2 - rp2; + const int64_t ne03 = ne3 - lp3 - rp3; + + const int64_t i00 = wrap_around(i0 - lp0, ne00); + const int64_t i01 = wrap_around(i1 - lp1, ne01); + const int64_t i02 = wrap_around(i2 - lp2, ne02); + const int64_t i03 = wrap_around(i3 - lp3, ne03); + + const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00; + + dst[dst_idx] = src[src_idx]; + } } } - // circular means on a torus, so x and y wrap around - else { - const int64_t ne00 = ne0 - lp0 - rp0; - const int64_t ne01 = ne1 - lp1 - rp1; - const int64_t ne02 = ne2 - lp2 - rp2; - const int64_t ne03 = ne3 - lp3 - rp3; - - const int64_t i00 = wrap_around(i0 - lp0, ne00); - const int64_t i01 = wrap_around(i1 - lp1, ne01); - const int64_t i02 = wrap_around(i2 - lp2, ne02); - const int64_t i03 = wrap_around(i3 - lp3, ne03); - - const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00; - - dst[dst_idx] = src[src_idx]; - } } @@ -67,7 +70,7 @@ static void pad_f32_cuda(const float * src, size_t s00, size_t s01, size_t s02, const int ne0, const int ne1, const int ne2, const int ne3, const bool circular, cudaStream_t stream) { int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE; - dim3 gridDim(num_blocks, ne1, ne2 * ne3); + dim3 gridDim(num_blocks, std::min(ne1, 65535), std::min(ne2 * ne3, 65535)); pad_f32<<<gridDim, CUDA_PAD_BLOCK_SIZE, 0, stream>>>(src, s00, s01, s02, s03, dst, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3, ne0, ne1, ne2, ne3, circular); diff --git a/ggml/src/ggml-cuda/pool2d.cu b/ggml/src/ggml-cuda/pool2d.cu index c6d51e4d6..83a49753c 100644 --- a/ggml/src/ggml-cuda/pool2d.cu +++ b/ggml/src/ggml-cuda/pool2d.cu @@ -50,6 +50,65 @@ static __global__ void pool2d_nchw_kernel( o_ptr[cur_oh * ow + cur_ow] = res; } +template <typename Ti, typename To> +static __global__ void pool2d_nchw_kernel_warp( + const int ih, const int iw, const int oh, const int ow, + const int kh, const int kw, const int sh, const int sw, + const int ph, const int pw, const int parallel_elements, + const Ti * __restrict__ src, To * __restrict__ dst, const enum ggml_op_pool op) { + const int warp_id = (threadIdx.x + blockIdx.x * blockDim.x) / WARP_SIZE; + const int lane = threadIdx.x % WARP_SIZE; + if (warp_id >= parallel_elements) { + return; + } + + const int I_HW = ih * iw; + const int O_HW = oh * ow; + const int nc = warp_id / O_HW; + const int cur_oh = warp_id % O_HW / ow; + const int cur_ow = warp_id % O_HW % ow; + const Ti* i_ptr = src + nc * I_HW; + + const int start_h = cur_oh * sh - ph; + const int bh = max(0, start_h); + const int eh = min(ih, start_h + kh); + const int start_w = cur_ow * sw - pw; + const int bw = max(0, start_w); + const int ew = min(iw, start_w + kw); + + const int win_w = ew - bw; + const int win_elems = (eh - bh) * win_w; + const To scale = 1. / (kh * kw); + + To res; + switch (op) { + case GGML_OP_POOL_AVG: res = 0; break; + case GGML_OP_POOL_MAX: res = -FLT_MAX; break; + default: res = 0; assert(false); + } + + for (int t = lane; t < win_elems; t += WARP_SIZE) { + const int i = bh + t / win_w; + const int j = bw + t % win_w; + const Ti cur = i_ptr[i * iw + j]; + switch (op) { + case GGML_OP_POOL_AVG: res += cur * scale; break; + case GGML_OP_POOL_MAX: res = max(res, (To)cur); break; + default: break; + } + } + +#pragma unroll + for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) { + const To other = __shfl_xor_sync(0xFFFFFFFF, res, offset, WARP_SIZE); + res = (op == GGML_OP_POOL_MAX) ? max(res, other) : res + other; + } + + if (lane == 0) { + dst[nc * O_HW + cur_oh * ow + cur_ow] = res; + } +} + static void pool2d_nchw_kernel_f32_f32_cuda( const int ih, const int iw, const int oh, const int ow, const int kh, const int kw, const int sh, const int sw, @@ -57,6 +116,13 @@ static void pool2d_nchw_kernel_f32_f32_cuda( const float * src, float * dst, const enum ggml_op_pool op, cudaStream_t stream) { + if (kh * kw >= CUDA_POOL2D_WARP_KERNEL_MIN_WINDOW) { + const int warps_per_block = CUDA_POOL2D_BLOCK_SIZE / WARP_SIZE; + const int num_blocks = (parallel_elements + warps_per_block - 1) / warps_per_block; + pool2d_nchw_kernel_warp<<<num_blocks, CUDA_POOL2D_BLOCK_SIZE, 0, stream>>>(ih, iw, oh, ow, kh, kw, sh, sw, ph, pw, parallel_elements, src, dst, op); + return; + } + const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE; dim3 block_nums(num_blocks); pool2d_nchw_kernel<<<block_nums, CUDA_POOL2D_BLOCK_SIZE, 0, stream>>>(ih, iw, oh, ow, kh, kw, sh, sw, ph, pw, parallel_elements, src, dst, op); diff --git a/ggml/src/ggml-cuda/pool2d.cuh b/ggml/src/ggml-cuda/pool2d.cuh index 7841292bc..e2c7fb441 100644 --- a/ggml/src/ggml-cuda/pool2d.cuh +++ b/ggml/src/ggml-cuda/pool2d.cuh @@ -1,5 +1,6 @@ #include "common.cuh" #define CUDA_POOL2D_BLOCK_SIZE 256 +#define CUDA_POOL2D_WARP_KERNEL_MIN_WINDOW 32 void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index 935b2bf4b..2acf1f6d0 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -83,21 +83,21 @@ static __global__ void quantize_q8_1( ggml_cuda_pdl_sync(); const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f; float amax = fabsf(xi); - float sum = xi; amax = warp_reduce_max<QK8_1>(amax); - sum = warp_reduce_sum<QK8_1>(sum); const float d = amax / 127.0f; const int8_t q = amax == 0.0f ? 0 : roundf(xi / d); + const int sum = warp_reduce_sum<QK8_1>((int) q); + y[ib].qs[iqs] = q; if (iqs > 0) { return; } - y[ib].ds = make_half2(d, sum); + y[ib].ds = make_half2(d, d * sum); } __device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) { @@ -500,17 +500,6 @@ static __global__ void quantize_mmq_q8_1( amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); } - float sum; - if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { - sum = xi.x + xi.y + xi.z + xi.w; - - // Calculate sums across vals_per_sum/4 threads. -#pragma unroll - for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { - sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); - } - } - const float d_inv = 127.0f / amax; char4 q; q.x = roundf(xi.x*d_inv); @@ -519,6 +508,19 @@ static __global__ void quantize_mmq_q8_1( q.w = roundf(xi.w*d_inv); const float d = 1.0f / d_inv; + float sum = 0.0f; + if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + int sum_q = int(q.x) + int(q.y) + int(q.z) + int(q.w); + + // Calculate sums across vals_per_sum/4 threads. +#pragma unroll + for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { + sum_q += __shfl_xor_sync(0xFFFFFFFF, sum_q, offset, WARP_SIZE); + } + + sum = d * float(sum_q); + } + // write the block once (normal) or to each of the token's compact rows (scatter) const int nwrite = scatter ? n_expert_used : 1; #pragma unroll @@ -555,10 +557,9 @@ static __global__ void quantize_mmq_q8_1( } void quantize_row_q8_1_cuda( - const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, + const float * x, void * vy, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { - GGML_ASSERT(!ids); GGML_ASSERT(ne0 % QK8_1 == 0); const uint3 ne2_fastdiv = init_fastdiv_values(ne2); @@ -568,7 +569,6 @@ void quantize_row_q8_1_cuda( const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream); ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv); - GGML_UNUSED(type_src0); } void quantize_mmq_q8_1_cuda( @@ -597,7 +597,81 @@ void quantize_mmq_q8_1_cuda( break; default: GGML_ABORT("fatal error"); + } +} + +// Each thread quantizes 4 values of one column, see quantize.cuh for the layout. +template <mmq_q8_1_ds_layout ds_layout> +static __global__ void quantize_mmq_few_q8_1( + const float * x_ptr, void * vy_ptr, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int ne1, const int ne2) { + ggml_cuda_pdl_lc(); + const float * GGML_CUDA_RESTRICT x = x_ptr; + int4 * GGML_CUDA_RESTRICT vy = (int4 *) vy_ptr; + const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4; + + if (i0 >= ne00) { + return; + } + ggml_cuda_pdl_sync(); + + const int64_t i2 = blockIdx.z % ne2; + const int64_t i3 = blockIdx.z / ne2; + const float4 xi = (int) blockIdx.x < ne1 ? *(const float4 *) (x + i3*s03 + i2*s02 + blockIdx.x*s01 + i0) : make_float4(0.0f, 0.0f, 0.0f, 0.0f); + + // the 32 values of a sub-block are held by 8 threads + float amax = fmaxf(fmaxf(fabsf(xi.x), fabsf(xi.y)), fmaxf(fabsf(xi.z), fabsf(xi.w))); + amax = warp_reduce_max<8>(amax); + + const float d_inv = 127.0f / amax; + char4 q; + q.x = roundf(xi.x*d_inv); + q.y = roundf(xi.y*d_inv); + q.z = roundf(xi.z*d_inv); + q.w = roundf(xi.w*d_inv); + const float d = 1.0f / d_inv; + + // as in quantize_mmq_q8_1: the sum of the quantized values; the int warp_reduce_sum reduces the whole warp on Ampere+ + [[maybe_unused]] float sum = 0.0f; + if constexpr (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + sum = d*warp_reduce_sum<8>(float(q.x + q.y + q.z + q.w)); + } + + const int s = i0 % QK_K / QK8_1; // sub-block + const int k = i0 % QK8_1; // first value in sub-block + const int g = blockIdx.x % MMQ_FEW_NCOLS; // column in group + int4 * yg = vy + ((blockIdx.z*(ne00/QK_K) + i0/QK_K)*(gridDim.x/MMQ_FEW_NCOLS) + blockIdx.x/MMQ_FEW_NCOLS)*MMQ_FEW_Q8_1_INT4; + ((int *) (yg + (s/2)*WARP_SIZE + 4*g + (k % 16)/4))[2*(s % 2) + k/16] = *(const int *) &q; + if (k == 0) { + int4 * ds = yg + 4*WARP_SIZE + 4*(g/2) + s/2; + if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + ((float *) ds)[2*(s % 2) + g % 2] = d; + } else { + ((half2 *) ds)[2*(s % 2) + g % 2] = make_half2(d, sum); + } + } +} + +void quantize_mmq_few_q8_1_cuda( + const float * x, void * vy, const mmq_q8_1_ds_layout ds_layout, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { + GGML_ASSERT(ne00 % QK_K == 0); + + const int64_t block_num_y = (ne00 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const dim3 num_blocks(GGML_PAD(ne1, MMQ_FEW_NCOLS), block_num_y, ne2*ne3); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream); + switch (ds_layout) { + case MMQ_Q8_1_DS_LAYOUT_D4: + ggml_cuda_kernel_launch(quantize_mmq_few_q8_1<MMQ_Q8_1_DS_LAYOUT_D4>, launch_params, + x, vy, ne00, s01, s02, s03, (int) ne1, (int) ne2); + break; + case MMQ_Q8_1_DS_LAYOUT_DS4: + ggml_cuda_kernel_launch(quantize_mmq_few_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4>, launch_params, + x, vy, ne00, s01, s02, s03, (int) ne1, (int) ne2); break; + default: + GGML_ABORT("fatal error"); } } @@ -627,7 +701,6 @@ void quantize_scatter_mmq_q8_1_cuda( break; default: GGML_ABORT("fatal error"); - break; } } diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh index 5f08dcbfe..11720ef93 100644 --- a/ggml/src/ggml-cuda/quantize.cuh +++ b/ggml/src/ggml-cuda/quantize.cuh @@ -11,14 +11,9 @@ static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access."); static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access."); -typedef void (*quantize_cuda_t)( - const float * x, const int32_t * ids, void * vy, - ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, - int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); - void quantize_row_q8_1_cuda( - const float * x, const int32_t * ids, void * vy, - ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, + const float * x, void * vy, + int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); void quantize_mmq_q8_1_cuda( @@ -26,6 +21,17 @@ void quantize_mmq_q8_1_cuda( ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); +// src1 of the few-row MMQ in the layout of the MMA fragments. Per channel, 256 values and group of MMQ_FEW_NCOLS columns: +// qs: int4[4][32], [j][lane] holds the B fragments of sub-blocks 2j and 2j+1 of lane (g, t): the values [4t, 4t + 4) and [16 + 4t, 16 + 4t + 4) of column g. +// ds: int4[4][4], [t][j] holds d (D4, float) or (d, sum) (DS4, half2) of the columns 2t and 2t+1 for sub-block 2j, then 2j+1. +// The column groups of a unit are adjacent, columns beyond ne1 are zero. +static constexpr int MMQ_FEW_NCOLS = 8; // src1 columns per column group +static constexpr int MMQ_FEW_Q8_1_INT4 = 4*WARP_SIZE + 4*4; // int4 per 256 values and column group + +void quantize_mmq_few_q8_1_cuda( + const float * x, void * vy, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, + int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); + void quantize_mmq_fp4_cuda(const float * x, const int32_t * ids, void * vy, diff --git a/ggml/src/ggml-cuda/roll.cu b/ggml/src/ggml-cuda/roll.cu index a339dfc1a..c9bbe76f0 100644 --- a/ggml/src/ggml-cuda/roll.cu +++ b/ggml/src/ggml-cuda/roll.cu @@ -17,6 +17,10 @@ static __global__ void roll_f32_cuda(const float * __restrict__ src, const int64_t ne01, const int64_t ne02, const int64_t ne03, + const int64_t nb00, + const int64_t nb01, + const int64_t nb02, + const int64_t nb03, const int s0, const int s1, const int s2, @@ -39,7 +43,7 @@ static __global__ void roll_f32_cuda(const float * __restrict__ src, const int64_t d3 = wrap_index(i3 - s3, ne03); dst[i3 * (ne00 * ne01 * ne02) + i2 * (ne01 * ne00) + i1 * ne00 + i0] = - src[d3 * (ne00 * ne01 * ne02) + d2 * (ne01 * ne00) + d1 * ne00 + d0]; + src[(d3 * nb03 + d2 * nb02 + d1 * nb01 + d0 * nb00) / sizeof(float)]; } void ggml_cuda_op_roll(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -63,5 +67,5 @@ void ggml_cuda_op_roll(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { int64_t num_blocks = (sz + CUDA_ROLL_BLOCK_SIZE - 1) / CUDA_ROLL_BLOCK_SIZE; roll_f32_cuda<<<num_blocks, CUDA_ROLL_BLOCK_SIZE, 0, stream>>>( - src0_d, dst_d, ne00, ne01, ne02, ne03, s0, s1, s2, s3); + src0_d, dst_d, ne00, ne01, ne02, ne03, nb00, nb01, nb02, nb03, s0, s1, s2, s3); } diff --git a/ggml/src/ggml-cuda/rope.cu b/ggml/src/ggml-cuda/rope.cu index e546fb655..2604d014f 100644 --- a/ggml/src/ggml-cuda/rope.cu +++ b/ggml/src/ggml-cuda/rope.cu @@ -196,9 +196,9 @@ static __global__ void rope_neox(const T * x, dst[idst + n_offs/2 + n_dims / 2] = ggml_cuda_cast<D>(x0 * sin_theta + x1 * cos_theta); } -template <bool forward, bool has_ff, typename T> +template <bool forward, bool has_ff, typename T, typename D> static __global__ void rope_multi(const T * x, - T * dst, + D * dst, const int ne00, const int ne01, const int ne02, @@ -219,6 +219,8 @@ static __global__ void rope_multi(const T * x, const float * freq_factors, const mrope_sections sections, const bool is_imrope, + const int64_t * row_indices, + const int set_rows_stride, const bool inplace) { const int i0 = 2 * (blockDim.y * blockIdx.y + threadIdx.y); @@ -236,12 +238,20 @@ static __global__ void rope_multi(const T * x, const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03; ggml_cuda_pdl_sync(); + + // Fusion optimization: ROPE + VIEW + SET_ROWS. + // The rope output is viewed as a 1D tensor and offset based on a row index in row_indices. + if (set_rows_stride != 0) { + idst = i1 * s1 + i0 / 2; + idst += row_indices[i2] * set_rows_stride; + } + if (i0 < n_offs || i0 >= n_offs + n_dims) { if (inplace) { return; } - dst[idst + i0/2 + 0] = x[ix + i0/2 + 0]; - dst[idst + i0/2 + 1] = x[ix + i0/2 + 1]; + dst[idst + i0/2 + 0] = ggml_cuda_cast<D>(x[ix + i0/2 + 0]); + dst[idst + i0/2 + 1] = ggml_cuda_cast<D>(x[ix + i0/2 + 1]); return; } @@ -286,8 +296,8 @@ static __global__ void rope_multi(const T * x, const float x0 = x[ix + n_offs/2 + 0]; const float x1 = x[ix + n_offs/2 + n_dims/2]; - dst[idst + n_offs/2 + 0] = x0*cos_theta - x1*sin_theta; - dst[idst + n_offs/2 + n_dims/2] = x0*sin_theta + x1*cos_theta; + dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta); + dst[idst + n_offs/2 + n_dims/2] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta); } template <bool forward, bool has_ff, typename T> @@ -443,9 +453,9 @@ static void rope_neox_cuda(const T * x, } } -template <bool forward, typename T> +template <bool forward, typename T, typename D> static void rope_multi_cuda(const T * x, - T * dst, + D * dst, const int ne00, const int ne01, const int ne02, @@ -467,6 +477,8 @@ static void rope_multi_cuda(const T * x, const float * freq_factors, const mrope_sections sections, const bool is_imrope, + const int64_t * row_indices, + const int set_rows_stride, const bool inplace, cudaStream_t stream) { GGML_ASSERT(ne00 % 2 == 0); @@ -478,14 +490,14 @@ static void rope_multi_cuda(const T * x, if (freq_factors == nullptr) { const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(rope_multi<forward, false, T>, launch_params, + ggml_cuda_kernel_launch(rope_multi<forward, false, T, D>, launch_params, x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor, - attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, inplace); + attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, row_indices, set_rows_stride, inplace); } else { const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(rope_multi<forward, true, T>, launch_params, + ggml_cuda_kernel_launch(rope_multi<forward, true, T, D>, launch_params, x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor, - attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, inplace); + attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, row_indices, set_rows_stride, inplace); } } @@ -579,7 +591,6 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, //const int n_past = ((int32_t *) dst->op_params)[0]; const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; - //const int n_ctx = ((int32_t *) dst->op_params)[3]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; const int n_offs = ((int32_t *) dst->op_params)[15]; mrope_sections sections; @@ -625,7 +636,7 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, } rope_corr_dims corr_dims; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims.v); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(dst), corr_dims.v); // compute if (is_neox) { @@ -648,14 +659,21 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, GGML_ABORT("fatal error"); } } else if (is_mrope && !is_vision) { - if (src0->type == GGML_TYPE_F32) { - rope_multi_cuda<forward>((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1, - s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, - corr_dims, freq_factors, sections, is_imrope, inplace, stream); - } else if (src0->type == GGML_TYPE_F16) { - rope_multi_cuda<forward>((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1, - s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, - corr_dims, freq_factors, sections, is_imrope, inplace, stream); + if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) { + rope_multi_cuda<forward, float, float>((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, + row_indices, set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) { + rope_multi_cuda<forward, float, half>((const float *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, + row_indices, set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) { + rope_multi_cuda<forward, half, half>((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, + row_indices, set_rows_stride, inplace, stream); } else { GGML_ABORT("fatal error"); } @@ -709,7 +727,7 @@ void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rope // one block per row: block_reduce gives the norm scale, then each thread applies mul and rope to the elements it owns template <int block_size, bool has_ff, typename D> static __global__ void rms_norm_mul_rope_f32( - const float * x, D * dst, const int ncols, + const float * x, D * dst, const int ncols, const int nchannels, const int nsamples, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t s1, const int64_t s2, const int64_t s3, const float eps, @@ -724,66 +742,76 @@ static __global__ void rms_norm_mul_rope_f32( const int64_t * row_indices, const int set_rows_stride, const bool is_neox) { ggml_cuda_pdl_lc(); - const int row = blockIdx.x; - const int channel = blockIdx.y; - const int sample = blockIdx.z; - const int tid = threadIdx.x; - - x += sample*s03 + channel*s02 + row*s01; - - const uint32_t mul_row = fastmodulo(row, mul_nrows_packed); - const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed); - const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed); - mul += mul_sample*mul_s03 + mul_channel*mul_s02 + mul_row*mul_s01; - - float tmp = 0.0f; - - ggml_cuda_pdl_sync(); - for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; - tmp += xi * xi; - } + const int row = blockIdx.x; + const int tid = threadIdx.x; extern __shared__ float s_sum[]; - tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum); - const float scale = rsqrtf(tmp/ncols + eps); - - int64_t idst = sample*s3 + channel*s2 + row*s1; - if (set_rows_stride != 0) { - idst = row*s1 + row_indices[channel]*set_rows_stride; - } - dst += idst; - - for (int i0 = 2*tid; i0 < ncols; i0 += 2*block_size) { - int ix0; - int ix1; - if (is_neox && i0 < n_dims) { - ix0 = i0/2; - ix1 = i0/2 + n_dims/2; - } else { - ix0 = i0 + 0; - ix1 = i0 + 1; - } - - const float x0 = scale * x[ix0] * mul[fastmodulo(ix0, mul_ncols_packed)]; - const float x1 = scale * x[ix1] * mul[fastmodulo(ix1, mul_ncols_packed)]; + ggml_cuda_pdl_sync(); - if (i0 >= n_dims) { - dst[ix0] = ggml_cuda_cast<D>(x0); - dst[ix1] = ggml_cuda_cast<D>(x1); - continue; + // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples + for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + const float * xc = x + sample*s03 + channel*s02 + row*s01; + + const uint32_t mul_row = fastmodulo(row, mul_nrows_packed); + const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed); + const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed); + const float * mulc = mul + mul_sample*mul_s03 + mul_channel*mul_s02 + mul_row*mul_s01; + + float tmp = 0.0f; + + for (int col = tid; col < ncols; col += block_size) { + const float xi = xc[col]; + tmp += xi * xi; + } + + tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum); + + const float scale = rsqrtf(tmp/ncols + eps); + + int64_t idst = sample*s3 + channel*s2 + row*s1; + if (set_rows_stride != 0) { + idst = row*s1 + row_indices[channel]*set_rows_stride; + } + D * dstc = dst + idst; + + for (int i0 = 2*tid; i0 < ncols; i0 += 2*block_size) { + int ix0; + int ix1; + if (is_neox && i0 < n_dims) { + ix0 = i0/2; + ix1 = i0/2 + n_dims/2; + } else { + ix0 = i0 + 0; + ix1 = i0 + 1; + } + + const float x0 = scale * xc[ix0] * mulc[fastmodulo(ix0, mul_ncols_packed)]; + const float x1 = scale * xc[ix1] * mulc[fastmodulo(ix1, mul_ncols_packed)]; + + if (i0 >= n_dims) { + dstc[ix0] = ggml_cuda_cast<D>(x0); + dstc[ix1] = ggml_cuda_cast<D>(x1); + continue; + } + + const float theta_base = pos[channel]*powf(theta_scale, i0/2.0f); + const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f; + + float cos_theta; + float sin_theta; + rope_yarn<true>(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta); + + dstc[ix0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta); + dstc[ix1] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta); + } + + if constexpr (block_size > WARP_SIZE) { + // sync is needed as we reuse s_sum across block_reduce invocations, see #26385 + __syncthreads(); + } } - - const float theta_base = pos[channel]*powf(theta_scale, i0/2.0f); - const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f; - - float cos_theta; - float sin_theta; - rope_yarn<true>(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta); - - dst[ix0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta); - dst[ix1] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta); } } @@ -806,7 +834,7 @@ static void rms_norm_mul_rope_cuda( const bool is_neox, cudaStream_t stream) { GGML_ASSERT(ncols % 2 == 0); - const dim3 blocks_num(nrows, nchannels, nsamples); + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); const float theta_scale = powf(freq_base, -2.0f/n_dims); @@ -820,13 +848,13 @@ static void rms_norm_mul_rope_cuda( const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream}; if (freq_factors == nullptr) { ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, false, D>, launch_params, - x, dst, ncols, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, is_neox); } else { ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, true, D>, launch_params, - x, dst, ncols, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, is_neox); @@ -836,13 +864,13 @@ static void rms_norm_mul_rope_cuda( const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream}; if (freq_factors == nullptr) { ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<1024, false, D>, launch_params, - x, dst, ncols, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, is_neox); } else { ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<1024, true, D>, launch_params, - x, dst, ncols, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, is_neox); @@ -900,7 +928,7 @@ void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, const float * freq_factors = rope->src[2] != nullptr ? (const float *) rope->src[2]->data : nullptr; rope_corr_dims corr_dims; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims.v); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(rope), corr_dims.v); const size_t ts0 = ggml_type_size(x->type); GGML_ASSERT(x->nb[0] == ts0); diff --git a/ggml/src/ggml-cuda/scale.cu b/ggml/src/ggml-cuda/scale.cu index 5648b9261..d53f906c5 100644 --- a/ggml/src/ggml-cuda/scale.cu +++ b/ggml/src/ggml-cuda/scale.cu @@ -3,23 +3,45 @@ #define MAX_GRIDDIM_X 0x7FFFFFFF +// src and dst may be views: rows are contiguous, dims 1..3 follow the strides (in elements). template <typename T> -static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const int64_t nelements) { +static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const uint32_t k, + const uint3 ne0, const uint3 ne1, const uint3 ne2, + const uint32_t s01, const uint32_t s02, const uint32_t s03, + const uint32_t s1, const uint32_t s2, const uint32_t s3) { ggml_cuda_pdl_lc(); - int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x; - int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x; + const uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; ggml_cuda_pdl_sync(); - for (int64_t i = tid; i < nelements; i += stride) { - dst[i] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i]) + bias); + if (i >= k) { + return; } + + const uint2 d0 = fast_div_modulo(i, ne0); // <i / ne0, i0> + const uint2 d1 = fast_div_modulo(d0.x, ne1); // <i / (ne0*ne1), i1> + const uint2 d2 = fast_div_modulo(d1.x, ne2); // <i3, i2> + + const size_t i_src = d0.y + size_t(d1.y)*s01 + size_t(d2.y)*s02 + size_t(d2.x)*s03; + const size_t i_dst = d0.y + size_t(d1.y)*s1 + size_t(d2.y)*s2 + size_t(d2.x)*s3; + + dst[i_dst] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i_src]) + bias); } template <typename T> -static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) { - const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE; +static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const ggml_tensor * src0, const ggml_tensor * t, cudaStream_t stream) { + const int64_t k = ggml_nelements(src0); + const size_t ts = sizeof(T); + GGML_ASSERT(k <= std::numeric_limits<uint32_t>::max()); + + const uint3 ne0 = init_fastdiv_values(src0->ne[0]); + const uint3 ne1 = init_fastdiv_values(src0->ne[1]); + const uint3 ne2 = init_fastdiv_values(src0->ne[2]); + + const int64_t num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE; const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(MIN(MAX_GRIDDIM_X, num_blocks), CUDA_SCALE_BLOCK_SIZE, 0, stream); - ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, nelements); + ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, (uint32_t) k, ne0, ne1, ne2, + src0->nb[1]/ts, src0->nb[2]/ts, src0->nb[3]/ts, + t->nb[1]/ts, t->nb[2]/ts, t->nb[3]/ts); } void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -28,6 +50,7 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16); GGML_ASSERT(dst->type == src0->type); + GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst)); float scale; float bias; @@ -35,8 +58,8 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { memcpy(&bias, (float *) dst->op_params + 1, sizeof(float)); if (src0->type == GGML_TYPE_BF16) { - scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, ggml_nelements(src0), stream); + scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, src0, dst, stream); } else { - scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, ggml_nelements(src0), stream); + scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, src0, dst, stream); } } diff --git a/ggml/src/ggml-cuda/set-rows.cu b/ggml/src/ggml-cuda/set-rows.cu index 465997065..4be3e649b 100644 --- a/ggml/src/ggml-cuda/set-rows.cu +++ b/ggml/src/ggml-cuda/set-rows.cu @@ -257,6 +257,16 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s nb1, nb2, nb3, stream ); + } else if (dst->type == GGML_TYPE_F8_E4M3) { + set_rows_cuda( + src0_d, src1_d, (ggml_fp8_e4m3_t *) dst->data, + ne00, ne01, ne02, ne03, + ne10, ne11, ne12, ne13, + nb01, nb02, nb03, + nb10, nb11, nb12, + nb1, nb2, nb3, + stream + ); } else if (dst->type == GGML_TYPE_Q4_0) { set_rows_cuda_quant<idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>( src0_d, src1_d, (block_q4_0*)dst->data, diff --git a/ggml/src/ggml-cuda/ssm-conv.cu b/ggml/src/ggml-cuda/ssm-conv.cu index 1463169cf..f41438b31 100644 --- a/ggml/src/ggml-cuda/ssm-conv.cu +++ b/ggml/src/ggml-cuda/ssm-conv.cu @@ -151,9 +151,10 @@ static void ssm_conv_f32_cuda(const float * src0, const float * src1, const floa case 3: launch_kernel(std::integral_constant<int, 3 >{}); break; case 4: launch_kernel(std::integral_constant<int, 4 >{}); break; case 5: launch_kernel(std::integral_constant<int, 5 >{}); break; + case 7: launch_kernel(std::integral_constant<int, 7 >{}); break; case 9: launch_kernel(std::integral_constant<int, 9 >{}); break; case 15: launch_kernel(std::integral_constant<int, 15>{}); break; - default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 9, 15 right now."); + default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 7, 9, 15 right now."); } } diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cu index f6a41b407..b88c9d6ea 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cu +++ b/ggml/src/ggml-cuda/ssm-scan.cu @@ -149,7 +149,7 @@ __global__ void __launch_bounds__(d_state, 1) const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3, const int src2_nb1, const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3, - const int64_t s_off, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok, const int64_t K) { + char * s_base, const int64_t s_slot_bytes, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok, const int64_t K) { const float * GGML_CUDA_RESTRICT src0 = src0_ptr; const float * GGML_CUDA_RESTRICT src1 = src1_ptr; const float * GGML_CUDA_RESTRICT src2 = src2_ptr; @@ -184,7 +184,7 @@ __global__ void __launch_bounds__(d_state, 1) const float * B_warp = (const float *) ((const char *) src4 + (seq_idx * src4_nb3) + (group_off)); const float * C_warp = (const float *) ((const char *) src5 + (seq_idx * src5_nb3) + (group_off)); float * y_warp = dst + (seq_idx * n_tok * n_head * d_head) + warp_idx; - float * s_warp = (float *) ((char *) dst + s_off + seq_idx * src0_nb3 + head_idx * src0_nb2 + head_off * d_state); + float * s_warp = (float *) (s_base + seq_idx * src0_nb3 + head_idx * src0_nb2 + head_off * d_state); // strides across n_seq_tokens const int stride_x = src1_nb2 / sizeof(float); @@ -227,7 +227,7 @@ __global__ void __launch_bounds__(d_state, 1) // Slot 0 is the final state written below; slots 1..K-1 are rollback snapshots. const int64_t slot = n_tok - 1 - i; if (K > 1 && slot > 0 && slot < K) { - float * s_snapshot_warp = (float *) ((char *) dst + s_off + (slot * gridDim.y + seq_idx) * src0_nb3 + head_idx * src0_nb2 + head_off * d_state); + float * s_snapshot_warp = (float *) ((char *) s_warp + slot * s_slot_bytes); #pragma unroll for (int j = 0; j < c_factor; j++) { s_snapshot_warp[WARP_SIZE * j + lane] = state[j]; @@ -248,11 +248,25 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3, const int64_t s_off, const int64_t d_state, const int64_t head_dim, const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq, - const int64_t K, cudaStream_t stream) { + const int64_t K, const ggml_cuda_ssm_scan_fused_cache * cache, cudaStream_t stream) { + // when fused, the states go straight into the recurrent cache and the dst tail is left alone + char * const s_base = cache ? (char *) cache->data : (char *) dst + s_off; + const int64_t s_slot_bytes = cache ? cache->slot_stride * (int64_t) sizeof(float) : n_seq * (int64_t) src0_nb3; + // NOTE: if you change conditions here, be sure to update the corresponding supports_op condition! if (src3_nb1 == sizeof(float)) { // Mamba-2 - if (d_state == 96) { + if (d_state == 64) { // Zamba2 + constexpr int threads = 64; + constexpr int num_warps = threads/WARP_SIZE; + + const dim3 blocks((n_head * head_dim + (num_warps - 1)) / num_warps, n_seq, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream); + ggml_cuda_kernel_launch(ssm_scan_f32_group<64/WARP_SIZE, 64>, launch_params, + src0, src1, src2, src3, src4, src5, src6, dst, + src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, + src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); + } else if (d_state == 96) { constexpr int threads = 96; constexpr int num_warps = threads/WARP_SIZE; @@ -261,7 +275,7 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa ggml_cuda_kernel_launch(ssm_scan_f32_group<96/WARP_SIZE, 96>, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, - src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K); + src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); } else if (d_state == 128) { constexpr int threads = 128; constexpr int num_warps = threads/WARP_SIZE; @@ -271,7 +285,7 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa ggml_cuda_kernel_launch(ssm_scan_f32_group<128/WARP_SIZE, 128>, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, - src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K); + src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); } else if (d_state == 256) { // Falcon-H1 constexpr int threads = 256; constexpr int num_warps = threads/WARP_SIZE; @@ -281,9 +295,9 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa ggml_cuda_kernel_launch(ssm_scan_f32_group<256/WARP_SIZE, 256>, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, - src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K); + src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); } else { - GGML_ABORT("doesn't support d_state!=(96, 128 or 256)."); + GGML_ABORT("doesn't support d_state!=(64, 96, 128 or 256)."); } } else { // Mamba-1 @@ -570,12 +584,13 @@ __global__ void ssm_ssd_scale_state_kernel( } // Copy initial state from src0[ids[s]] into s_cur for each sequence. +// src0 and s_cur can alias when the state is written straight into the cache. // Grid: (ceil(d_state * head_dim * n_head / BLOCK), n_seqs) template <int BLOCK_SIZE> __global__ void ssm_ssd_init_state_kernel( - const float * __restrict__ src0, // {d_state, head_dim, n_head, n_rs} + const float * src0, // {d_state, head_dim, n_head, n_rs} const int32_t * __restrict__ ids, // {n_seqs} - float * __restrict__ s_cur, // {d_state, head_dim, n_head, n_seqs} + float * s_cur, // {d_state, head_dim, n_head, n_seqs} const int state_size, // d_state * head_dim * n_head const int64_t s0_stride_seq) { // elements between state rows const int s = blockIdx.y; @@ -599,7 +614,8 @@ static void ssm_scan_ssd_f32_cuda( const int A_stride, // A (src3) stride between heads const int B_stride_tok, const int B_stride_seq, // B (src4) strides const int C_stride_tok, const int C_stride_seq, // C (src5) strides - const int64_t s_off, const int64_t d_state, const int64_t head_dim, + float * s_cur, // state: dst state tail, or the cache when fused + const int64_t d_state, const int64_t head_dim, const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq) { cudaStream_t stream = ctx.stream(); @@ -625,7 +641,6 @@ static void ssm_scan_ssd_f32_cuda( matmul_t * X_dt = X_dt_buf.get(); matmul_t * B_weighted = B_w_buf.get(); float * C_scaled = C_s_buf.get(); - float * s_cur = (float *)((char *)dst_d + s_off); // write state directly to dst // Step 1: softplus(dt) and parallel prefix sum over full sequence { @@ -780,7 +795,8 @@ static void ssm_scan_ssd_f32_cuda( } #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) -void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +static void ggml_cuda_op_ssm_scan_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + const ggml_cuda_ssm_scan_fused_cache * cache) { const struct ggml_tensor * src0 = dst->src[0]; // s const struct ggml_tensor * src1 = dst->src[1]; // x const struct ggml_tensor * src2 = dst->src[2]; // dt @@ -864,12 +880,21 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { (int)(src3->nb[1] / sizeof(float)), (int)(src4->nb[2] / sizeof(float)), (int)(src4->nb[3] / sizeof(float)), (int)(src5->nb[2] / sizeof(float)), (int)(src5->nb[3] / sizeof(float)), - s_off, nc, nr, nh, ng, n_t, n_s); + cache ? cache->data : (float *) ((char *) dst_d + s_off), nc, nr, nh, ng, n_t, n_s); return; } #endif ssm_scan_f32_cuda(src0_d, src1_d, src2_d, src3_d, src4_d, src5_d, src6_d, dst_d, src0->nb[2], src0->nb[3], src1->nb[2], src1->nb[3], src2->nb[1], src2->nb[2], src3->nb[1], src4->nb[2], src4->nb[3], src5->nb[2], src5->nb[3], - s_off, nc, nr, nh, ng, n_t, n_s, K, stream); + s_off, nc, nr, nh, ng, n_t, n_s, K, cache, stream); +} + +void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + ggml_cuda_op_ssm_scan_impl(ctx, dst, nullptr); +} + +void ggml_cuda_op_ssm_scan_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_cuda_ssm_scan_fused_cache cache) { + ggml_cuda_op_ssm_scan_impl(ctx, dst, &cache); } diff --git a/ggml/src/ggml-cuda/ssm-scan.cuh b/ggml/src/ggml-cuda/ssm-scan.cuh index ee078f5eb..407a4c712 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cuh +++ b/ggml/src/ggml-cuda/ssm-scan.cuh @@ -1,3 +1,13 @@ #include "common.cuh" +// fused-kernel recurrent-state output; strides in elements (per-seq stride is always the state row size, set in-kernel) +struct ggml_cuda_ssm_scan_fused_cache { + float * data; // rollback slot 0 + int64_t slot_stride; // between rollback slots +}; + void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// same op, but writes the state snapshot(s) into the cache instead of dst (see ggml_cuda_try_ssm_scan_cache_fusion) +void ggml_cuda_op_ssm_scan_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_cuda_ssm_scan_fused_cache cache); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu index 6ae77bec8..3c861c2c0 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 1, 8); DECL_FATTN_MMA_F16_CASE(64, 64, 1, 8); +DECL_FATTN_MMA_F16_CASE(72, 72, 1, 8); DECL_FATTN_MMA_F16_CASE(80, 80, 1, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 1, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 1, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu index 9d3cfd8ed..865015364 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 16, 1); DECL_FATTN_MMA_F16_CASE(64, 64, 16, 1); +DECL_FATTN_MMA_F16_CASE(72, 72, 16, 1); DECL_FATTN_MMA_F16_CASE(80, 80, 16, 1); DECL_FATTN_MMA_F16_CASE(96, 96, 16, 1); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu index 0c1a37fd0..8cf304825 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 16, 2); DECL_FATTN_MMA_F16_CASE(64, 64, 16, 2); +DECL_FATTN_MMA_F16_CASE(72, 72, 16, 2); DECL_FATTN_MMA_F16_CASE(80, 80, 16, 2); DECL_FATTN_MMA_F16_CASE(96, 96, 16, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu index d2415bfa9..51c91c204 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 16, 4); DECL_FATTN_MMA_F16_CASE(64, 64, 16, 4); +DECL_FATTN_MMA_F16_CASE(72, 72, 16, 4); DECL_FATTN_MMA_F16_CASE(80, 80, 16, 4); DECL_FATTN_MMA_F16_CASE(96, 96, 16, 4); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu index 8eec1d74e..b6f99eb72 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 2, 4); DECL_FATTN_MMA_F16_CASE(64, 64, 2, 4); +DECL_FATTN_MMA_F16_CASE(72, 72, 2, 4); DECL_FATTN_MMA_F16_CASE(80, 80, 2, 4); DECL_FATTN_MMA_F16_CASE(96, 96, 2, 4); DECL_FATTN_MMA_F16_CASE(112, 112, 2, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu index 9f4bef11a..14c120243 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 2, 8); DECL_FATTN_MMA_F16_CASE(64, 64, 2, 8); +DECL_FATTN_MMA_F16_CASE(72, 72, 2, 8); DECL_FATTN_MMA_F16_CASE(80, 80, 2, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 2, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 2, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu index 0543532ea..dbc58be55 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 32, 1); DECL_FATTN_MMA_F16_CASE(64, 64, 32, 1); +DECL_FATTN_MMA_F16_CASE(72, 72, 32, 1); DECL_FATTN_MMA_F16_CASE(80, 80, 32, 1); DECL_FATTN_MMA_F16_CASE(96, 96, 32, 1); DECL_FATTN_MMA_F16_CASE(112, 112, 32, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu index d5a4d160a..784a1f919 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 32, 2); DECL_FATTN_MMA_F16_CASE(64, 64, 32, 2); +DECL_FATTN_MMA_F16_CASE(72, 72, 32, 2); DECL_FATTN_MMA_F16_CASE(80, 80, 32, 2); DECL_FATTN_MMA_F16_CASE(96, 96, 32, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 32, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu index d8e7588cf..7ace51905 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 4, 2); DECL_FATTN_MMA_F16_CASE(64, 64, 4, 2); +DECL_FATTN_MMA_F16_CASE(72, 72, 4, 2); DECL_FATTN_MMA_F16_CASE(80, 80, 4, 2); DECL_FATTN_MMA_F16_CASE(96, 96, 4, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu index 3475dfea0..f27640e84 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 4, 4); DECL_FATTN_MMA_F16_CASE(64, 64, 4, 4); +DECL_FATTN_MMA_F16_CASE(72, 72, 4, 4); DECL_FATTN_MMA_F16_CASE(80, 80, 4, 4); DECL_FATTN_MMA_F16_CASE(96, 96, 4, 4); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu index 859bea5c5..28a961b6b 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 4, 8); DECL_FATTN_MMA_F16_CASE(64, 64, 4, 8); +DECL_FATTN_MMA_F16_CASE(72, 72, 4, 8); DECL_FATTN_MMA_F16_CASE(80, 80, 4, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 4, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu index 0d7a9c728..1ad67c0d8 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 64, 1); DECL_FATTN_MMA_F16_CASE(64, 64, 64, 1); +DECL_FATTN_MMA_F16_CASE(72, 72, 64, 1); DECL_FATTN_MMA_F16_CASE(80, 80, 64, 1); DECL_FATTN_MMA_F16_CASE(96, 96, 64, 1); DECL_FATTN_MMA_F16_CASE(112, 112, 64, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu index 9d5a9976f..695782ca4 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 8, 1); DECL_FATTN_MMA_F16_CASE(64, 64, 8, 1); +DECL_FATTN_MMA_F16_CASE(72, 72, 8, 1); DECL_FATTN_MMA_F16_CASE(80, 80, 8, 1); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 1); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu index e48e28c4e..de2659815 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 8, 2); DECL_FATTN_MMA_F16_CASE(64, 64, 8, 2); +DECL_FATTN_MMA_F16_CASE(72, 72, 8, 2); DECL_FATTN_MMA_F16_CASE(80, 80, 8, 2); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu index 684cd25ce..9d9c748ae 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 8, 4); DECL_FATTN_MMA_F16_CASE(64, 64, 8, 4); +DECL_FATTN_MMA_F16_CASE(72, 72, 8, 4); DECL_FATTN_MMA_F16_CASE(80, 80, 8, 4); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 4); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu index c975ce6b9..50f17f2f8 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 8, 8); DECL_FATTN_MMA_F16_CASE(64, 64, 8, 8); +DECL_FATTN_MMA_F16_CASE(72, 72, 8, 8); DECL_FATTN_MMA_F16_CASE(80, 80, 8, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu new file mode 100644 index 000000000..ae9672a9e --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu new file mode 100644 index 000000000..2afaefa83 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu new file mode 100644 index 000000000..9dfb9c45e --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu new file mode 100644 index 000000000..9e7570d36 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_F16); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_F16); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_F16); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu new file mode 100644 index 000000000..6c380e54c --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu new file mode 100644 index 000000000..fe22bb964 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu new file mode 100644 index 000000000..317273afc --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu new file mode 100644 index 000000000..e05b1ee38 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu new file mode 100644 index 000000000..6cc638687 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu new file mode 100644 index 000000000..0a88e240b --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu new file mode 100644 index 000000000..44b4d4fc9 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu new file mode 100644 index 000000000..f16490953 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu new file mode 100644 index 000000000..6dab7b408 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu new file mode 100644 index 000000000..2cdaa289f --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu new file mode 100644 index 000000000..2556c8c82 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index 7be409bd3..f49a015ae 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -8,7 +8,7 @@ # DKQ -> DV override for asymmetric head dims. HEAD_SIZES_V_OVERRIDE = {576: 512, 320: 256, 192: 128} -TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_BF16"] +TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_BF16", "GGML_TYPE_F8_E4M3"] SOURCE_FATTN_TILE = """// This file has been autogenerated by generate_cu_files.py, do not edit manually. @@ -90,10 +90,6 @@ def get_short_name(long_quant_name): f.write(SOURCE_FATTN_MMA_START) for head_size_kq in HEAD_SIZES_KQ: - if head_size_kq == 40: - continue - if head_size_kq == 72: - continue # Skip compilation of unused ncols2 values for niche head sizes: if head_size_kq == 192 and ncols2 not in (8, 16): # MiMo-V2.5 continue diff --git a/ggml/src/ggml-cuda/top-k-mask.cu b/ggml/src/ggml-cuda/top-k-mask.cu new file mode 100644 index 000000000..9ff995240 --- /dev/null +++ b/ggml/src/ggml-cuda/top-k-mask.cu @@ -0,0 +1,141 @@ +#include "top-k-mask.cuh" +#include "convert.cuh" + +#define CUDA_TOP_K_MASK_BLOCK_SIZE 512 + +// one block per row: fill the row with -inf, then copy the mask cells of the live picks and the extra cells +template <typename T> +static __global__ void top_k_mask_kernel( + const T * mask, const int32_t * top_k, const float * score, const int32_t * blocks, const int32_t * extra, T * dst, + const int64_t n_kv, const int64_t ne1, const int64_t ne2, + const int64_t m1, const int64_t m2, const int64_t m3, + const int64_t n_top, const int64_t n_pick, const int64_t n_blk, const int64_t n_extra, + const int64_t sm1, const int64_t sm2, const int64_t sm3, + const int64_t sd1, const int64_t sd2, const int64_t sd3, + const int64_t sk0, const int64_t sk1, const int64_t sk2, const int64_t sk3, + const int64_t ss0, const int64_t ss1, const int64_t ss2, const int64_t ss3, + const int64_t sb0, const int64_t sb1, + const int64_t se0, const int64_t se1, const int64_t se2, const int64_t se3, + const bool vec) { + ggml_cuda_pdl_lc(); + + const int64_t ir = blockIdx.x; + const int64_t i1 = ir % ne1; + const int64_t i2 = (ir / ne1) % ne2; + const int64_t i3 = ir / (ne1*ne2); + + const T * mask_row = mask + (i1 % m1)*sm1 + (i2 % m2)*sm2 + (i3 % m3)*sm3; + T * dst_row = dst + i1*sd1 + i2*sd2 + i3*sd3; + const int32_t * top_k_row = top_k + i1*sk1 + i2*sk2 + i3*sk3; + + ggml_cuda_pdl_sync(); + + const T ninf = ggml_cuda_cast<T>(-INFINITY); + if (vec) { + constexpr int nv = sizeof(int4)/sizeof(T); + T fill[nv]; +#pragma unroll + for (int j = 0; j < nv; ++j) { + fill[j] = ninf; + } + const int4 fill4 = *(const int4 *) fill; + for (int64_t i = threadIdx.x; i < n_kv/nv; i += blockDim.x) { + ((int4 *) dst_row)[i] = fill4; + } + } else { + for (int64_t i = threadIdx.x; i < n_kv; i += blockDim.x) { + dst_row[i] = ninf; + } + } + + __syncthreads(); + + // one thread per pick or extra cell; cells kept twice get the same value + for (int64_t i = threadIdx.x; i < n_top + n_extra; i += blockDim.x) { + if (i >= n_top) { + const int32_t c = extra[(i - n_top)*se0 + i1*se1 + i2*se2 + i3*se3]; + if (c >= 0 && c < n_kv) { + dst_row[c] = mask_row[c]; + } + continue; + } + + const int32_t p = top_k_row[i*sk0]; + if (p < 0 || p >= n_pick) { + continue; + } + if (score && !(score[p*ss0 + i1*ss1 + i2*ss2 + i3*ss3] > -INFINITY)) { + continue; + } + if (!blocks) { + if (p < n_kv) { + dst_row[p] = mask_row[p]; + } + continue; + } + for (int64_t k = 0; k < n_blk; ++k) { + const int32_t c = blocks[k*sb0 + p*sb1]; + if (c >= 0 && c < n_kv) { + dst_row[c] = mask_row[c]; + } + } + } +} + +template <typename T> +static void top_k_mask_cuda(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * mask = dst->src[0]; + const ggml_tensor * top_k = dst->src[1]; + const ggml_tensor * score = dst->src[2]; + const ggml_tensor * blocks = dst->src[3]; + const ggml_tensor * extra = dst->src[4]; + + const int64_t n_kv = mask->ne[0]; + + GGML_ASSERT(dst->nb[0] == sizeof(T) && mask->nb[0] == sizeof(T)); + GGML_ASSERT(dst->nb[1] % sizeof(T) == 0 && mask->nb[1] % sizeof(T) == 0); + + // 16-byte stores need every dst row aligned + const bool vec = (n_kv*sizeof(T)) % sizeof(int4) == 0 && + dst->nb[1] % sizeof(int4) == 0 && dst->nb[2] % sizeof(int4) == 0 && dst->nb[3] % sizeof(int4) == 0 && + (uintptr_t) dst->data % sizeof(int4) == 0; + + const int64_t n_pick = blocks ? blocks->ne[1] : score ? score->ne[0] : n_kv; + + const size_t si = sizeof(int32_t); + const size_t sf = sizeof(float); + + const dim3 block_dims(CUDA_TOP_K_MASK_BLOCK_SIZE, 1, 1); + const dim3 grid_dims(ggml_nrows(dst), 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream()); + + ggml_cuda_kernel_launch(top_k_mask_kernel<T>, launch_params, + (const T *) mask->data, (const int32_t *) top_k->data, + score ? (const float *) score->data : nullptr, + blocks ? (const int32_t *) blocks->data : nullptr, + extra ? (const int32_t *) extra->data : nullptr, + (T *) dst->data, + n_kv, dst->ne[1], dst->ne[2], + mask->ne[1], mask->ne[2], mask->ne[3], + top_k->ne[0], n_pick, blocks ? blocks->ne[0] : 1, extra ? extra->ne[0] : 0, + mask->nb[1]/sizeof(T), mask->nb[2]/sizeof(T), mask->nb[3]/sizeof(T), + dst->nb[1]/sizeof(T), dst->nb[2]/sizeof(T), dst->nb[3]/sizeof(T), + top_k->nb[0]/si, top_k->nb[1]/si, top_k->nb[2]/si, top_k->nb[3]/si, + score ? score->nb[0]/sf : 0, score ? score->nb[1]/sf : 0, score ? score->nb[2]/sf : 0, score ? score->nb[3]/sf : 0, + blocks ? blocks->nb[0]/si : 0, blocks ? blocks->nb[1]/si : 0, + extra ? extra->nb[0]/si : 0, extra ? extra->nb[1]/si : 0, extra ? extra->nb[2]/si : 0, extra ? extra->nb[3]/si : 0, + vec); +} + +void ggml_cuda_op_top_k_mask(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + switch (dst->type) { + case GGML_TYPE_F16: + top_k_mask_cuda<half>(ctx, dst); + break; + case GGML_TYPE_F32: + top_k_mask_cuda<float>(ctx, dst); + break; + default: + GGML_ABORT("unsupported type"); + } +} diff --git a/ggml/src/ggml-cuda/top-k-mask.cuh b/ggml/src/ggml-cuda/top-k-mask.cuh new file mode 100644 index 000000000..930d6667b --- /dev/null +++ b/ggml/src/ggml-cuda/top-k-mask.cuh @@ -0,0 +1,3 @@ +#include "common.cuh" + +void ggml_cuda_op_top_k_mask(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/top-k.cu b/ggml/src/ggml-cuda/top-k.cu index 3ffbba839..19cb45d1f 100644 --- a/ggml/src/ggml-cuda/top-k.cu +++ b/ggml/src/ggml-cuda/top-k.cu @@ -1,6 +1,29 @@ #include "argsort.cuh" #include "top-k.cuh" +// Adjusted implementation thresholds from #28547, can be overridden at build time +#ifndef GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC +# if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) +// not measured on HIP/MUSA, keep the old split +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC 1024 +# else +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC 512 +# endif +#endif // GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC + +#ifndef GGML_CUDA_TOP_K_NCOLS_THRESHOLD_ARGSORT +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_ARGSORT 4096 +#endif // GGML_CUDA_TOP_K_NCOLS_THRESHOLD_ARGSORT + +// bitonic up to this width while nrows fits in one wave of SMs, 0 disables +#ifndef GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS +# if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS 0 +# else +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS 1024 +# endif +#endif // GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS + #ifdef GGML_CUDA_USE_CUB # include <cub/cub.cuh> // DeviceTopK has a race condition before CCCL 3.4.3. @@ -14,6 +37,15 @@ using namespace cub; # endif // CCCL >= 3.4.3 #endif // GGML_CUDA_USE_CUB +// max rows for the per-row DeviceTopK / CUB argsort path before switching to radix / bitonic +#ifndef GGML_CUDA_TOP_K_NROWS_THRESHOLD +# ifdef CUB_TOP_K_AVAILABLE +# define GGML_CUDA_TOP_K_NROWS_THRESHOLD 2 +# else +# define GGML_CUDA_TOP_K_NROWS_THRESHOLD 1 +# endif +#endif // GGML_CUDA_TOP_K_NROWS_THRESHOLD + #ifdef CUB_TOP_K_AVAILABLE static void top_k_cub(ggml_cuda_pool & pool, @@ -40,7 +72,7 @@ static void top_k_cub(ggml_cuda_pool & pool, ncols, k, env)); } -#elif defined(GGML_CUDA_USE_CUB) // CUB_TOP_K_AVAILABLE +#endif // CUB_TOP_K_AVAILABLE static int next_power_of_2(int x) { int n = 1; @@ -50,28 +82,24 @@ static int next_power_of_2(int x) { return n; } -#endif // CUB_TOP_K_AVAILABLE - -#if !defined(GGML_CUDA_USE_CUB) && defined(GGML_USE_HIP) - static __device__ __forceinline__ uint32_t top_k_float_to_ordered(float value) { const uint32_t bits = __float_as_uint(value); const uint32_t mask = (uint32_t) (-(int32_t) (bits >> 31)) | 0x80000000U; return bits ^ mask; } -struct top_k_radix_state { +// one state per cache line, so the gather atomics of different rows do not compete +struct alignas(128) top_k_radix_state { uint32_t prefix; uint32_t prefix_mask; int rank; int greater_count; - int equal_count; }; static __global__ void top_k_radix_init(top_k_radix_state * states, int nrows, int k) { const int row = blockIdx.x * blockDim.x + threadIdx.x; if (row < nrows) { - states[row] = {0, 0, k, 0, 0}; + states[row] = {0, 0, k, 0}; } } @@ -81,6 +109,7 @@ static __global__ void top_k_radix_histogram( const top_k_radix_state * __restrict__ states, int * __restrict__ block_histograms, int ncols, + int block_cols, int blocks_per_row, int shift) { constexpr int NBINS = 1 << RADIX_BITS; @@ -89,15 +118,15 @@ static __global__ void top_k_radix_histogram( const int row_block = blockIdx.x % blocks_per_row; const int tid = threadIdx.x; const float * row_src = src + (size_t) row * ncols; + const int64_t col_begin = (int64_t) row_block * block_cols; + const int64_t col_end = min(col_begin + block_cols, (int64_t) ncols); __shared__ int histogram[NBINS]; histogram[tid] = 0; __syncthreads(); const top_k_radix_state state = states[row]; - for (int col = row_block * BLOCK_SIZE + tid; - col < ncols; - col += blocks_per_row * BLOCK_SIZE) { + for (int64_t col = col_begin + tid; col < col_end; col += BLOCK_SIZE) { const uint32_t key = top_k_float_to_ordered(row_src[col]); if ((key & state.prefix_mask) == state.prefix) { atomicAdd(&histogram[(key >> shift) & (NBINS - 1)], 1); @@ -142,77 +171,161 @@ static __global__ void top_k_radix_select( } } -static __global__ void top_k_radix_reset_counters(top_k_radix_state * states, int nrows) { - const int row = blockIdx.x * blockDim.x + threadIdx.x; - if (row < nrows) { - states[row].greater_count = 0; - states[row].equal_count = 0; - } -} - -template<int BLOCK_SIZE> +template<int BLOCK_SIZE, int RADIX_BITS> static __global__ void top_k_radix_gather( const float * __restrict__ src, int * __restrict__ dst, + const int * __restrict__ block_histograms, top_k_radix_state * __restrict__ states, int ncols, + int block_cols, int k, int blocks_per_row) { + constexpr int NBINS = 1 << RADIX_BITS; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = BLOCK_SIZE / warp_size; + using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>; + // HIP requires a 64-bit mask also for a 32-wide warp + constexpr uint64_t ballot_mask = ~(lane_mask_t) 0; + const auto popc = [](const lane_mask_t mask) { + if constexpr (warp_size == 64) { + return __popcll(mask); + } else { + return __popc(mask); + } + }; + __shared__ int warp_ties[nwarps]; + const int row = blockIdx.x / blocks_per_row; const int row_block = blockIdx.x % blocks_per_row; - const int tid = threadIdx.x; + const int lane = threadIdx.x % warp_size; + const int warp = threadIdx.x / warp_size; const float * row_src = src + (size_t) row * ncols; int * row_dst = dst + (size_t) row * k; top_k_radix_state * state = &states[row]; + const uint32_t threshold = state->prefix; + const int rank = state->rank; - for (int col = row_block * BLOCK_SIZE + tid; - col < ncols; - col += blocks_per_row * BLOCK_SIZE) { + // each warp reads a contiguous part of the contiguous range of the block + const int warp_cols = block_cols / nwarps; + const int64_t warp_begin = (int64_t) row_block * block_cols + warp * warp_cols; + const int64_t warp_end = min(warp_begin + warp_cols, (int64_t) ncols); + + int nties = 0; + for (int64_t col = warp_begin + lane; col < warp_end; col += warp_size) { const uint32_t key = top_k_float_to_ordered(row_src[col]); - if (key > state->prefix) { + if (key > threshold) { const int pos = atomicAdd(&state->greater_count, 1); row_dst[pos] = col; - } else if (key == state->prefix) { - const int pos = atomicAdd(&state->equal_count, 1); - if (pos < state->rank) { - row_dst[k - state->rank + pos] = col; - } + } else if (key == threshold) { + nties++; + } + } + + // the ties fill the last rank slots in column order, the lowest columns win. + // the histograms of the last pass count the ties of each block in the bin of the threshold. + const int * block_ties = block_histograms + (size_t) row * blocks_per_row * NBINS + (threshold & (NBINS - 1)); + int pos = 0; + for (int b = lane; b < row_block; b += warp_size) { + pos += block_ties[b * NBINS]; + } + pos = warp_reduce_sum<warp_size>(pos); + if (pos >= rank || block_ties[row_block * NBINS] == 0) { + return; // the same result in all threads of the block + } + + nties = warp_reduce_sum<warp_size>(nties); + if (lane == 0) { + warp_ties[warp] = nties; + } + __syncthreads(); + for (int w = 0; w < warp; ++w) { + pos += warp_ties[w]; + } + + // pos is the same in all lanes of the warp, so the lanes leave the loop together + const int pos_end = min(pos + nties, rank); + for (int64_t tile = warp_begin; tile < warp_end && pos < pos_end; tile += warp_size) { + const int64_t col = tile + lane; + const bool is_tie = col < warp_end && top_k_float_to_ordered(row_src[col]) == threshold; + + const lane_mask_t ties = (lane_mask_t) __ballot_sync(ballot_mask, is_tie); + const int tie_pos = pos + popc(ties & (((lane_mask_t) 1 << lane) - 1)); + if (is_tie && tie_pos < pos_end) { + row_dst[k - rank + tie_pos] = col; } + pos += popc(ties); } } static void top_k_radix_cuda( ggml_cuda_pool & pool, - const float * src, int * dst, int ncols, int nrows, int k, cudaStream_t stream) { + const float * src, int * dst, int ncols, int64_t nrows, int k, cudaStream_t stream) { constexpr int BLOCK_SIZE = 256; constexpr int RADIX_BITS = 8; constexpr int NBINS = 1 << RADIX_BITS; - const int blocks_per_row = std::min((ncols + 1023) / 1024, 64); + const int blocks_per_row = (int) std::min<int64_t>(((int64_t) ncols + 1023) / 1024, 64); + // each block of a row reads a contiguous range of whole tiles + const int block_cols = (int) (((int64_t) ncols + blocks_per_row * BLOCK_SIZE - 1) / (blocks_per_row * BLOCK_SIZE) * BLOCK_SIZE); - ggml_cuda_pool_alloc<top_k_radix_state> states_alloc(pool, nrows); - ggml_cuda_pool_alloc<int> histograms_alloc(pool, (size_t) nrows * blocks_per_row * NBINS); + // chunk the rows to bound the histogram memory to 64 MB + const int64_t chunk_nrows = ggml_cuda_chunk_nrows((size_t) blocks_per_row * NBINS * sizeof(int), nrows); + + ggml_cuda_pool_alloc<top_k_radix_state> states_alloc(pool, chunk_nrows); + ggml_cuda_pool_alloc<int> histograms_alloc(pool, (size_t) chunk_nrows * blocks_per_row * NBINS); top_k_radix_state * states = states_alloc.get(); int * histograms = histograms_alloc.get(); - top_k_radix_init<<<(nrows + BLOCK_SIZE - 1) / BLOCK_SIZE, BLOCK_SIZE, 0, stream>>>(states, nrows, k); + for (int64_t i = 0; i < nrows; i += chunk_nrows) { + const int iter_nrows = std::min(chunk_nrows, nrows - i); - const dim3 row_grid(blocks_per_row * nrows); - for (int shift = 32 - RADIX_BITS; shift >= 0; shift -= RADIX_BITS) { - top_k_radix_histogram<BLOCK_SIZE, RADIX_BITS> + top_k_radix_init<<<(iter_nrows + BLOCK_SIZE - 1) / BLOCK_SIZE, BLOCK_SIZE, 0, stream>>>(states, iter_nrows, k); + + const dim3 row_grid(blocks_per_row * iter_nrows); + for (int shift = 32 - RADIX_BITS; shift >= 0; shift -= RADIX_BITS) { + top_k_radix_histogram<BLOCK_SIZE, RADIX_BITS> + <<<row_grid, BLOCK_SIZE, 0, stream>>>( + src, states, histograms, ncols, block_cols, blocks_per_row, shift); + top_k_radix_select<BLOCK_SIZE, RADIX_BITS> + <<<iter_nrows, BLOCK_SIZE, 0, stream>>>(histograms, states, blocks_per_row, shift); + } + + top_k_radix_gather<BLOCK_SIZE, RADIX_BITS> <<<row_grid, BLOCK_SIZE, 0, stream>>>( - src, states, histograms, ncols, blocks_per_row, shift); - top_k_radix_select<BLOCK_SIZE, RADIX_BITS> - <<<nrows, BLOCK_SIZE, 0, stream>>>(histograms, states, blocks_per_row, shift); - } + src, dst, histograms, states, ncols, block_cols, k, blocks_per_row); - top_k_radix_reset_counters - <<<(nrows + BLOCK_SIZE - 1) / BLOCK_SIZE, BLOCK_SIZE, 0, stream>>>(states, nrows); - top_k_radix_gather<BLOCK_SIZE> - <<<row_grid, BLOCK_SIZE, 0, stream>>>( - src, dst, states, ncols, k, blocks_per_row); + src += (size_t) ncols * iter_nrows; + dst += (size_t) k * iter_nrows; + } } -#endif // !defined(GGML_CUDA_USE_CUB) && defined(GGML_USE_HIP) +static void top_k_argsort_cuda( + ggml_cuda_pool & pool, + const float * src, int * dst, int ncols, int64_t nrows, int k, bool use_cub, cudaStream_t stream) { + const int64_t chunk_nrows = ggml_cuda_chunk_nrows((size_t) ncols * sizeof(int), nrows); + + ggml_cuda_pool_alloc<int> tmp_alloc(pool, (size_t) ncols * chunk_nrows); + int * tmp = tmp_alloc.get(); + + for (int64_t i = 0; i < nrows; i += chunk_nrows) { + const int iter_nrows = std::min(chunk_nrows, nrows - i); + + if (use_cub) { +#ifdef GGML_CUDA_USE_CUB + argsort_f32_i32_cuda_cub(pool, src, tmp, ncols, iter_nrows, GGML_SORT_ORDER_DESC, stream); +#else + GGML_ABORT("CUB is not available"); +#endif // GGML_CUDA_USE_CUB + } else { + argsort_f32_i32_cuda_bitonic(src, tmp, ncols, iter_nrows, GGML_SORT_ORDER_DESC, stream); + } + CUDA_CHECK(cudaMemcpy2DAsync(dst, k * sizeof(int), tmp, ncols * sizeof(int), k * sizeof(int), iter_nrows, + cudaMemcpyDeviceToDevice, stream)); + + src += (size_t) ncols * iter_nrows; + dst += (size_t) k * iter_nrows; + } +} void ggml_cuda_op_top_k(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; @@ -229,51 +342,45 @@ void ggml_cuda_op_top_k(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int64_t nrows = ggml_nrows(src0); const int64_t k = dst->ne[0]; ggml_cuda_pool & pool = ctx.pool(); -#ifdef CUB_TOP_K_AVAILABLE - // TODO: Switch to `DeviceSegmentedTopK` for multi-row TopK once implemented - // https://github.com/NVIDIA/cccl/issues/6391 - // TODO: investigate if there exists a point where parallelized argsort is faster than sequential top-k - for (int i = 0; i < nrows; i++) { - top_k_cub(pool, src0_d + i * ncols, dst_d + i * k, ncols, k, stream); - } -#elif defined(GGML_CUDA_USE_CUB) // CUB_TOP_K_AVAILABLE - // Fall back to argsort + copy - const int ncols_pad = next_power_of_2(ncols); - const size_t shared_mem = ncols_pad * sizeof(int); - const size_t max_shared_mem = ggml_cuda_info().devices[ggml_cuda_get_device()].smpb; - const bool use_bitonic = shared_mem <= max_shared_mem && ncols <= 1024; - const int chunk_nrows = argsort_f32_i32_cuda_cub_chunk_nrows(src0->nb[1], nrows); - ggml_cuda_pool_alloc<int> temp_dst_alloc(pool, ncols * chunk_nrows); - int * tmp_dst = temp_dst_alloc.get(); + const int device = ggml_cuda_get_device(); - for (int64_t i = 0; i < nrows; i += chunk_nrows) { - int iter_nrows = std::min((int64_t) chunk_nrows, nrows - i); - - if (use_bitonic) { - argsort_f32_i32_cuda_bitonic(src0_d, tmp_dst, ncols, iter_nrows, GGML_SORT_ORDER_DESC, stream); - } else { - argsort_f32_i32_cuda_cub(pool, src0_d, tmp_dst, ncols, iter_nrows, GGML_SORT_ORDER_DESC, stream); +#ifdef CUB_TOP_K_AVAILABLE + // a single row always uses DeviceTopK if available + const bool bitonic_short = nrows > 1 && ncols <= GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC; +#else + const bool bitonic_short = ncols <= GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC; +#endif // CUB_TOP_K_AVAILABLE + const bool bitonic_few_rows = nrows > GGML_CUDA_TOP_K_NROWS_THRESHOLD && + ncols <= GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS && + nrows <= ggml_cuda_info().devices[device].nsm; + + if (bitonic_short || bitonic_few_rows) { + // the padded row must fit in shared memory + const int ncols_pad = next_power_of_2(ncols); + if (ncols_pad * sizeof(int) <= ggml_cuda_info().devices[device].smpb) { + top_k_argsort_cuda(pool, src0_d, dst_d, ncols, nrows, k, false, stream); + return; } - CUDA_CHECK(cudaMemcpy2DAsync(dst_d, k * sizeof(int), tmp_dst, ncols * sizeof(int), k * sizeof(int), iter_nrows, - cudaMemcpyDeviceToDevice, stream)); - - src0_d += ncols * iter_nrows; - dst_d += k * iter_nrows; } -#else // GGML_CUDA_USE_CUB -#if defined(GGML_USE_HIP) - if (ncols > 1024) { + + if (nrows > GGML_CUDA_TOP_K_NROWS_THRESHOLD) { top_k_radix_cuda(pool, src0_d, dst_d, ncols, nrows, k, stream); + return; + } + +#ifdef CUB_TOP_K_AVAILABLE + // TODO: Assess perf of `DeviceBatchedTopK` for multi-row TopK & CCCL >= 3.5.0, re-running perf sweep of https://github.com/ggml-org/llama.cpp/pull/28713 + for (int64_t i = 0; i < nrows; i++) { + top_k_cub(pool, src0_d + i * ncols, dst_d + i * k, ncols, k, stream); + } +#elif defined(GGML_CUDA_USE_CUB) // CUB_TOP_K_AVAILABLE + if (ncols <= GGML_CUDA_TOP_K_NCOLS_THRESHOLD_ARGSORT) { + top_k_argsort_cuda(pool, src0_d, dst_d, ncols, nrows, k, true, stream); } else { -#endif // defined(GGML_USE_HIP) - ggml_cuda_pool_alloc<int> temp_dst_alloc(pool, ncols * nrows); - int * tmp_dst = temp_dst_alloc.get(); - argsort_f32_i32_cuda_bitonic(src0_d, tmp_dst, ncols, nrows, GGML_SORT_ORDER_DESC, stream); - CUDA_CHECK(cudaMemcpy2DAsync(dst_d, k * sizeof(int), tmp_dst, ncols * sizeof(int), k * sizeof(int), nrows, - cudaMemcpyDeviceToDevice, stream)); -#if defined(GGML_USE_HIP) + top_k_radix_cuda(pool, src0_d, dst_d, ncols, nrows, k, stream); } -#endif // defined(GGML_USE_HIP) -#endif +#else // GGML_CUDA_USE_CUB + top_k_radix_cuda(pool, src0_d, dst_d, ncols, nrows, k, stream); +#endif // CUB_TOP_K_AVAILABLE } diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu index ee903bf21..e99b2e84a 100644 --- a/ggml/src/ggml-cuda/topk-moe.cu +++ b/ggml/src/ggml-cuda/topk-moe.cu @@ -11,6 +11,8 @@ struct topk_moe_config { bool use_sqrt_softplus; bool with_norm; bool delayed_softmax; + int n_groups; + int n_group_used; }; // Warp-local softmax used for both the pre-top-k logits and the post-top-k delayed path. @@ -74,10 +76,57 @@ __device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], co for (int i = 0; i < experts_per_thread; i++) { const int idx = lane + i * WARP_SIZE; const bool active = !use_limit || (idx < limit); - vals[i] = active ? sqrtf(vals[i] > 20.0f ? vals[i] : logf(1.0f + expf(vals[i]))) : -INFINITY; + vals[i] = active ? sqrtf(vals[i] > 20.0f ? vals[i] : log1pf(expf(vals[i]))) : -INFINITY; } } +#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 +// Set -inf for experts outside the top n_group_used groups. Group score is the sum of its top-2 experts (DeepSeek V3). +template <int n_experts, int experts_per_thread> +__device__ void group_mask_warp_inplace(float (&vals)[experts_per_thread], const int n_groups, const int n_group_used, const int lane) { + const int n_per_group = n_experts / n_groups; + + float score = -INFINITY; // lane g holds the score of group g + for (int g = 0; g < n_groups; g++) { + float top1 = -INFINITY; + float top2 = -INFINITY; +#pragma unroll + for (int i = 0; i < experts_per_thread; i++) { + if ((i * WARP_SIZE + lane) / n_per_group == g) { + top2 = fmaxf(top2, fminf(top1, vals[i])); + top1 = fmaxf(top1, vals[i]); + } + } +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { + const float o1 = __shfl_xor_sync(0xFFFFFFFF, top1, offset, WARP_SIZE); + const float o2 = __shfl_xor_sync(0xFFFFFFFF, top2, offset, WARP_SIZE); + top2 = fmaxf(fminf(top1, o1), fmaxf(top2, o2)); + top1 = fmaxf(top1, o1); + } + if (lane == g) { + score = top1 + top2; + } + } + + // rank of each group, lower index wins ties + int rank = 0; + for (int g = 0; g < n_groups; g++) { + const float other = __shfl_sync(0xFFFFFFFF, score, g, WARP_SIZE); + rank += other > score || (other == score && g < lane); + } + const auto keep = __ballot_sync(0xFFFFFFFFULL, lane < n_groups && rank < n_group_used); + +#pragma unroll + for (int i = 0; i < experts_per_thread; i++) { + const int g = (i * WARP_SIZE + lane) / n_per_group; + if (g >= n_groups || !((keep >> g) & 1)) { + vals[i] = -INFINITY; + } + } +} +#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 + /* This kernel does the following: 1. optionally softmax over the logits per token [n_experts, n_tokens] @@ -168,8 +217,22 @@ __global__ void topk_moe_cuda(const float * logits, const int expert = i + threadIdx.x; selection_wt[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? wt[i / WARP_SIZE] + bias[expert] : -INFINITY; + if (__isnanf(selection_wt[i / WARP_SIZE])) { + selection_wt[i / WARP_SIZE] = -FLT_MAX; + } + } + } + +#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 + // without bias, masking wt is safe: the matcher ensures the kept groups hold all used experts + if (config.n_groups > 0) { + if constexpr (has_bias) { + group_mask_warp_inplace<n_experts>(selection_wt, config.n_groups, config.n_group_used, threadIdx.x); + } else { + group_mask_warp_inplace<n_experts>(wt, config.n_groups, config.n_group_used, threadIdx.x); } } +#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 //at this point, each thread holds either a portion of the softmax distribution //or the raw logits. We do the argmax reduce over n_expert_used, each time marking @@ -297,6 +360,10 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx, cudaStream_t stream = ctx.stream(); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); +#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000 + GGML_ASSERT(config.n_groups == 0 && "expert groups fusion is not supported on this HIP version"); +#endif + switch (n_expert) { case 1: ggml_cuda_kernel_launch(topk_moe_cuda<1, has_bias>, launch_params, @@ -338,6 +405,10 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx, ggml_cuda_kernel_launch(topk_moe_cuda<288, has_bias>, launch_params, logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; + case 384: // DeepSeek V4.1 + ggml_cuda_kernel_launch(topk_moe_cuda<384, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); + break; case 512: ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params, logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); @@ -348,7 +419,6 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx, break; default: GGML_ASSERT(false && "fatal error"); - break; } } @@ -390,6 +460,8 @@ void ggml_cuda_op_topk_moe(ggml_backend_cuda_context & ctx, config.use_sqrt_softplus = args.sqrt_softplus; config.with_norm = with_norm; config.delayed_softmax = args.delayed_softmax; + config.n_groups = args.n_groups; + config.n_group_used = args.n_group_used; if (bias) { launch_topk_moe_cuda<true>(ctx, logits_d, weights_d, ids_d, bias_d, n_rows, n_experts, n_expert_used, clamp_val, @@ -407,7 +479,7 @@ bool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op, // must match an instantiation of launch_topk_moe_cuda: a power of 2 up to 512, // or one of the non-power-of-2 expert counts of supported models const int n_expert = ids->nb[1] / ids->nb[0]; - if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 576) { + if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 384 && n_expert != 576) { return false; } diff --git a/ggml/src/ggml-cuda/topk-moe.cuh b/ggml/src/ggml-cuda/topk-moe.cuh index 061b37e29..3637500ff 100644 --- a/ggml/src/ggml-cuda/topk-moe.cuh +++ b/ggml/src/ggml-cuda/topk-moe.cuh @@ -14,6 +14,8 @@ struct ggml_cuda_topk_moe_args { bool prob_bias{}; bool norm{}; bool scale{}; + int n_groups{}; // > 0: pick experts only from the top n_group_used expert groups + int n_group_used{}; }; void ggml_cuda_op_topk_moe(ggml_backend_cuda_context & ctx, diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu index 84788c5dc..b3b6aa48d 100644 --- a/ggml/src/ggml-cuda/unary.cu +++ b/ggml/src/ggml-cuda/unary.cu @@ -91,7 +91,7 @@ static __device__ __forceinline__ float op_expm1(float x) { } static __device__ __forceinline__ float op_softplus(float x) { - return (x > 20.0f) ? x : logf(1.0f + expf(x)); + return (x > 20.0f) ? x : log1pf(expf(x)); } static __device__ __forceinline__ float op_elu(float x) { @@ -107,7 +107,7 @@ static __device__ __forceinline__ float op_ceil(float x) { } static __device__ __forceinline__ float op_round(float x) { - return round(x); + return roundf(x); } static __device__ __forceinline__ float op_trunc(float x) { @@ -134,24 +134,67 @@ static void unary_cuda(const T * x, T * dst, const int k, cudaStream_t stream) { ggml_cuda_kernel_launch(unary_op_kernel<op, T>, launch_params, x, dst, k); } +template <float (*op)(float), typename T> +static __global__ void unary_op_kernel_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03) { + ggml_cuda_pdl_lc(); + const int i = blockDim.x*blockIdx.x + threadIdx.x; + + if (i >= k) { + return; + } + + int64_t rem = i; + const int64_t i0 = rem % ne00; rem /= ne00; + const int64_t i1 = rem % ne01; rem /= ne01; + const int64_t i2 = rem % ne02; + const int64_t i3 = rem / ne02; + const size_t src_byte_offset = i0 * nb00 + i1 * nb01 + i2 * nb02 + i3 * nb03; + const T * src_ptr = (const T *)((const char *)x + src_byte_offset); + + ggml_cuda_pdl_sync(); + dst[i] = ggml_cuda_cast<T>(op(ggml_cuda_cast<float>(*src_ptr))); +} + +template <float (*op)(float), typename T> +static void unary_cuda_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03, cudaStream_t stream) { + const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE; + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_NEG_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(unary_op_kernel_strided<op, T>, launch_params, x, dst, k, ne00,ne01,ne02,nb00,nb01,nb02,nb03); +} + template <float (*op)(float)> void ggml_cuda_op_unary(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const void * src0_d = src0->data; void * dst_d = dst->data; - cudaStream_t stream = ctx.stream(); - GGML_ASSERT(ggml_is_contiguous(src0)); + cudaStream_t stream = ctx.stream(); GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); GGML_ASSERT(src0->type == dst->type); - if (src0->type == GGML_TYPE_F16) { - unary_cuda<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream); - } else if (src0->type == GGML_TYPE_BF16) { - unary_cuda<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream); + if (ggml_is_contiguous(src0)) { + if (src0->type == GGML_TYPE_F16) { + unary_cuda<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream); + } else if (src0->type == GGML_TYPE_BF16) { + unary_cuda<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream); + } else { + unary_cuda<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream); + } } else { - unary_cuda<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream); + if (src0->type == GGML_TYPE_F16) { + unary_cuda_strided<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } else if (src0->type == GGML_TYPE_BF16) { + unary_cuda_strided<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } else { + unary_cuda_strided<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } } } @@ -547,8 +590,8 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); + GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16); GGML_ASSERT(src0->type == dst->type); const float alpha_n = ggml_get_op_params_f32(dst, 1); @@ -558,6 +601,8 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { if (src0->type == GGML_TYPE_F16) { xielu_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream); + } else if (src0->type == GGML_TYPE_BF16) { + xielu_cuda((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream); } else { xielu_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream); } diff --git a/ggml/src/ggml-cuda/unary.cuh b/ggml/src/ggml-cuda/unary.cuh index 04f3af644..c6ffdb9d6 100644 --- a/ggml/src/ggml-cuda/unary.cuh +++ b/ggml/src/ggml-cuda/unary.cuh @@ -121,3 +121,18 @@ __device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, fl return ggml_cuda_op_silu_single(gate) * up; } + +__device__ __forceinline__ float ggml_cuda_op_glu_single(ggml_glu_op op, float gate, float up, float limit) { + switch (op) { + case GGML_GLU_OP_SWIGLU: + return up * ggml_cuda_op_silu_single(gate); + case GGML_GLU_OP_GEGLU: + return up * ggml_cuda_op_gelu_single(gate); + case GGML_GLU_OP_SWIGLU_OAI: + return ggml_cuda_op_swiglu_oai_single(gate, up); + case GGML_GLU_OP_SWIGLU_CLAMP: + return ggml_cuda_op_swiglu_clamp_single(gate, up, limit); + default: + return up * gate; + } +} diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index f65fe530e..d5ff93b91 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -40,18 +40,18 @@ static __device__ __forceinline__ int2 get_int_from_table_16(const int & q4, con const uint32_t q_odd = (q4 >> 4); // Perform lookups in the lower half of the table (indices 0-7). - uint32_t v_even_low = __builtin_amdgcn_perm(values[1], values[0], q_even & 0x07070707); - uint32_t v_odd_low = __builtin_amdgcn_perm(values[1], values[0], q_odd & 0x07070707); + uint32_t v_even_low = ggml_hip_perm(values[1], values[0], q_even & 0x07070707); + uint32_t v_odd_low = ggml_hip_perm(values[1], values[0], q_odd & 0x07070707); // Perform lookups in the upper half of the table (indices 8-15). - uint32_t v_even_high = __builtin_amdgcn_perm(values[3], values[2], q_even & 0x07070707); - uint32_t v_odd_high = __builtin_amdgcn_perm(values[3], values[2], q_odd & 0x07070707); + uint32_t v_even_high = ggml_hip_perm(values[3], values[2], q_even & 0x07070707); + uint32_t v_odd_high = ggml_hip_perm(values[3], values[2], q_odd & 0x07070707); // Select between the low and high results based on the MSB of each index nibble. uint32_t mask_even = 0x03020100 | ((q_even & 0x08080808) >> 1); - uint32_t res_x = __builtin_amdgcn_perm(v_even_high, v_even_low, mask_even); + uint32_t res_x = ggml_hip_perm(v_even_high, v_even_low, mask_even); uint32_t mask_odd = 0x03020100 | ((q_odd & 0x08080808) >> 1); - uint32_t res_y = __builtin_amdgcn_perm(v_odd_high, v_odd_low, mask_odd); + uint32_t res_y = ggml_hip_perm(v_odd_high, v_odd_low, mask_odd); return make_int2(res_x, res_y); #elif !defined(GGML_USE_MUSA) @@ -360,12 +360,12 @@ static __device__ __forceinline__ float vec_dot_nvfp4_q8_1( return sum; } -#define VDR_Q2_K_Q8_1_MMVQ 1 +#define VDR_Q2_K_Q8_1_MMVQ 2 #define VDR_Q2_K_Q8_1_MMQ 4 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq( - const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales, + const int * __restrict__ v, const int * __restrict__ u, const int * __restrict__ scales, const half2 & dm2, const float * __restrict__ d8) { float sumf_d = 0.0f; @@ -373,17 +373,20 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq( #pragma unroll for (int i = 0; i < QR2_K; ++i) { - const int sc = scales[2*i]; + const int sc = scales[i/2] >> (16*(i%2)); - const int vi = (v >> (2*i)) & 0x03030303; + int sumi_d = 0; + int sumi_m = 0; +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + const int vi = (v[j] >> (2*i)) & 0x03030303; - sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product + sumi_d = ggml_cuda_dp4a(vi, u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_d); // SIMD dot product + sumi_m = ggml_cuda_dp4a(0x01010101, u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_m); // sum of q8_1 values + } - // fill int with 4x m - int m = sc >> 4; - m |= m << 8; - m |= m << 16; - sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values + sumf_d += d8[i] * (sumi_d * (sc & 0xF)); + sumf_m += d8[i] * (sumi_m * ((sc >> 4) & 0xF)); // multiply constant q2_K part with sum of q8_1 values } const float2 dm2f = __half22float2(dm2); @@ -443,37 +446,35 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq( return sumf + d8*sumf_d8; } -#define VDR_Q3_K_Q8_1_MMVQ 1 +#define VDR_Q3_K_Q8_1_MMVQ 2 #define VDR_Q3_K_Q8_1_MMQ 2 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq( - const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales, - const int & scale_offset, const float & d3, const float * __restrict__ d8) { + const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int * __restrict__ scales, + const float & d3, const float * __restrict__ d8) { float sumf = 0.0f; #pragma unroll for (int i = 0; i < QR3_K; ++i) { - const int isc = scale_offset + 2*i; - - const int isc_low = isc % (QK_K/32); - const int sc_shift_low = 4 * (isc / (QK_K/32)); - const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF; - - const int isc_high = isc % (QK_K/64); - const int sc_shift_high = 2 * (isc / (QK_K/64)); - const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4; + // 4 low bits from scales[0..7], 2 high bits from scales[8..11] + const int sc_low = (scales[i/2] >> (16*(i%2))) & 0x0F; + const int sc_high = (scales[2] >> (16*(i%2) + 2*(i/2))) & 0x03; + const int sc = (sc_low | (sc_high << 4)) - 32; - const int sc = (sc_low | sc_high) - 32; - - const int vil = (vl >> (2*i)) & 0x03030303; - - const int vih = ((vh >> i) << 2) & 0x04040404; + int sumi = 0; +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + const int vil = (vl[j] >> (2*i)) & 0x03030303; + const int vih = ((vh[j] >> i) << 2) & 0x04040404; - const int vi = __vsub4(vil, vih); + // (vil | vih) - 4 for each byte, the - 4 is done by the second dp4a + sumi = ggml_cuda_dp4a(vil | vih, u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product + sumi = ggml_cuda_dp4a(0xFCFCFCFC, u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi); + } - sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product + sumf += d8[i] * (sumi * sc); } return d3 * sumf; @@ -620,12 +621,12 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq( return dm4f.x*sumf_d - dm4f.y*sumf_m; } -#define VDR_Q6_K_Q8_1_MMVQ 1 +#define VDR_Q6_K_Q8_1_MMVQ 2 #define VDR_Q6_K_Q8_1_MMQ 8 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq( - const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales, + const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int8_t * __restrict__ scales, const float & d, const float * __restrict__ d8) { float sumf = 0.0f; @@ -634,13 +635,18 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq( for (int i = 0; i < QR6_K; ++i) { const int sc = scales[4*i]; - const int vil = (vl >> (4*i)) & 0x0F0F0F0F; - - const int vih = ((vh >> (4*i)) << 4) & 0x30303030; + int sumi = 0; +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + const int vil = (vl[j] >> (4*i)) & 0x0F0F0F0F; + const int vih = ((vh[j] >> (4*i)) << 4) & 0x30303030; - const int vi = __vsub4((vil | vih), 0x20202020); // vi = (vil | vih) - 32 + // (vil | vih) - 32 for each byte, the - 32 is done by the second dp4a + sumi = ggml_cuda_dp4a(vil | vih, u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product + sumi = ggml_cuda_dp4a(0xE0E0E0E0, u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi); + } - sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product + sumf += d8[i] * (sumi * sc); } return d*sumf; @@ -700,6 +706,25 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1( const int u2 = get_int_b4(bq8_1_chunk->qs, j*4+2); const int u3 = get_int_b4(bq8_1_chunk->qs, j*4+3); +#if defined(GGML_USE_HIP) + const uint32_t q_bits = (uint16_t) q; + + // Expand each group of four Q1_0 bits into byte selectors for v_perm_b32. + const uint32_t q0_bits = q_bits >> 0; + const uint32_t q1_bits = q_bits >> 4; + const uint32_t q2_bits = q_bits >> 8; + const uint32_t q3_bits = q_bits >> 12; + + const uint32_t q0_indices = (q0_bits & 0x01) | ((q0_bits & 0x02) << 7) | ((q0_bits & 0x04) << 14) | ((q0_bits & 0x08) << 21); + const uint32_t q1_indices = (q1_bits & 0x01) | ((q1_bits & 0x02) << 7) | ((q1_bits & 0x04) << 14) | ((q1_bits & 0x08) << 21); + const uint32_t q2_indices = (q2_bits & 0x01) | ((q2_bits & 0x02) << 7) | ((q2_bits & 0x04) << 14) | ((q2_bits & 0x08) << 21); + const uint32_t q3_indices = (q3_bits & 0x01) | ((q3_bits & 0x02) << 7) | ((q3_bits & 0x04) << 14) | ((q3_bits & 0x08) << 21); + + const int v0 = ggml_hip_perm(0x000001FF, 0x000001FF, q0_indices); + const int v1 = ggml_hip_perm(0x000001FF, 0x000001FF, q1_indices); + const int v2 = ggml_hip_perm(0x000001FF, 0x000001FF, q2_indices); + const int v3 = ggml_hip_perm(0x000001FF, 0x000001FF, q3_indices); +#else // unpack crumbs into nibble indices const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13] const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15] @@ -713,6 +738,7 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1( const int v1 = __byte_perm(s0, s1, 0x7632); const int v2 = __byte_perm(s2, s3, 0x5410); const int v3 = __byte_perm(s2, s3, 0x7632); +#endif // defined(GGML_USE_HIP) sumi = ggml_cuda_dp4a(v0, u0, sumi); sumi = ggml_cuda_dp4a(v1, u1, sumi); @@ -751,8 +777,8 @@ static __device__ __forceinline__ float vec_dot_q2_0_q8_1( const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18); const uint32_t qy_bits = q >> 8; const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18); - const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices); - const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices); + const int qx = ggml_hip_perm(0x020100FF, 0x020100FF, qx_indices); + const int qy = ggml_hip_perm(0x020100FF, 0x020100FF, qy_indices); #else // unpack even and odd crumbs into byte values const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); @@ -870,18 +896,31 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1( const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx; + // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift) const int bq8_offset = QR2_K * (iqs / QI8_1); - const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + const int iqs8 = iqs % QI8_1; - const uint8_t * scales = bq2_K->scales + scale_offset; + int v[VDR_Q2_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + v[j] = get_int_b4(bq2_K->qs, iqs + j); + } - const int v = get_int_b4(bq2_K->qs, iqs); - int u[QR2_K]; + // scale of shift i is byte 2*i + (iqs8 >= QI8_1/2) of the 8 bytes for this half + const int sc_shift = 8 * (iqs8 / (QI8_1/2)); + int scales[2]; + scales[0] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 0) >> sc_shift; + scales[1] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 1) >> sc_shift; + + int u[VDR_Q2_K_Q8_1_MMVQ*QR2_K]; float d8[QR2_K]; #pragma unroll - for (int i = 0; i < QR2_K; ++ i) { - u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + for (int i = 0; i < QR2_K; ++i) { +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + u[VDR_Q2_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + i].ds); } @@ -893,26 +932,42 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1( const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx; + // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift) const int bq8_offset = QR3_K * (iqs / (QI3_K/2)); - const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + const int iqs8 = iqs % QI8_1; const float d = bq3_K->d; - const int vl = get_int_b2(bq3_K->qs, iqs); + int vl[VDR_Q3_K_Q8_1_MMVQ]; + int vh[VDR_Q3_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + vl[j] = get_int_b2(bq3_K->qs, iqs + j); + vh[j] = get_int_b2(bq3_K->hmask, iqs8 + j) >> bq8_offset; + } - // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted - const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset; + // sub-block 8*(iqs/8) + 2*i + (iqs8 >= QI8_1/2) for shift i: + // 4 low bits in nibble iqs/8 of bytes, 2 high bits at bit 4*(iqs/8) + 2*(i/2) of bytes 8..11 + const int sc_shift = 8 * (iqs8 / (QI8_1/2)) + 4 * (iqs / (QI3_K/2)); + int scales[3]; +#pragma unroll + for (int l = 0; l < 3; ++l) { + scales[l] = get_int_b2(bq3_K->scales, l) >> sc_shift; + } - int u[QR3_K]; + int u[VDR_Q3_K_Q8_1_MMVQ*QR3_K]; float d8[QR3_K]; #pragma unroll for (int i = 0; i < QR3_K; ++i) { - u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + u[VDR_Q3_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + i].ds); } - return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8); + return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, scales, d, d8); } static __device__ __forceinline__ float vec_dot_q4_K_q8_1( @@ -1025,17 +1080,25 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1( const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8); const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4)); - const int vl = get_int_b2(bq6_K->ql, iqs); - const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift; + int vl[VDR_Q6_K_Q8_1_MMVQ]; + int vh[VDR_Q6_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + vl[j] = get_int_b2(bq6_K->ql, iqs + j); + vh[j] = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4) + j) >> vh_shift; + } const int8_t * scales = bq6_K->scales + scale_offset; - int u[QR6_K]; + int u[VDR_Q6_K_Q8_1_MMVQ*QR6_K]; float d8[QR6_K]; #pragma unroll for (int i = 0; i < QR6_K; ++i) { - u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1); +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + u[VDR_Q6_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds); } diff --git a/ggml/src/ggml-cuda/vendors/cuda.h b/ggml/src/ggml-cuda/vendors/cuda.h index 323c98019..86c1ed9ce 100644 --- a/ggml/src/ggml-cuda/vendors/cuda.h +++ b/ggml/src/ggml-cuda/vendors/cuda.h @@ -3,6 +3,7 @@ #include <cuda_runtime.h> #include <cuda.h> #include <cublas_v2.h> +#include <cublasLt.h> #include <cuda_bf16.h> #include <cuda_fp16.h> diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index da7e3da0b..da159dea1 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -43,6 +43,9 @@ #define cublasHandle_t hipblasHandle_t #define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS #define cublasSetStream hipblasSetStream +#if HIP_VERSION >= 70000000 +#define cublasSetWorkspace hipblasSetWorkspace +#endif // HIP_VERSION >= 70000000 #define cublasSgemm hipblasSgemm #define cublasSgemmBatched hipblasSgemmBatched #define cublasSgemmStridedBatched hipblasSgemmStridedBatched @@ -57,7 +60,9 @@ #define cudaDeviceProp hipDeviceProp_t #define cudaDeviceSynchronize hipDeviceSynchronize #define cudaError_t hipError_t +#define CUresult hipError_t #define cudaErrorMemoryAllocation hipErrorOutOfMemory +#define CUDA_ERROR_OUT_OF_MEMORY hipErrorOutOfMemory #define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled #define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled #define cudaEventCreateWithFlags hipEventCreateWithFlags @@ -216,9 +221,13 @@ #define RDNA4 #endif // defined(__GFX12__) -#if defined(__GFX11__) +#if defined(__gfx1170__) || defined(__gfx1171__) +#define RDNA4_M +#endif // defined(__gfx1170__) || defined(__gfx1171__) + +#if defined(__GFX11__) && !defined(RDNA4_M) #define RDNA3 -#endif // defined(__GFX11__) +#endif // defined(__GFX11__) && !defined(RDNA4_M) #if defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__) #define RDNA3_5 @@ -237,9 +246,9 @@ #define RDNA1 #endif // defined(__gfx1010__) || defined(__gfx1012__) -#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1) +#if defined(RDNA4) || defined(RDNA4_M) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1) #define RDNA // For the entire family -#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1) +#endif // defined(RDNA4) || defined(RDNA4_M) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1) #ifndef __has_builtin #define __has_builtin(x) 0 @@ -254,6 +263,10 @@ typedef __hip_fp8_e4m3 __nv_fp8_e4m3; #define FP8_AVAILABLE #endif // HIP_VERSION >= 60300000 +#if HIP_VERSION < 70000000 +#define __ballot_sync(mask, var) __ballot(var) +#endif //HIP_VERSION < 70000000 + typedef int8_t int8x4_t __attribute__((ext_vector_type(4))); typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4))); static __device__ __forceinline__ int __vsubss4(const int a, const int b) { @@ -311,3 +324,22 @@ static __device__ __forceinline__ unsigned int __vcmpne4(unsigned int a, unsigne return (ne_any_bit >> 7) * 0xff; } + +// v_perm_b32 with selector bytes in [0, 7]: 0-3 pick bytes of lo, 4-7 pick bytes of hi. +static __device__ __forceinline__ uint32_t ggml_hip_perm(const uint32_t hi, const uint32_t lo, const uint32_t sel) { +#if __has_builtin(__builtin_amdgcn_is_invocable) + if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_perm)) { + return __builtin_amdgcn_perm(hi, lo, sel); + } + const uint64_t src = ((uint64_t) hi << 32) | lo; + uint32_t res = 0; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const uint32_t byte_sel = (sel >> (8 * i)) & 0x07; + res |= (uint32_t) ((src >> (8 * byte_sel)) & 0xFF) << (8 * i); + } + return res; +#else + return __builtin_amdgcn_perm(hi, lo, sel); +#endif // __has_builtin(__builtin_amdgcn_is_invocable) +} diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 22126decf..ff13fa052 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -129,6 +129,7 @@ // Additional mappings for MUSA graphs #define CUDA_SUCCESS MUSA_SUCCESS +#define CUDA_ERROR_OUT_OF_MEMORY MUSA_ERROR_OUT_OF_MEMORY #define CUresult MUresult #define cuGetErrorString muGetErrorString #define cudaErrorGraphExecUpdateFailure musaErrorGraphExecUpdateFailure diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index 876513867..c51d16f82 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -1523,7 +1523,7 @@ bool ggml_et_op_flash_attn_ext(ggml_backend_et_device_context * dev_ctx, const g } const ggml_prec prec = ggml_flash_attn_ext_get_prec(node); - if (prec != GGML_PREC_F32 && prec != GGML_PREC_DEFAULT) { + if (prec != GGML_PREC_F32 && prec != GGML_PREC_BF16 && prec != GGML_PREC_UNDEFINED) { GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel only supports F32 precision\n"); return false; } diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp index 54a6d166d..53e64b323 100644 --- a/ggml/src/ggml-et/ggml-et.cpp +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -373,6 +373,9 @@ static const struct ggml_backend_buffer_i ggml_backend_et_buffer_i = { /* .cpy_tensor = */ ggml_backend_et_buffer_cpy_tensor, /* .clear = */ ggml_backend_et_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; static const char * ggml_backend_et_buffer_type_get_name(ggml_backend_buffer_type_t buft) { @@ -938,6 +941,9 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm op->nb[1] == op->ne[0] * sizeof(float); break; case GGML_OP_MUL_MAT: + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + break; + } // Support Q8_0 x F32 -> F32, F16 x F32 -> F32, F16 x F16 -> F32, and F32 x F32 -> F32 matrix multiplication // Stride requirements: first dimension must be contiguous for all tensors if (op->type == GGML_TYPE_F32 && @@ -1066,7 +1072,7 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm const bool zero_rot_offset = ggml_get_op_params_i32(op, 15) == 0; supported = - zero_view_offset && zero_rot_offset && ndims <= 512 && + ggml_rope_get_truncate(op) && zero_view_offset && zero_rot_offset && ndims <= 512 && (is_normal || (is_neox && ndims % 16 == 0) || (is_imrope && ndims % 16 == 0 && has_sections)); } else { supported = false; @@ -1270,7 +1276,7 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_FLASH_ATTN_EXT: if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && op->src[2] && - (op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr && + (op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr && op->src[5] == nullptr && ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0])) { float max_bias = 0.0f; float logit_softcap = 0.0f; @@ -1296,7 +1302,7 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm const bool me_eligible = op->src[1]->type == GGML_TYPE_F16 && op->src[2]->type == GGML_TYPE_F16 && (op->src[0]->ne[0] % 32) == 0; - supported = me_eligible && mask_ok && (prec == GGML_PREC_F32 || prec == GGML_PREC_DEFAULT) && + supported = me_eligible && mask_ok && (prec == GGML_PREC_F32 || prec == GGML_PREC_BF16 || prec == GGML_PREC_UNDEFINED) && max_bias == 0.0f && logit_softcap == 0.0f && op->src[0]->nb[0] == sizeof(float) && op->src[1]->nb[0] == k_elem && op->src[2]->nb[0] == v_elem && op->nb[0] == sizeof(float) && op->src[0]->ne[0] == op->src[1]->ne[0] && // dk matches diff --git a/ggml/src/ggml-feats.h b/ggml/src/ggml-feats.h index 79a0afd87..c45e4ad08 100644 --- a/ggml/src/ggml-feats.h +++ b/ggml/src/ggml-feats.h @@ -22,6 +22,10 @@ #define HWCAP2_I8MM (1ULL << 13) #endif +#if !defined(HWCAP2_SVEI8MM) +#define HWCAP2_SVEI8MM (1ULL << 9) +#endif + #if !defined(HWCAP_ASIMDDP) #define HWCAP_ASIMDDP (1 << 20) #endif @@ -63,6 +67,10 @@ #define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47 #endif +#if !defined(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE 57 +#endif + #if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) #define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66 #endif @@ -87,6 +95,7 @@ typedef struct ggml_feats_arch64_runtime { bool has_sve; bool has_sve2; bool has_i8mm; + bool has_svei8mm; bool has_sme; bool has_sme2; int sve_cnt; @@ -104,6 +113,7 @@ static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) { runtime_feat.has_sve = !!(hwcap & HWCAP_SVE); runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2); runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM); + runtime_feat.has_svei8mm = !!(hwcap2 & HWCAP2_SVEI8MM); runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME); runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2); @@ -153,6 +163,7 @@ static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) { runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0; runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0; runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_svei8mm = IsProcessorFeaturePresent(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE) != 0; runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0; runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0; diff --git a/ggml/src/ggml-hexagon/CMakeLists.txt b/ggml/src/ggml-hexagon/CMakeLists.txt index 20c720263..fea5770c1 100644 --- a/ggml/src/ggml-hexagon/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/CMakeLists.txt @@ -21,6 +21,7 @@ message(STATUS "hexagon: using ${HEXAGON_SDK_ROOT} and ${HEXAGON_TOOLS_ROOT} for include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake) include(ExternalProject) +find_package(Python3 REQUIRED COMPONENTS Interpreter) option(GGML_HEXAGON_HTP_DEBUG "ggml-hexagon: enable HTP debug output" OFF) set(GGML_HEXAGON_HTP_BUILD_TYPE "Release" CACHE STRING "ggml-hexagon: HTP skel build type (Release, RelWithDebInfo, Debug)") @@ -32,10 +33,11 @@ add_library(htp_iface OBJECT set_target_properties(htp_iface PROPERTIES POSITION_INDEPENDENT_CODE ON) target_include_directories(htp_iface PUBLIC ${HEXAGON_SDK_ROOT}/incs - ${HEXAGON_SDK_ROOT}/incs/stddef ${HEXAGON_SDK_ROOT}/utils/examples ${CMAKE_CURRENT_SOURCE_DIR}/htp ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(htp_iface PRIVATE + ${HEXAGON_SDK_ROOT}/incs/stddef) build_idl(htp/htp_iface.idl htp_iface) @@ -50,13 +52,28 @@ endif() set(TARGET_NAME ggml-hexagon) ggml_add_backend_library(${TARGET_NAME} ggml-hexagon.cpp + htp-common.h + htp-kparams.cpp + htp-kparams.h + htp-opnode.cpp + htp-opnode.h + htp-fusion.cpp + htp-fusion.h + htp-repack.cpp + htp-repack.h htp-drv.cpp htp-drv.h libdl.h ../../include/ggml-hexagon.h) +if (WIN32) + # some clang-cl drivers do not predefine ARM64 macros for winnt.h + target_compile_definitions(${TARGET_NAME} PRIVATE _M_ARM64=1 _ARM64_=1) +endif() + target_link_libraries(${TARGET_NAME} PRIVATE htp_iface) target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/htp ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${HEXAGON_SDK_ROOT}/incs/stddef) # Build HTP skels set(HTP_SKELS) @@ -72,7 +89,8 @@ function(build_htp_skel V) -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT} -DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG} -DDSP_VERSION=${V} - -DPREBUILT_LIB_DIR="toolv19_${V}") + -DPython3_EXECUTABLE=${Python3_EXECUTABLE} + -DPREBUILT_LIB_DIR="toolv21_${V}") list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so) list(APPEND HTP_PROJECTS htp-${V}) set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE) @@ -83,6 +101,7 @@ build_htp_skel(v73) build_htp_skel(v75) build_htp_skel(v79) build_htp_skel(v81) +build_htp_skel(v85) # Install Hexagon skels required at runtime install(FILES ${HTP_SKELS} TYPE LIB) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 454594472..03da443c8 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -1,3 +1,5 @@ +#define _USE_MATH_DEFINES + #include <assert.h> #include <inttypes.h> #include <stdio.h> @@ -47,25 +49,19 @@ #define GGML_COMMON_IMPL_CPP #include "ggml-backend-impl.h" +#include "ggml-alloc.h" #include "ggml-common.h" #include "ggml-hexagon.h" #include "ggml-impl.h" #include "ggml-quants.h" #include "htp-opnode.h" -#include "htp-ops.h" -#include "htp/matmul-ops.h" -#include "htp/binary-ops.h" -#include "htp/flash-attn-ops.h" -#include "htp/unary-ops.h" -#include "htp/get-rows-ops.h" -#include "htp/set-rows-ops.h" -#include "htp/softmax-ops.h" -#include "htp/rope-ops.h" -#include "htp/ssm-conv.h" -#include "htp/gated-delta-net-ops.h" -#include "htp/argsort-ops.h" +#include "htp/htp-ops.h" #include "htp_iface.h" #include "htp-drv.h" +#include "htp-repack.h" +#include "htp-common.h" +#include "htp-kparams.h" +#include "htp-fusion.h" using intvec = std::vector<int>; using uintvec = std::vector<unsigned int>; @@ -73,8 +69,6 @@ using u32vec = std::vector<uint32_t>; #define GGML_HEXAGON_MAX_SESSIONS 16 -#define GGML_HEXAGON_FENCE_SLOT_SIZE 128 - struct ggml_hexagon_device_config { int physical_idx = 0; int virtual_idx = 0; @@ -87,55 +81,79 @@ struct ggml_hexagon_device_config { static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS]; -static int opt_arch = 0; // autodetect -static size_t opt_ndev = 1; -static size_t opt_nhvx = 0; // use all -static int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only -static size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings -static size_t opt_mbuf = 1ul * 1024 * 1024 * 1024; // max buffer size -static int opt_etm = 0; -static int opt_verbose = 0; -static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu) -static bool opt_hostbuf = false; -static bool opt_dma64 = false; - -static int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) -static int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) -static int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) -static int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE -static int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction +int opt_arch = 0; // autodetect +size_t opt_ndev = 1; +size_t opt_nhvx = 0; // use all +int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only +size_t opt_vtcm_size = 0; // VTCM size override in bytes; 0 = auto-detect +size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings +int opt_etm = 0; +int opt_verbose = 0; +int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu) +bool opt_hostbuf = true; +bool opt_dma64 = false; +int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start + +static const std::pair<const char *, int> htp_power_mode_names[] = { + { "low_power_saver", HTP_POWER_MODE_LOW_POWER_SAVER }, + { "power_saver", HTP_POWER_MODE_POWER_SAVER }, + { "high_power_saver", HTP_POWER_MODE_HIGH_POWER_SAVER }, + { "low_balanced", HTP_POWER_MODE_LOW_BALANCED }, + { "balanced", HTP_POWER_MODE_BALANCED }, + { "high_performance", HTP_POWER_MODE_HIGH_PERFORMANCE }, + { "sustained_high_performance", HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE }, + { "burst", HTP_POWER_MODE_BURST }, + { "default", HTP_POWER_MODE_BURST }, // flip this to change the default mode +}; + +static int htp_power_mode_from_str(const char * s) { + for (auto & e : htp_power_mode_names) { +#if defined(_MSC_VER) + if (_stricmp(s, e.first) == 0) { +#else + if (strcasecmp(s, e.first) == 0) { +#endif + return e.second; + } + } + return -1; +} + +static const char * htp_power_mode_to_str(int mode) { + for (auto & e : htp_power_mode_names) { + if (e.second == mode) { + return e.first; + } + } + return "unknown"; +} + +size_t opt_mbuf_dyn = 512ul * 1024 * 1024; // max dynamic (compute) buffer size +size_t opt_mbuf_static = 1ul * 1024 * 1024 * 1024; // max static (weight/KV) buffer size +size_t opt_mbuf_total = 0; // total buffer space limit (0 = unconstrained) + +int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) +int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) +int opt_fa_split = 1; // 1 = head-split + team-split when n_kv_heads < N_cores (default), 0 = off +int opt_li_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) +int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) +int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE +int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction // Default PMU events, if profiling with PMU (mode=2) is enabled // See https://docs.qualcomm.com/doc/80-N2040-60/topic/pmu-events.html // https://docs.qualcomm.com/doc/80-N2040-61/topic/hvx-pmu-events.html static u32vec opt_pmu_evt { 0x3, 0x111, 0x100, 0x105, 0x240, 0x256, 0x7D, 0x8C }; -static int opt_opbatch = 1280; // max number of ops in a batch -static int opt_opqueue = 32; // max number of pending batches -static int opt_optrace = 0; // trace buffer size per thread (0 means default) -static int opt_oppoll = 0; // polling for batch completions -static int opt_opfusion = 1; // enable/disable op fusion - -enum ggml_hexagon_fusion_flags { - GGML_HEXAGON_FUSE_ALLREDUCE_ADD = (1 << 1), // 2 - GGML_HEXAGON_FUSE_RMS_NORM_MUL = (1 << 2), // 4 - GGML_HEXAGON_FUSE_MUL_MAT_ADD = (1 << 3), // 8 - GGML_HEXAGON_FUSE_MUL_MAT_NX = (1 << 4), // 16 - GGML_HEXAGON_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32 - GGML_HEXAGON_FUSE_GDN_CPY = (1 << 6), // 64 -}; - -static inline bool ggml_hexagon_is_fusion_enabled(int flag) { - if (opt_opfusion <= 0) return false; - if (opt_opfusion == 1) return true; // 1 enables all - return (opt_opfusion & flag) != 0; -} +int opt_opbatch = 2048; // max number of ops in a batch +int opt_opqueue = 8; // max number of pending batches +int opt_optrace = 0; // trace buffer size per thread (0 means default) +int opt_oppoll = 0; // polling for batch completions +int opt_opfusion = 1; // enable/disable op fusion +int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = all tileable weights (default) static std::regex* opt_opfilter = NULL; // regex of ops to not claim -#define HEX_VERBOSE(...) \ - if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__) - static const char * status_to_str(uint32_t status) { switch (status) { case HTP_STATUS_OK: @@ -155,20 +173,18 @@ static const char * status_to_str(uint32_t status) { // ** debug helpers -static void ggml_hexagon_dump_op_exec(const std::string &sess_name, const htp_opnode & node, const uint32_t req_flags) { +static void ggml_hexagon_dump_op_exec(const char * sess_name, const htp_opnode & node, const uint32_t req_flags) { if (!opt_verbose) return; - htp_opformat fmt(node); - GGML_LOG_DEBUG("ggml-hex: %s execute-op %s|%s|%s|%s|%s|%s|%s|flags 0x%x\n", sess_name.c_str(), - node.op_name().c_str(), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.buffs, fmt.kparams, req_flags); + GGML_LOG_DEBUG("ggml-hex: %s execute-op %s|flags 0x%x\n", + sess_name, htp_format_op(node, HTP_OPFMT_EXEC).c_str(), req_flags); } -static void ggml_hexagon_dump_op_supp(const std::string &sess_name, const struct ggml_tensor * op, bool supp) { +static void ggml_hexagon_dump_op_supp(const char * sess_name, const struct ggml_tensor * op, bool supp) { if (!opt_verbose) return; - htp_opformat fmt(htp_opformat(htp_opnode(HTP_OP_INVALID, const_cast<ggml_tensor*>(op)))); - GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s|%s|%s|%s|%s|%s\n", sess_name.c_str(), - ggml_op_desc(op), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.buffs, supp ? "yes" : "no"); + GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s\n", + sess_name, htp_format_op(op, HTP_OPFMT_SUPP).c_str(), supp ? "yes" : "no"); } static const char * htp_event_name(uint16_t id) { @@ -201,7 +217,7 @@ static const char * htp_event_name(uint16_t id) { } } -static void ggml_hexagon_dump_op_prof(const std::string &sess_name, const htp_opnode & node, const htp_prof_desc & pd) { +static void ggml_hexagon_dump_op_prof(const char * sess_name, const htp_opnode & node, const htp_prof_desc & pd) { if (!opt_profile) return; uint32_t op_usec = pd.usecs; @@ -215,237 +231,46 @@ static void ggml_hexagon_dump_op_prof(const std::string &sess_name, const htp_op pmu[0], pmu[1], pmu[2], pmu[3], pmu[4], pmu[5], pmu[6], pmu[7]); } - htp_opformat fmt(node); float mhz = op_usec > 0 ? (float) op_cycles / op_usec : 0.0f; - GGML_LOG_DEBUG("ggml-hex: %s profile-op %s|%s|%s|%s|%s|%s|usec %u cycles %u start %u mhz %.1f%s\n", sess_name.c_str(), - node.op_name().c_str(), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.kparams, op_usec, op_cycles, pd.cycles_start, mhz, pmu_str); + GGML_LOG_DEBUG("ggml-hex: %s profile-op %s|usec %u cycles %u start %u mhz %.1f%s\n", + sess_name, htp_format_op(node, HTP_OPFMT_PROF).c_str(), op_usec, op_cycles, pd.cycles_start, mhz, pmu_str); } -static void ggml_hexagon_dump_batch_prof(const std::string & sess_name, const htp_opbatch_rsp & rsp) { +static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbatch_rsp & rsp) { uint64_t batch_cycles = rsp.cycles_stop - rsp.cycles_start; float batch_mhz = rsp.usecs > 0 ? (float) batch_cycles / rsp.usecs : 0.0f; char evt_str[256] = "----"; if (opt_profile == 3) { - snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u", + static_assert(HTP_OP_MAX_NTHREADS == 10, "current implementation assumes 10 trace counters"); + snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u", rsp.n_traces[0], rsp.n_traces[1], rsp.n_traces[2], rsp.n_traces[3], rsp.n_traces[4], rsp.n_traces[5], rsp.n_traces[6], rsp.n_traces[7], - rsp.n_traces[8], rsp.n_traces[9], rsp.n_traces[10]); + rsp.n_traces[8], rsp.n_traces[9]); } GGML_LOG_DEBUG("ggml-hex: %s profile-op OPBATCH|----|n-ops %u|%s|----|----|usec %u cycles %llu start %llu mhz %.1f\n", - sess_name.c_str(), rsp.n_ops, evt_str, rsp.usecs, (unsigned long long) batch_cycles, (unsigned long long) rsp.cycles_start, batch_mhz); + sess_name, rsp.n_ops, evt_str, rsp.usecs, (unsigned long long) batch_cycles, (unsigned long long) rsp.cycles_start, batch_mhz); } -static void ggml_hexagon_dump_trace_events(const std::string & sess_name, const htp_opbatch_rsp & rsp, - const htp_trace_desc * trace_events, uint32_t n_traces) { +static void ggml_hexagon_dump_trace_events(const char * sess_name, const htp_opbatch_rsp & rsp, + const htp_trace_desc * trace_events, uint32_t n_threads, uint32_t n_traces) { if (opt_profile == 3 && trace_events) { - uint32_t valid_cnt[HTP_MAX_NTHREADS + 1] = {0}; - for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) { - uint32_t count = rsp.n_traces[t]; - valid_cnt[t] = count > n_traces ? n_traces : count; - } - - for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) { - for (uint32_t idx = 0; idx < valid_cnt[t]; idx++) { + for (uint32_t t = 0; t < n_threads; t++) { + uint32_t count = std::min(rsp.n_traces[t], n_traces); + for (uint32_t idx = 0; idx < count; idx++) { const auto & e = trace_events[t * n_traces + idx]; bool is_stop = (e.info & 0x8000) != 0; uint16_t info = e.info & 0x7FFF; GGML_LOG_DEBUG("ggml-hex: %s trace-evt %s: thread %u info %u %s %u\n", - sess_name.c_str(), htp_event_name(e.id), t, info, is_stop ? "stop" : "start", e.cycles); + sess_name, htp_event_name(e.id), t, info, is_stop ? "stop" : "start", e.cycles); } } } } -enum ggml_hexagon_tensor_flags { - GGML_HEXAGON_TENSOR_REPACK = (1 << 0), - GGML_HEXAGON_TENSOR_WEIGHT = (1 << 1), - GGML_HEXAGON_TENSOR_FENCE = (1 << 2), - GGML_HEXAGON_TENSOR_FUSEABLE = (1 << 3), -}; - -static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) { - return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || - type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL || - type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q6_K || - type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || - type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K; -} - -// Size of one repacked row in the DSP tiled layout. The K-quant tiles store uncompressed scales/mins, -// so they are larger than the ggml blocks. For the other repack types the tile has the same size as the ggml blocks. -static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne0) { - if (type == GGML_TYPE_Q6_K) { - return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q6_K / 32); - } - if (type == GGML_TYPE_Q4_K) { - return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q4_1 / 32); - } - if (type == GGML_TYPE_Q5_K) { - return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q5_K / 32); - } - if (type == GGML_TYPE_Q3_K) { - return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q3_K / 32); - } - if (type == GGML_TYPE_Q2_K) { - return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q2_K / 32); - } - return ggml_row_size(type, ne0); -} - -static inline bool ggml_hexagon_is_hmx_weight_type(enum ggml_type type) { - return type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || ggml_hexagon_is_repack_type(type); -} - -struct ggml_hexagon_session; - -static void ggml_hexagon_precompute_matmul_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_mm_kernel_params * kparams -); - -static void ggml_hexagon_precompute_fused_matmul_add_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * src2, - const struct ggml_tensor * dst, - struct htp_mm_kernel_params * kparams -); - -static void ggml_hexagon_precompute_unary_params( - const struct ggml_hexagon_session * sess, - uint32_t op, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_unary_kernel_params * kparams -); - -static bool ggml_hexagon_precompute_binary_params( - const struct ggml_hexagon_session * sess, - uint32_t op, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_binary_kernel_params * kparams -); - -static void ggml_hexagon_precompute_get_rows_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_get_rows_kernel_params * kparams -); - -static void ggml_hexagon_precompute_set_rows_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_set_rows_kernel_params * kparams -); - -static void ggml_hexagon_precompute_softmax_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - struct htp_softmax_kernel_params * kparams -); - -static void ggml_hexagon_precompute_rope_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - struct htp_rope_kernel_params * kparams -); - -static void ggml_hexagon_precompute_ssm_conv_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_ssm_conv_kernel_params * kparams -); - -static void ggml_hexagon_precompute_gated_delta_net_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - struct htp_gdn_kernel_params * kparams -); - -static void ggml_hexagon_precompute_sort_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - bool is_top_k, - struct htp_sort_kernel_params * kparams -); - -static void ggml_hexagon_precompute_fused_mmnx_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - int32_t n_weights, - struct htp_mm_kernel_params * kparams -); - -static void ggml_hexagon_precompute_fused_mmidnx_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - int32_t n_weights, - struct htp_mm_kernel_params * kparams -); - -static bool ggml_hexagon_precompute_allreduce_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * dst, - uint32_t rank, - uint32_t n_ranks, - bool has_add, - bool is_row_bcast, - bool is_shard_ok, - struct htp_allreduce_kernel_params * kparams -); - -static bool mm_is_hmx_eligible(const ggml_tensor * t); -static htp_op_code op_remap_to_htp(const ggml_tensor * t); -static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams); -static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams); -static bool is_mergeable_mul_mat(const ggml_tensor * t); -static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2); -static bool is_mergeable_mul_mat_id(const ggml_tensor * t); -static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tensor * n2); - // ** backend sessions -struct ggml_hexagon_tensor_extra { - std::vector<uint8_t> shadow_buf; - size_t shadow_size { 0 }; - uint32_t flags { 0 }; -}; - -static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) { - if (!t->extra) return false; - auto extra = (const struct ggml_hexagon_tensor_extra *) t->extra; - return (extra->flags & GGML_HEXAGON_TENSOR_FUSEABLE) != 0; -} - -static inline bool ggml_hexagon_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) { - const uintptr_t a0 = (uintptr_t) a->data; - const uintptr_t b0 = (uintptr_t) b->data; - const uintptr_t a1 = a0 + ggml_nbytes(a); - const uintptr_t b1 = b0 + ggml_nbytes(b); - - return a0 < b1 && b0 < a1; -} - -struct htp_opnode; - struct ggml_hexagon_opbatch; struct ggml_hexagon_opqueue; struct ggml_hexagon_shared_buffer; @@ -509,8 +334,6 @@ struct ggml_hexagon_session { uint64_t cached_uid = 0; std::vector<htp_opnode> cached_nodes; - mutable std::unordered_set<const ggml_tensor *> needs_repack; - ggml_hexagon_mdev_group mdev; ggml_backend_dev_t dev = nullptr; ggml_backend_hexagon_device_context * dev_ctx = nullptr; @@ -524,8 +347,13 @@ struct ggml_hexagon_session { void allocate(const ggml_hexagon_device_config & config) noexcept(false); void release() noexcept(true); - uint8_t * alloc_fence(uint32_t n_slots = 1); - void free_fence(void * ptr, uint32_t n_slots = 1); + uint8_t * alloc_fence(uint32_t n_slots = 1); + void free_fence(void * ptr, uint32_t n_slots = 1); + ggml_tensor make_fence_tensor(const volatile void * slot, uint32_t n_slots = 1) const; + uint32_t next_fence_seq() { + if (++fence_seq == 0) fence_seq = 1; + return fence_seq; + } uint8_t * mdev_fence_slot = nullptr; std::unordered_map<uint64_t, volatile uint32_t *> cpy_fence_slots; @@ -559,6 +387,12 @@ struct ggml_hexagon_session { } }; +const char * htp_session_name(const struct ggml_hexagon_session * sess) { return sess->c_name(); } +size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess) { return sess->vtcm_size; } +uint32_t htp_session_n_hvx(const struct ggml_hexagon_session * sess) { return sess->n_hvx; } +uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess) { return sess->n_hmx; } +uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess) { return sess->mdev.count; } + // ** backend buffers struct ggml_backend_hexagon_device_context { @@ -625,7 +459,6 @@ struct ggml_hexagon_rpcmem_block { struct ggml_hexagon_shared_buffer { ggml_hexagon_session * sess; std::shared_ptr<ggml_hexagon_rpcmem_block> mem; - std::vector<ggml_hexagon_tensor_extra *> tensor_extra; bool mapped; bool pinned; bool extended; @@ -648,6 +481,9 @@ struct ggml_hexagon_shared_buffer { if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(), sess->domain_id, size(), fd(), (unsigned) err); + if (!this->pinned) { + GGML_LOG_ERROR("ggml-hex: %s possibly out of IOVA space, try --no-host\n", sess->c_name()); + } throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)"); } @@ -723,9 +559,6 @@ struct ggml_hexagon_shared_buffer { ~ggml_hexagon_shared_buffer() { free(); - for (auto * extra : tensor_extra) { - delete extra; - } } }; @@ -737,7 +570,7 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size) : ggml_hexagon_shared_buffer(sess, size, false /* pinned */), - slot_count(size / GGML_HEXAGON_FENCE_SLOT_SIZE), + slot_count(size / HTP_FENCE_SLOT_SIZE), slot_head(0) { backend_buffer.buft = buft; backend_buffer.context = static_cast<ggml_hexagon_shared_buffer *>(this); @@ -750,21 +583,21 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { if (n_slots == 1 && !free_slots.empty()) { uint32_t slot = free_slots.back(); free_slots.pop_back(); - ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE; } else if (slot_head + n_slots <= slot_count) { uint32_t slot = slot_head; slot_head += n_slots; - ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE; } if (ptr) { - memset(ptr, 0, (size_t) n_slots * GGML_HEXAGON_FENCE_SLOT_SIZE); + memset(ptr, 0, (size_t) n_slots * HTP_FENCE_SLOT_SIZE); } return ptr; } void free_slot(void * ptr, uint32_t n_slots = 1) { if (!ptr) return; - uint32_t slot = ((uint8_t *) ptr - base()) / GGML_HEXAGON_FENCE_SLOT_SIZE; + uint32_t slot = ((uint8_t *) ptr - base()) / HTP_FENCE_SLOT_SIZE; for (uint32_t i = 0; i < n_slots; i++) { free_slots.push_back(slot + i); } @@ -783,6 +616,24 @@ inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) { } } +inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * slot, uint32_t n_slots) const { + ggml_tensor t {}; + t.buffer = &this->fence_buf->backend_buffer; + t.extra = (void *) (uintptr_t) HTP_TENSOR_FENCE; + t.data = const_cast<void *>(slot); + t.type = GGML_TYPE_I8; + t.ne[0] = HTP_FENCE_SLOT_SIZE; + t.ne[1] = (int64_t) n_slots; + t.ne[2] = 1; + t.ne[3] = 1; + t.nb[0] = 1; + t.nb[1] = HTP_FENCE_SLOT_SIZE; + t.nb[2] = t.nb[1] * t.ne[1]; + t.nb[3] = t.nb[2]; + t.op = GGML_OP_NONE; + return t; +} + static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) { auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context); sbuf->sess->unclone_buffer(sbuf); @@ -801,6239 +652,1755 @@ static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buf HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(), tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor)); - auto extra = new ggml_hexagon_tensor_extra(); - sbuf->tensor_extra.push_back(extra); - - tensor->extra = extra; - if (ggml_hexagon_is_repack_type(tensor->type)) { - if (sess->needs_repack.count(tensor)) { - extra->flags |= GGML_HEXAGON_TENSOR_REPACK; - sess->needs_repack.erase(tensor); - } - } + tensor->extra = (void *) (uintptr_t) 0; return GGML_STATUS_SUCCESS; } -// ** Repack helpers for tiled quantized weights +static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; -static void unpack_q4_0_quants(uint8_t * qs, const block_q4_0 * x, unsigned int bi) { - static const int qk = QK4_0; + sess->flush_sync(); - for (unsigned int i = 0; i < qk / 2; ++i) { - const int x0 = (x->qs[i] & 0x0F); - const int x1 = (x->qs[i] >> 4); - qs[bi * qk + i + 0] = x0; - qs[bi * qk + i + qk / 2] = x1; + if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT); } -} -static void pack_q4_0_quants(block_q4_0 * x, const uint8_t * qs, unsigned int bi) { - static const int qk = QK4_0; + HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor)); - for (unsigned int i = 0; i < qk / 2; ++i) { - const uint8_t x0 = qs[bi * qk + i + 0]; - const uint8_t x1 = qs[bi * qk + i + qk / 2]; - x->qs[i] = x0 | (x1 << 4); - } + memcpy((char *) tensor->data + offset, data, size); } -static void unpack_q4_1_quants(uint8_t * qs, const block_q4_1 * x, unsigned int bi) { - static const int qk = QK4_1; - - for (unsigned int i = 0; i < qk / 2; ++i) { - const int x0 = (x->qs[i] & 0x0F); - const int x1 = (x->qs[i] >> 4); - qs[bi * qk + i + 0] = x0; - qs[bi * qk + i + qk / 2] = x1; - } -} +static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor); -static void pack_q4_1_quants(block_q4_1 * x, const uint8_t * qs, unsigned int bi) { - static const int qk = QK4_1; + HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, flags); - for (unsigned int i = 0; i < qk / 2; ++i) { - const uint8_t x0 = qs[bi * qk + i + 0]; - const uint8_t x1 = qs[bi * qk + i + qk / 2]; - x->qs[i] = x0 | (x1 << 4); + if ((flags & HTP_TENSOR_TILED) == 0) { + memcpy(data, (const char *) tensor->data + offset, size); + return; } -} -static void unpack_mxfp4_quants(uint8_t * qs, const block_mxfp4 * x, unsigned int bi) { - static const int qk = QK_MXFP4; - - for (unsigned int i = 0; i < qk / 2; ++i) { - const int x0 = (x->qs[i] & 0x0F); - const int x1 = (x->qs[i] >> 4); - qs[bi * qk + i + 0] = x0; - qs[bi * qk + i + qk / 2] = x1; - } + htp_unpack_tensor_tiled(data, tensor, offset, size); } -static void pack_mxfp4_quants(block_mxfp4 * x, const uint8_t * qs, unsigned int bi) { - static const int qk = QK_MXFP4; +static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer, + const struct ggml_tensor * src, + struct ggml_tensor * dst) { + // we might optimize this later, for now take the slow path (ie get/set_tensor) + return false; - for (unsigned int i = 0; i < qk / 2; ++i) { - const uint8_t x0 = qs[bi * qk + i + 0]; - const uint8_t x1 = qs[bi * qk + i + qk / 2]; - x->qs[i] = x0 | (x1 << 4); - } + GGML_UNUSED(buffer); + GGML_UNUSED(src); + GGML_UNUSED(dst); } -// repack q4_0 data into q4_0_tiled tensor -static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - const block_q4_0 * src_matrix = (const block_q4_0 *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - int n_col_tiles = ne1_padded / 32; - int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; - const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - - size_t slice_size = ne1 * ggml_row_size(t->type, ne0); - int64_t start_slice = offset / slice_size; - int64_t end_slice = (offset + size + slice_size - 1) / slice_size; - if (end_slice > ne2 * ne3) { - end_slice = ne2 * ne3; - } - - for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { - const block_q4_0 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32)); - uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; - - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; - - uint8_t tile_quants[32][32]; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - unpack_q4_0_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0); - } else { - memset(tile_quants[row], 8, 32); - } - } +static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; - } - } + sess->flush_sync(); - ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].d : 0; - } - } - } + if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT); } -} - -// repack q4_0_tiled tensor into q4_0 data -static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { - block_q4_0 * dst_matrix = (block_q4_0 *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - int n_col_tiles = ne1_padded / 32; - int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; - const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - - size_t slice_size = ne1 * ggml_row_size(t->type, ne0); - size_t row_size_bytes = ggml_row_size(t->type, ne0); - int64_t start_slice = offset / slice_size; - int64_t end_slice = (offset + size + slice_size - 1) / slice_size; - if (end_slice > ne2 * ne3) { - end_slice = ne2 * ne3; - } - - for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { - size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); - size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); - size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; - size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; - - int64_t start_row = slice_offset_start / row_size_bytes; - int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes; - end_row = (std::min)(end_row, ne1); - - int start_ct = start_row / 32; - int end_ct = (end_row + 31) / 32; - end_ct = (std::min)(end_ct, n_col_tiles); - - block_q4_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q4_0); - const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; - - for (int ct = start_ct; ct < end_ct; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; - - uint8_t tile_quants[32][32]; - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - uint8_t val = tile_src[cp * 32 + row]; - tile_quants[row][2 * cp + 0] = val & 0x0F; - tile_quants[row][2 * cp + 1] = val >> 4; - } - } - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r >= start_row && r < end_row && kt < ne0 / 32) { - pack_q4_0_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0); - } - } + HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor)); - const ggml_half * scale_src = (const ggml_half *)(tile_src + 512); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r >= start_row && r < end_row && kt < ne0 / 32) { - dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[row]; - } - } - } - } + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); } } -// repack q4_1 data into q4_1_tiled tensor -static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - const block_q4_1 * src_matrix = (const block_q4_1 *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - int n_col_tiles = ne1_padded / 32; - int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; - const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - - size_t slice_size = ne1 * ggml_row_size(t->type, ne0); - int64_t start_slice = offset / slice_size; - int64_t end_slice = (offset + size + slice_size - 1) / slice_size; - if (end_slice > ne2 * ne3) { - end_slice = ne2 * ne3; - } - - for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { - const block_q4_1 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32)); - uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; - - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; - - uint8_t tile_quants[32][32]; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - unpack_q4_1_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0); - } else { - memset(tile_quants[row], 0, 32); - } - } +static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor); - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; - } - } + HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage); - ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - scale_dst[2 * row + 0] = src_slice[r * (ne0 / 32) + kt].d; - scale_dst[2 * row + 1] = src_slice[r * (ne0 / 32) + kt].m; - } else { - scale_dst[2 * row + 0] = 0; - scale_dst[2 * row + 1] = 0; - } - } - } + if ((flags & HTP_TENSOR_TILED) == 0) { + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size); } + return; } -} - -// repack q4_1_tiled tensor into q4_1 data -static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t offset, size_t size) { - block_q4_1 * dst_matrix = (block_q4_1 *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - int n_col_tiles = ne1_padded / 32; - int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; - const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - - size_t slice_size = ne1 * ggml_row_size(t->type, ne0); - size_t row_size_bytes = ggml_row_size(t->type, ne0); - int64_t start_slice = offset / slice_size; - int64_t end_slice = (offset + size + slice_size - 1) / slice_size; - if (end_slice > ne2 * ne3) { - end_slice = ne2 * ne3; - } - - for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { - size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); - size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); - size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; - size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; - - int64_t start_row = slice_offset_start / row_size_bytes; - int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes; - end_row = (std::min)(end_row, ne1); - - int start_ct = start_row / 32; - int end_ct = (end_row + 31) / 32; - end_ct = (std::min)(end_ct, n_col_tiles); - - block_q4_1 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q4_1); - const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; - - for (int ct = start_ct; ct < end_ct; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; - - uint8_t tile_quants[32][32]; - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - uint8_t val = tile_src[cp * 32 + row]; - tile_quants[row][2 * cp + 0] = val & 0x0F; - tile_quants[row][2 * cp + 1] = val >> 4; - } - } - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r >= start_row && r < end_row && kt < ne0 / 32) { - pack_q4_1_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0); - } - } + size_t temp_size = n_copies > 0 ? (n_copies - 1) * stride_tensor + size : 0; + size_t slice_size = tensor->ne[1] * ggml_row_size(tensor->type, tensor->ne[0]); + size_t slice_offset = offset % slice_size; + size_t row_size_bytes = ggml_row_size(tensor->type, tensor->ne[0]); - const ggml_half * scale_src = (const ggml_half *)(tile_src + 512); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r >= start_row && r < end_row && kt < ne0 / 32) { - dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[2 * row]; - dst_slice[(r - start_row) * (ne0 / 32) + kt].m = scale_src[2 * row + 1]; - } - } - } - } - } -} + GGML_ASSERT((slice_offset % row_size_bytes) == 0 && "offset must be aligned to row boundary"); + GGML_ASSERT((temp_size % row_size_bytes) == 0 && "temp_size must be a multiple of row size"); + GGML_ASSERT((offset + temp_size) <= ggml_nbytes(tensor)); -// repack q8_0 data into q8_0_tiled tensor -static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - const block_q8_0 * src_matrix = (const block_q8_0 *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - int n_col_tiles = ne1_padded / 32; - int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q8_0; - const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - - size_t slice_size = ne1 * ggml_row_size(t->type, ne0); - int64_t start_slice = offset / slice_size; - int64_t end_slice = (offset + size + slice_size - 1) / slice_size; - if (end_slice > ne2 * ne3) { - end_slice = ne2 * ne3; - } - - for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { - const block_q8_0 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32)); - uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; - - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; - - for (int cp = 0; cp < 16; cp++) { - int col0 = cp * 2; - int col1 = col0 + 1; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_slice[r * (ne0 / 32) + kt] : NULL; - tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0; - tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0; - } - } + std::vector<uint8_t> temp_buf(temp_size); + htp_unpack_tensor_tiled(temp_buf.data(), tensor, offset, temp_size); - ggml_half * scale_dst = (ggml_half *)(tile_dst + 1024); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].d : 0; - } - } - } + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *) data + i * stride_data, temp_buf.data() + i * stride_tensor, size); } } -// repack q8_0_tiled tensor into q8_0 data -static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { - block_q8_0 * dst_matrix = (block_q8_0 *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - int n_col_tiles = ne1_padded / 32; - int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q8_0; - const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - - size_t slice_size = ne1 * ggml_row_size(t->type, ne0); - size_t row_size_bytes = ggml_row_size(t->type, ne0); - int64_t start_slice = offset / slice_size; - int64_t end_slice = (offset + size + slice_size - 1) / slice_size; - if (end_slice > ne2 * ne3) { - end_slice = ne2 * ne3; - } - - for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { - size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); - size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); - size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; - size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; - - int64_t start_row = slice_offset_start / row_size_bytes; - int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes; - end_row = (std::min)(end_row, ne1); - - int start_ct = start_row / 32; - int end_ct = (end_row + 31) / 32; - end_ct = (std::min)(end_ct, n_col_tiles); - - block_q8_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q8_0); - const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; - - for (int ct = start_ct; ct < end_ct; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; - - for (int cp = 0; cp < 16; cp++) { - int col0 = cp * 2; - int col1 = col0 + 1; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r >= start_row && r < end_row && kt < ne0 / 32) { - block_q8_0 & b = dst_slice[(r - start_row) * (ne0 / 32) + kt]; - b.qs[col0] = tile_src[cp * 64 + 2 * row + 0]; - b.qs[col1] = tile_src[cp * 64 + 2 * row + 1]; - } - } - } +static void ggml_backend_hexagon_buffer_set_tensor_n(ggml_backend_buffer_t buffer, + struct ggml_tensor * const * tensors, + const void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + const bool is_weights = ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; - const ggml_half * scale_src = (const ggml_half *)(tile_src + 1024); - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r >= start_row && r < end_row && kt < ne0 / 32) { - dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[row]; - } - } - } + sess->flush_sync(); + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + if (is_weights) { + ggml_hexagon_tensor_set_flags(tensors[i], HTP_TENSOR_WEIGHT); } + memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]); } } -// repack mxfp4 data into mxfp4_tiled tensor -static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - const block_mxfp4 * src_matrix = (const block_mxfp4 *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - int n_col_tiles = ne1_padded / 32; - int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_MXFP4; - const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - - size_t slice_size = ne1 * ggml_row_size(t->type, ne0); - int64_t start_slice = offset / slice_size; - int64_t end_slice = (offset + size + slice_size - 1) / slice_size; - if (end_slice > ne2 * ne3) { - end_slice = ne2 * ne3; - } - - for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { - const block_mxfp4 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32)); - uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; - - for (int ct = 0; ct < n_col_tiles; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; - - uint8_t tile_quants[32][32]; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r < ne1 && kt < ne0 / 32) { - unpack_mxfp4_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0); - } else { - memset(tile_quants[row], 0, 32); - } - } +static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + HEX_VERBOSE("ggml-hex: %s clear-buff base %p size %zu\n", sess->c_name(), (void *) sbuf->base(), sbuf->size()); + memset(sbuf->base(), value, sbuf->size()); +} - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; - } - } +static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { + /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, + /* .get_base = */ ggml_backend_hexagon_buffer_get_base, + /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, + /* .memset_tensor = */ NULL, + /* .set_tensor = */ ggml_backend_hexagon_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_hexagon_buffer_get_tensor, + /* .set_tensor_2d = */ ggml_backend_hexagon_buffer_set_tensor_2d, + /* .get_tensor_2d = */ ggml_backend_hexagon_buffer_get_tensor_2d, + /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, + /* .clear = */ ggml_backend_hexagon_buffer_clear, + /* .reset = */ NULL, + /* .set_tensor_n = */ ggml_backend_hexagon_buffer_set_tensor_n, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, +}; - uint8_t * scale_dst = tile_dst + 512; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].e : 0; - } - } - } - } -} +// ** backend buffer type -// repack mxfp4_tiled tensor into mxfp4 data -static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset, size_t size) { - block_mxfp4 * dst_matrix = (block_mxfp4 *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - int n_col_tiles = ne1_padded / 32; - int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_MXFP4; - const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size; - - size_t slice_size = ne1 * ggml_row_size(t->type, ne0); - size_t row_size_bytes = ggml_row_size(t->type, ne0); - int64_t start_slice = offset / slice_size; - int64_t end_slice = (offset + size + slice_size - 1) / slice_size; - if (end_slice > ne2 * ne3) { - end_slice = ne2 * ne3; - } - - for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { - size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); - size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); - size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; - size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; - - int64_t start_row = slice_offset_start / row_size_bytes; - int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes; - end_row = (std::min)(end_row, ne1); - - int start_ct = start_row / 32; - int end_ct = (end_row + 31) / 32; - end_ct = (std::min)(end_ct, n_col_tiles); - - block_mxfp4 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_mxfp4); - const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; - - for (int ct = start_ct; ct < end_ct; ct++) { - for (int kt = 0; kt < n_k_tiles; kt++) { - const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; - - uint8_t tile_quants[32][32]; - for (int cp = 0; cp < 16; cp++) { - for (int row = 0; row < 32; row++) { - uint8_t val = tile_src[cp * 32 + row]; - tile_quants[row][2 * cp + 0] = val & 0x0F; - tile_quants[row][2 * cp + 1] = val >> 4; - } - } +static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r >= start_row && r < end_row && kt < ne0 / 32) { - pack_mxfp4_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0); - } - } + sess->flush_sync(); - const uint8_t * scale_src = tile_src + 512; - for (int row = 0; row < 32; row++) { - int64_t r = ct * 32 + row; - if (r >= start_row && r < end_row && kt < ne0 / 32) { - dst_slice[(r - start_row) * (ne0 / 32) + kt].e = scale_src[row]; - } - } - } - } - } + memcpy((char *) tensor->data + offset, data, size); } -// unsigned 6-bit value (0..63) of element e of a Q6_K block, same bit layout as dequantize_row_q6_K -static inline uint8_t q6_K_get_quant(const block_q6_K * b, int e) { - const int c = e / 128; - const int w = e % 128; - const int g = w / 32; - const int l = w % 32; - const uint8_t * ql = b->ql + c * 64; - const uint8_t * qh = b->qh + c * 32; - uint8_t lo, hi; - switch (g) { - case 0: lo = ql[l] & 0xF; hi = (qh[l] >> 0) & 3; break; - case 1: lo = ql[l + 32] & 0xF; hi = (qh[l] >> 2) & 3; break; - case 2: lo = ql[l] >> 4; hi = (qh[l] >> 4) & 3; break; - default: lo = ql[l + 32] >> 4; hi = (qh[l] >> 6) & 3; break; - } - return (uint8_t) (lo | (hi << 4)); +static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size) { + memcpy(data, (const char *) tensor->data + offset, size); + GGML_UNUSED(buffer); } -// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q6_K in htp/matmul-ops.h -static void repack_q6_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); - - const block_q6_K * src_matrix = (const block_q6_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - GGML_ASSERT(ne0 % QK_K == 0); - - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q6_K; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; - - const int64_t sb_per_row = ne0 / QK_K; - - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_q6_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; - - memset(matrix_dst, 0, matrix_size); // padding rows and the OR-ed nibbles below need zeroed tiles - - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - const block_q6_K * src_row = src_slice + r * sb_per_row; - - for (int kt = 0; kt < n_k_tiles; kt++) { - const int kt_local = kt % 8; // k-tile within the super-block - const block_q6_K * b = &src_row[kt / 8]; - const float d = GGML_FP16_TO_FP32(b->d); - - uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; - uint8_t * lo_pl = tile; - uint8_t * hi_pl = tile + 512; - ggml_half * sc_pl = (ggml_half *) (tile + 768); - - for (int lk = 0; lk < 32; lk++) { - const uint8_t q6 = q6_K_get_quant(b, kt_local * 32 + lk); - const int g = lk >> 2; - const int pos = row * 4 + (lk & 3); - lo_pl[(g >> 1) * 128 + pos] |= (uint8_t) ((q6 & 0xF) << ((g & 1) * 4)); - hi_pl[(g >> 2) * 128 + pos] |= (uint8_t) ((q6 >> 4) << ((g & 3) * 2)); - } - for (int sub = 0; sub < 2; sub++) { - sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) b->scales[kt_local * 2 + sub]); - } - } - } +static void ggml_backend_hexagon_host_buffer_set_tensor_n(ggml_backend_buffer_t buffer, + struct ggml_tensor * const * tensors, + const void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; } + memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]); } - - GGML_UNUSED(size); } -// Reverse of repack_q6_K_tiled. Unpacks quants losslessly and normalizes sub-block scales. Read-back only. -static void repack_tiled_q6_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); - - block_q6_K * dst_matrix = (block_q6_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - GGML_ASSERT(ne0 % QK_K == 0); - - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q6_K; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; - - const int64_t sb_per_row = ne0 / QK_K; - - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_q6_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; - - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - block_q6_K * dst_row = dst_slice + r * sb_per_row; - - for (int64_t sb = 0; sb < sb_per_row; sb++) { - block_q6_K * b = &dst_row[sb]; - memset(b, 0, sizeof(block_q6_K)); - - float sub_scales[16]; - for (int kt_local = 0; kt_local < 8; kt_local++) { - const int kt = sb * 8 + kt_local; - const uint8_t * tile = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; - const uint8_t * lo_pl = tile; - const uint8_t * hi_pl = tile + 512; - const ggml_half * sc_pl = (const ggml_half *) (tile + 768); - - const int c = kt_local / 4; - const int g = kt_local % 4; - uint8_t * ql = b->ql + c * 64; - uint8_t * qh = b->qh + c * 32; - - for (int lk = 0; lk < 32; lk++) { - const int g_tile = lk >> 2; - const int pos = row * 4 + (lk & 3); - const uint8_t lo = (lo_pl[(g_tile >> 1) * 128 + pos] >> ((g_tile & 1) * 4)) & 0xF; - const uint8_t hi = (hi_pl[(g_tile >> 2) * 128 + pos] >> ((g_tile & 3) * 2)) & 3; - - switch (g) { - case 0: - ql[lk] |= lo; - qh[lk] |= (hi << 0); - break; - case 1: - ql[lk + 32] |= lo; - qh[lk] |= (hi << 2); - break; - case 2: - ql[lk] |= (lo << 4); - qh[lk] |= (hi << 4); - break; - default: - ql[lk + 32] |= (lo << 4); - qh[lk] |= (hi << 6); - break; - } - } +static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = { + /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, + /* .get_base = */ ggml_backend_hexagon_buffer_get_base, + /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, + /* .memset_tensor = */ NULL, + /* .set_tensor = */ ggml_backend_hexagon_host_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_hexagon_host_buffer_get_tensor, + /* .set_tensor_2d = */ NULL, + /* .get_tensor_2d = */ NULL, + /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, + /* .clear = */ ggml_backend_hexagon_buffer_clear, + /* .reset = */ NULL, + /* .set_tensor_n = */ ggml_backend_hexagon_host_buffer_set_tensor_n, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, +}; - for (int sub = 0; sub < 2; sub++) { - sub_scales[kt_local * 2 + sub] = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]); - } - } +// ** backend buffer type - float max_abs_scale = 0.0f; - for (int s = 0; s < 16; s++) { - float abs_scale = fabsf(sub_scales[s]); - if (abs_scale > max_abs_scale) { - max_abs_scale = abs_scale; - } - } +static const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_type_t buffer_type) { + return static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->name.c_str(); +} - if (max_abs_scale == 0.0f) { - b->d = GGML_FP32_TO_FP16(0.0f); - memset(b->scales, 0, sizeof(b->scales)); - } else { - float d_flt = max_abs_scale / 127.0f; - b->d = GGML_FP32_TO_FP16(d_flt); - float d_actual = GGML_FP16_TO_FP32(b->d); - float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f; - for (int s = 0; s < 16; s++) { - int sc = (int) roundf(sub_scales[s] * inv_d); - b->scales[s] = (int8_t) (std::max)(-128, (std::min)(127, sc)); - } - } - } - } - } +static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( + ggml_backend_buffer_type_t buffer_type, size_t size) { + auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx; + auto sess = dev_ctx->session(); + if (size > sess->max_vmem) { + GGML_LOG_ERROR("ggml-hex: %s buffer size %zu exceeds max_vmem %zu\n", + dev_ctx->c_name(), size, sess->max_vmem); + return nullptr; + } + try { + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); + return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); + } catch (const std::exception & exc) { + GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", dev_ctx->c_name(), exc.what()); + return nullptr; } - - GGML_UNUSED(size); } -// low 2 bits (0..3) of element e of a Q2_K or Q3_K block, same bit layout as dequantize_row_q2_K / q3_K -static inline uint8_t q2_3_K_get_low2(const uint8_t * qs, int e) { - const int c = e / 128; - const int j = (e % 128) / 32; - const int l = e % 32; - return (qs[c * 32 + l] >> (2 * j)) & 3; +static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( + ggml_backend_buffer_type_t buffer_type, size_t size) { + auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx; + auto sess = dev_ctx->session(); + if (size > sess->max_vmem) { + GGML_LOG_ERROR("ggml-hex: %s host buffer size %zu exceeds max_vmem %zu\n", + dev_ctx->c_name(), size, sess->max_vmem); + return nullptr; + } + try { + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); + return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size); + } catch (const std::exception & exc) { + GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", dev_ctx->c_name(), exc.what()); + return nullptr; + } } -// hmask bit of element e of a Q3_K block, same bit layout as dequantize_row_q3_K -static inline bool q3_K_get_hbit(const block_q3_K * b, int e) { - const int c = e / 128; - const int j = (e % 128) / 32; - const int l = e % 32; - return (b->hmask[l] >> (c * 4 + j)) & 1; +static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + return 128; // HVX alignment + GGML_UNUSED(buft); } -// signed 6-bit scale j (-32..31) of a Q3_K block, same packing as quantize_row_q3_K_ref -static inline int q3_K_get_scale(const uint8_t * scales, int j) { - const int lo = (j < 8) ? (scales[j] & 0xF) : (scales[j - 8] >> 4); - const int hi = (scales[8 + j % 4] >> (2 * (j / 4))) & 3; - return (lo | (hi << 4)) - 32; +static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) { + if (opt_rt_repack != 0 && !ggml_backend_buft_is_host(buft) && ggml_op_is_empty(t->op) && htp_is_tiled_weight_tensor(t)) { + int64_t ne0 = hex_round_up(t->ne[0], 32); + int64_t ne1 = hex_round_up(t->ne[1], 32); + int64_t ne2 = t->ne[2]; + int64_t ne3 = t->ne[3]; + size_t tiled_size = htp_tiled_row_size(t->type, ne0) * ne1 * ne2 * ne3; + return std::max(tiled_size, ggml_nbytes(t)); + } + return ggml_nbytes(t); } -// read-back: find fp16 d and l[j] in [lmin, lmax] with fp16(d * l[j]) == prod[j] for all j, false if none -static bool hexagon_recover_k_scales(const ggml_half * prod, int n, int lmin, int lmax, ggml_half * d_out, int * l_out) { - int jmax = 0; - for (int j = 1; j < n; j++) { - if (fabsf(GGML_FP16_TO_FP32(prod[j])) > fabsf(GGML_FP16_TO_FP32(prod[jmax]))) { - jmax = j; - } +static size_t parse_size(const char * str, size_t default_unit = 1024 * 1024) { + if (!str || str[0] == '\0') { + return 0; } - const float pmax = GGML_FP16_TO_FP32(prod[jmax]); - if (pmax == 0.0f) { - *d_out = GGML_FP32_TO_FP16(0.0f); - for (int j = 0; j < n; j++) { - l_out[j] = 0; - } - return true; + char * end = NULL; + double val = strtod(str, &end); + if (val < 0) { + return 0; } - - // the quantizers put the largest scale at or near the range end, so try large |l| first - const int lext = (std::max)(-lmin, lmax); - for (int a = lext; a >= 1; a--) { - for (int sign : { 1, -1 }) { - const int lj = sign * a; - if (lj < lmin || lj > lmax) { - continue; - } - const ggml_half d0 = GGML_FP32_TO_FP16(pmax / (float) lj); - for (int ulp : { 0, -1, 1 }) { - ggml_half d = d0; - uint16_t bits; - memcpy(&bits, &d, sizeof(bits)); - bits = (uint16_t) (bits + ulp); - memcpy(&d, &bits, sizeof(bits)); - - const float df = GGML_FP16_TO_FP32(d); - if (!std::isfinite(df) || df == 0.0f) { - continue; - } - bool ok = true; - for (int j = 0; j < n && ok; j++) { - const int l = (int) roundf(GGML_FP16_TO_FP32(prod[j]) / df); - const ggml_half p = GGML_FP32_TO_FP16(df * (float) l); - ok = l >= lmin && l <= lmax && memcmp(&p, &prod[j], sizeof(p)) == 0; - l_out[j] = l; - } - if (ok) { - *d_out = d; - return true; - } - } + if (end && *end) { + while (*end == ' ') end++; + if (*end == 'k' || *end == 'K') { + return (size_t) (val * 1024); + } else if (*end == 'm' || *end == 'M') { + return (size_t) (val * 1024 * 1024); + } else if (*end == 'g' || *end == 'G') { + return (size_t) (val * 1024 * 1024 * 1024); } } - return false; + return (size_t) (val * default_unit); } -// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q3_K in htp/matmul-ops.h -static void repack_q3_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); - - const block_q3_K * src_matrix = (const block_q3_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - GGML_ASSERT(ne0 % QK_K == 0); - - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q3_K; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; - - const int64_t sb_per_row = ne0 / QK_K; - - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_q3_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; - - memset(matrix_dst, 0, matrix_size); // padding rows and the OR-ed bits below need zeroed tiles - - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - const block_q3_K * src_row = src_slice + r * sb_per_row; - - for (int kt = 0; kt < n_k_tiles; kt++) { - const int kt_local = kt % 8; // k-tile within the super-block - const block_q3_K * b = &src_row[kt / 8]; - const float d = GGML_FP16_TO_FP32(b->d); - - uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; - uint8_t * lo_pl = tile; - uint8_t * neg_pl = tile + 256; - ggml_half * sc_pl = (ggml_half *) (tile + 384); - - for (int lk = 0; lk < 32; lk++) { - const int e = kt_local * 32 + lk; - const int g = lk >> 2; - const int pos = row * 4 + (lk & 3); - lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, e) << ((g & 3) * 2)); - if (!q3_K_get_hbit(b, e)) { - neg_pl[pos] |= (uint8_t) (1 << g); - } - } - for (int sub = 0; sub < 2; sub++) { - sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) q3_K_get_scale(b->scales, kt_local * 2 + sub)); - } - } - } - } - } - - GGML_UNUSED(size); +static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + return opt_mbuf_dyn; + GGML_UNUSED(buft); } -// Reverse of repack_q3_K_tiled. Unpacks quants losslessly and normalizes sub-block scales. Read-back only. -static void repack_tiled_q3_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); - - block_q3_K * dst_matrix = (block_q3_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - GGML_ASSERT(ne0 % QK_K == 0); - - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q3_K; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; - - const int64_t sb_per_row = ne0 / QK_K; - - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_q3_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; - - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - block_q3_K * dst_row = dst_slice + r * sb_per_row; - - for (int64_t sb = 0; sb < sb_per_row; sb++) { - block_q3_K * b = &dst_row[sb]; - memset(b, 0, sizeof(block_q3_K)); - - ggml_half sub_scales[16]; - for (int kt_local = 0; kt_local < 8; kt_local++) { - const int kt = sb * 8 + kt_local; - const uint8_t * tile = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; - const uint8_t * lo_pl = tile; - const uint8_t * neg_pl = tile + 256; - const ggml_half * sc_pl = (const ggml_half *) (tile + 384); - - for (int lk = 0; lk < 32; lk++) { - const int e = kt_local * 32 + lk; - const int g = lk >> 2; - const int pos = row * 4 + (lk & 3); - const uint8_t lo = (lo_pl[(g >> 2) * 128 + pos] >> ((g & 3) * 2)) & 3; - - const int c = e / 128; - const int j = (e % 128) / 32; - const int l = e % 32; - b->qs[c * 32 + l] |= (uint8_t) (lo << (2 * j)); - if (!((neg_pl[pos] >> g) & 1)) { - b->hmask[l] |= (uint8_t) (1 << (c * 4 + j)); - } - } - - for (int sub = 0; sub < 2; sub++) { - sub_scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row]; - } - } - - int ls[16]; - if (!hexagon_recover_k_scales(sub_scales, 16, -32, 31, &b->d, ls)) { - // no exact match: same scale choice as quantize_row_q3_K_ref - float max_scale = 0.0f; - for (int s = 0; s < 16; s++) { - if (fabsf(GGML_FP16_TO_FP32(sub_scales[s])) > fabsf(max_scale)) { - max_scale = GGML_FP16_TO_FP32(sub_scales[s]); - } - } - b->d = GGML_FP32_TO_FP16(-max_scale / 32.0f); - const float d_actual = GGML_FP16_TO_FP32(b->d); - const float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f; - for (int s = 0; s < 16; s++) { - ls[s] = (std::max)(-32, (std::min)(31, (int) roundf(GGML_FP16_TO_FP32(sub_scales[s]) * inv_d))); - } - } - - for (int s = 0; s < 16; s++) { - const int l = ls[s] + 32; - if (s < 8) { - b->scales[s] = l & 0xF; - } else { - b->scales[s - 8] |= (uint8_t) ((l & 0xF) << 4); - } - b->scales[s % 4 + 8] |= (uint8_t) ((l >> 4) << (2 * (s / 4))); - } - } - } - } - } - - GGML_UNUSED(size); +static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + return false; + GGML_UNUSED(buft); } -// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q2_K in htp/matmul-ops.h -static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); - - const block_q2_K * src_matrix = (const block_q2_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - GGML_ASSERT(ne0 % QK_K == 0); - - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q2_K; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; - - const int64_t sb_per_row = ne0 / QK_K; - - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_q2_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; - - memset(matrix_dst, 0, matrix_size); // padding rows and the OR-ed bits below need zeroed tiles - - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - const block_q2_K * src_row = src_slice + r * sb_per_row; - - for (int kt = 0; kt < n_k_tiles; kt++) { - const int kt_local = kt % 8; // k-tile within the super-block - const block_q2_K * b = &src_row[kt / 8]; - const float d = GGML_FP16_TO_FP32(b->d); - const float dmin = GGML_FP16_TO_FP32(b->dmin); - - uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; - uint8_t * lo_pl = tile; - ggml_half * sc_pl = (ggml_half *) (tile + 256); - ggml_half * m_pl = (ggml_half *) (tile + 384); - - for (int lk = 0; lk < 32; lk++) { - const int g = lk >> 2; - const int pos = row * 4 + (lk & 3); - lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, kt_local * 32 + lk) << ((g & 3) * 2)); - } - for (int sub = 0; sub < 2; sub++) { - const uint8_t sc = b->scales[kt_local * 2 + sub]; - sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16( d * (float) (sc & 0xF)); - m_pl [sub * 32 + row] = GGML_FP32_TO_FP16(-dmin * (float) (sc >> 4)); - } - } - } - } - } - - GGML_UNUSED(size); +static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + return true; + GGML_UNUSED(buft); } -// Reverse of repack_q2_K_tiled. Unpacks quants losslessly and normalizes scales/mins. Read-back only. -static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); - - block_q2_K * dst_matrix = (block_q2_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - GGML_ASSERT(ne0 % QK_K == 0); - - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q2_K; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; - - const int64_t sb_per_row = ne0 / QK_K; - - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_q2_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; - - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - block_q2_K * dst_row = dst_slice + r * sb_per_row; - - for (int64_t sb = 0; sb < sb_per_row; sb++) { - block_q2_K * b = &dst_row[sb]; - memset(b, 0, sizeof(block_q2_K)); - - ggml_half sub_scales[16]; - ggml_half sub_mins[16]; - for (int kt_local = 0; kt_local < 8; kt_local++) { - const int kt = sb * 8 + kt_local; - const uint8_t * tile = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; - const uint8_t * lo_pl = tile; - const ggml_half * sc_pl = (const ggml_half *) (tile + 256); - const ggml_half * m_pl = (const ggml_half *) (tile + 384); - - for (int lk = 0; lk < 32; lk++) { - const int e = kt_local * 32 + lk; - const int g = lk >> 2; - const int pos = row * 4 + (lk & 3); - const uint8_t lo = (lo_pl[(g >> 2) * 128 + pos] >> ((g & 3) * 2)) & 3; - b->qs[(e / 128) * 32 + e % 32] |= (uint8_t) (lo << (2 * ((e % 128) / 32))); - } - - for (int sub = 0; sub < 2; sub++) { - const float D = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]); - const float M = GGML_FP16_TO_FP32(m_pl[sub * 32 + row]); - sub_scales[kt_local * 2 + sub] = GGML_FP32_TO_FP16((D > 0.0f) ? D : 0.0f); - sub_mins[kt_local * 2 + sub] = GGML_FP32_TO_FP16((-M > 0.0f) ? -M : 0.0f); - } - } +struct ggml_backend_hexagon_alloc_buffer_n_plan_item { + size_t size; + int first; + int last; +}; - int ls[16]; - int lm[16]; - ggml_half * const dd[2] = { &b->d, &b->dmin }; - const ggml_half * const prod[2] = { sub_scales, sub_mins }; - int * const ll[2] = { ls, lm }; - for (int w = 0; w < 2; w++) { - if (hexagon_recover_k_scales(prod[w], 16, 0, 15, dd[w], ll[w])) { - continue; - } - float max_val = 0.0f; - for (int j = 0; j < 16; j++) { - max_val = (std::max)(max_val, GGML_FP16_TO_FP32(prod[w][j])); - } - *dd[w] = GGML_FP32_TO_FP16(max_val / 15.0f); - const float d_actual = GGML_FP16_TO_FP32(*dd[w]); - const float inv_d = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f; - for (int j = 0; j < 16; j++) { - ll[w][j] = (std::min)(15, (int) roundf(inv_d * GGML_FP16_TO_FP32(prod[w][j]))); - } - } +using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector<ggml_backend_hexagon_alloc_buffer_n_plan_item>; - for (int j = 0; j < 16; j++) { - b->scales[j] = (uint8_t) (ls[j] | (lm[j] << 4)); - } - } - } - } +static const char * ggml_hexagon_kv_layer_suffix(const struct ggml_tensor * t) { + if (strncmp(t->name, "cache_", 6) != 0) { + return NULL; } - - GGML_UNUSED(size); -} - -static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { - if (j < 4) { - *d = q[j] & 63; - *m = q[j + 4] & 63; - } else { - *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); - *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + const char * p = strstr(t->name, "_l"); + if (!p || !isdigit((unsigned char)p[2])) { + return NULL; } + return p; } -// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q4_1 in htp/matmul-ops.h -static void repack_q4_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); +struct ggml_backend_hexagon_alloc_unit { + size_t size; + int first; + int last; +}; - const block_q4_K * src_matrix = (const block_q4_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); +static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buffer_n_plan( + ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + ggml_backend_hexagon_alloc_buffer_n_plan_t plan; - GGML_ASSERT(ne0 % QK_K == 0); + const size_t alignment = ggml_backend_buft_get_alignment(buft); + const size_t max_size = opt_mbuf_static > 0 ? opt_mbuf_static : SIZE_MAX; - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + std::vector<ggml_backend_hexagon_alloc_unit> units; - const int64_t sb_per_row = ne0 / QK_K; + int i = 0; + while (i < n_tensors) { + struct ggml_tensor * t = tensors[i]; + size_t unit_size = 0; + int unit_first = i; + int unit_last = i + 1; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_q4_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + if (t->data == NULL && t->view_src == NULL) { + unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment); + } - memset(matrix_dst, 0, matrix_size); + const char * layer_suffix = ggml_hexagon_kv_layer_suffix(t); - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - const block_q4_K * src_row = src_slice + r * sb_per_row; + while (unit_last < n_tensors) { + struct ggml_tensor * next = tensors[unit_last]; - for (int kt = 0; kt < n_k_tiles; kt++) { - const int kt_local = kt % 8; - const block_q4_K * b = &src_row[kt / 8]; - const float d = GGML_FP16_TO_FP32(b->d); - const float dmin = GGML_FP16_TO_FP32(b->dmin); + if (next->view_src != NULL) { + unit_last++; + continue; + } - uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + if (layer_suffix != NULL) { + const char * next_suffix = ggml_hexagon_kv_layer_suffix(next); + if (next_suffix != NULL && strcmp(layer_suffix, next_suffix) == 0) { + if (next->data == NULL) { + unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment); + } + unit_last++; + continue; + } + } - uint8_t sc, m; - get_scale_min_k4(kt_local, b->scales, &sc, &m); + break; + } - const float D = d * (float) sc; - const float M = -dmin * (float) m; + units.push_back({ unit_size, unit_first, unit_last }); + i = unit_last; + } - const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; - const int shift = (kt_local & 1) ? 4 : 0; + size_t cur_buf_size = 0; + int cur_buf_first = 0; - for (int cp = 0; cp < 16; cp++) { - const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F; - const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F; - tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0); - } + for (const auto & unit : units) { + if (unit.size == 0) { + continue; + } - ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); - scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(D); - scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(M); - } - } + if (cur_buf_size > 0 && (cur_buf_size + unit.size) > max_size) { + plan.push_back({ cur_buf_size, cur_buf_first, unit.first }); + cur_buf_size = 0; + cur_buf_first = unit.first; } + + cur_buf_size += unit.size; + } + + if (cur_buf_size > 0) { + plan.push_back({ cur_buf_size, cur_buf_first, n_tensors }); } - GGML_UNUSED(size); + return plan; } -// Reverse of repack_q4_K_tiled. Unpacks quants and normalizes scales/mins. Read-back only. -static void repack_tiled_q4_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); - - block_q4_K * dst_matrix = (block_q4_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - GGML_ASSERT(ne0 % QK_K == 0); - - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; - - const int64_t sb_per_row = ne0 / QK_K; - - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_q4_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; - - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - block_q4_K * dst_row = dst_slice + r * sb_per_row; - - for (int64_t sb = 0; sb < sb_per_row; sb++) { - block_q4_K * b = &dst_row[sb]; - memset(b, 0, sizeof(block_q4_K)); - - float sub_scales[8]; - float sub_mins[8]; - - for (int kt_local = 0; kt_local < 8; kt_local++) { - const int kt = sb * 8 + kt_local; - const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; - const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); - - uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; - const int shift = (kt_local & 1) ? 4 : 0; - - for (int cp = 0; cp < 16; cp++) { - const uint8_t val = tile_src[cp * 32 + row]; - const uint8_t q0 = val & 0x0F; - const uint8_t q1 = val >> 4; - qs_sub[2 * cp + 0] |= (uint8_t) (q0 << shift); - qs_sub[2 * cp + 1] |= (uint8_t) (q1 << shift); - } +static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n( + ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + const ggml_backend_hexagon_alloc_buffer_n_plan_t plan = ggml_backend_hexagon_alloc_buffer_n_plan(buft, tensors, n_tensors); - const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]); - const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]); - sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f; - sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f; - } + std::vector<ggml_backend_buffer_t> buffers; + buffers.reserve(plan.size()); - float max_scale = 0.0f; - float max_min = 0.0f; - for (int j = 0; j < 8; j++) { - if (sub_scales[j] > max_scale) max_scale = sub_scales[j]; - if (sub_mins[j] > max_min) max_min = sub_mins[j]; - } + for (const auto & item : plan) { + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, item.size); + if (buffer == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), item.size); + for (ggml_backend_buffer_t b : buffers) { + ggml_backend_buffer_free(b); + } + return NULL; + } - float inv_scale = 0.0f; - if (max_scale > 0.0f) { - b->d = GGML_FP32_TO_FP16(max_scale / 63.0f); - const float d_actual = GGML_FP16_TO_FP32(b->d); - inv_scale = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f; - } else { - b->d = GGML_FP32_TO_FP16(0.0f); - } + struct ggml_tallocr tallocr = ggml_tallocr_new(buffer); - float inv_min = 0.0f; - if (max_min > 0.0f) { - b->dmin = GGML_FP32_TO_FP16(max_min / 63.0f); - const float dmin_actual = GGML_FP16_TO_FP32(b->dmin); - inv_min = (dmin_actual > 0.0f) ? (1.0f / dmin_actual) : 0.0f; - } else { - b->dmin = GGML_FP32_TO_FP16(0.0f); + struct ggml_tensor * t_failed = NULL; + for (int j = item.first; j < item.last; j++) { + struct ggml_tensor * t = tensors[j]; + if (t->data == NULL) { + if (t->view_src == NULL) { + if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; } - - for (int j = 0; j < 8; j++) { - uint8_t ls = (uint8_t) roundf(inv_scale * sub_scales[j]); - uint8_t lm = (uint8_t) roundf(inv_min * sub_mins[j]); - ls = (std::min)((uint8_t) 63, ls); - lm = (std::min)((uint8_t) 63, lm); - if (j < 4) { - b->scales[j] = ls; - b->scales[j + 4] = lm; - } else { - b->scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); - b->scales[j - 4] |= ((ls >> 4) << 6); - b->scales[j - 0] |= ((lm >> 4) << 6); - } + } else if (t->buffer == NULL) { + if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; + } + } + } else { + if (t->view_src != NULL && t->buffer == NULL) { + if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; } } } } + if (t_failed != NULL) { + GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name); + for (ggml_backend_buffer_t b : buffers) { + ggml_backend_buffer_free(b); + } + ggml_backend_buffer_free(buffer); + return NULL; + } + + buffers.push_back(buffer); } - GGML_UNUSED(size); -} + if (buffers.empty()) { + return NULL; + } -// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q5_K in htp/matmul-ops.h -static void repack_q5_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); + if (buffers.size() == 1) { + return buffers[0]; + } - const block_q5_K * src_matrix = (const block_q5_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); + return ggml_backend_multi_buffer_alloc_buffer(buffers.data(), buffers.size()); +} - GGML_ASSERT(ne0 % QK_K == 0); +static size_t ggml_backend_hexagon_buffer_type_get_alloc_size_n( + ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + const ggml_backend_hexagon_alloc_buffer_n_plan_t plan = ggml_backend_hexagon_alloc_buffer_n_plan(buft, tensors, n_tensors); - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q5_K; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + size_t total = 0; + for (const auto & item : plan) { + total += item.size; + } + return total; +} - const int64_t sb_per_row = ne0 / QK_K; +static ggml_backend_buffer_type_i ggml_backend_hexagon_buffer_type_interface = { + /* .get_name = */ ggml_backend_hexagon_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_hexagon_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ ggml_backend_hexagon_buffer_type_alloc_buffer_n, + /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ ggml_backend_hexagon_buffer_type_get_alloc_size_n, + /* .is_host = */ ggml_backend_hexagon_buffer_type_is_host, +}; - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - const block_q5_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; +static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interface = { + /* .get_name = */ ggml_backend_hexagon_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_hexagon_host_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ ggml_backend_hexagon_buffer_type_alloc_buffer_n, + /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ ggml_backend_hexagon_buffer_type_get_alloc_size_n, + /* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host, +}; - memset(matrix_dst, 0, matrix_size); +ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) + : dev_id(dev_id), config(config), dev(dev) { + buffer_type.device = dev; + buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; + buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name, this); - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - const block_q5_K * src_row = src_slice + r * sb_per_row; + host_buffer_type.device = dev; + host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; + host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-HOST", this); - for (int kt = 0; kt < n_k_tiles; kt++) { - const int kt_local = kt % 8; - const block_q5_K * b = &src_row[kt / 8]; - const float d = GGML_FP16_TO_FP32(b->d); - const float dmin = GGML_FP16_TO_FP32(b->dmin); + fence_buffer_type.device = dev; + fence_buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; + fence_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-FENCE", this); +} - uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; - uint8_t * plane = tile_dst + 640; +ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() { + delete static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type.context); + delete static_cast<ggml_backend_hexagon_buffer_type_context *>(host_buffer_type.context); + delete static_cast<ggml_backend_hexagon_buffer_type_context *>(fence_buffer_type.context); +} - uint8_t sc, m; - get_scale_min_k4(kt_local, b->scales, &sc, &m); +static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { + return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment; +} - const float D = d * (float) sc; - const float M = -dmin * (float) m; +struct ggml_hexagon_opbatch { + ggml_hexagon_session* sess; - const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; - const int shift = (kt_local & 1) ? 4 : 0; - const uint8_t hbit = (uint8_t) (1 << kt_local); + std::vector<htp_opnode> ops; // htp_opnode of ops - for (int cp = 0; cp < 16; cp++) { - const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F; - const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F; - tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0); + std::vector<htp_buf_desc> h_bufs; // htp buffer descriptors + std::vector<htp_tensor> h_tens; // htp tensor descriptors + std::vector<htp_op_desc> h_ops; // htp op descriptors - const int i = cp / 4; - const int lane = (cp % 4) * 32 + row; - if (b->qh[2 * cp + 0] & hbit) { - plane[lane] |= (uint8_t) (1 << (2 * i)); - } - if (b->qh[2 * cp + 1] & hbit) { - plane[lane] |= (uint8_t) (1 << (2 * i + 1)); - } - } + std::unordered_map<uint64_t, int> t_map; // tensor hash to index - ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); - scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(D); - scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(M); - } - } - } - } + unsigned int n_bufs; // num buffers in the batch + unsigned int n_tens; // num tensors ... + unsigned int n_ops; // num ops ... + size_t b_vmem; // sum of non-extended buffer sizes - GGML_UNUSED(size); -} + unsigned int n_bufs_max; + unsigned int n_tens_max; + unsigned int n_ops_max; + size_t b_vmem_max; -// Reverse of repack_q5_K_tiled. Unpacks quants losslessly and normalizes scales/mins. Read-back only. -static void repack_tiled_q5_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { - GGML_ASSERT(offset == 0); - - block_q5_K * dst_matrix = (block_q5_K *) data; - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - int64_t ne0_padded = hex_round_up(ne0, 32); - int64_t ne1_padded = hex_round_up(ne1, 32); - - GGML_ASSERT(ne0 % QK_K == 0); - - const int n_col_tiles = ne1_padded / 32; - const int n_k_tiles = ne0_padded / 32; - const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q5_K; - const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; - - const int64_t sb_per_row = ne0 / QK_K; - - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = 0; i2 < ne2; i2++) { - block_q5_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); - const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; - - for (int64_t r = 0; r < ne1; r++) { - const int ct = (int) (r / 32); - const int row = (int) (r % 32); - block_q5_K * dst_row = dst_slice + r * sb_per_row; - - for (int64_t sb = 0; sb < sb_per_row; sb++) { - block_q5_K * b = &dst_row[sb]; - memset(b, 0, sizeof(block_q5_K)); - - float sub_scales[8]; - float sub_mins[8]; - - for (int kt_local = 0; kt_local < 8; kt_local++) { - const int kt = sb * 8 + kt_local; - const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; - const uint8_t * plane = tile_src + 640; - const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); - - uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; - const int shift = (kt_local & 1) ? 4 : 0; - const uint8_t hbit = (uint8_t) (1 << kt_local); - - for (int cp = 0; cp < 16; cp++) { - const uint8_t val = tile_src[cp * 32 + row]; - const uint8_t q0 = val & 0x0F; - const uint8_t q1 = val >> 4; - qs_sub[2 * cp + 0] |= (uint8_t) (q0 << shift); - qs_sub[2 * cp + 1] |= (uint8_t) (q1 << shift); - - const int i = cp / 4; - const int lane = (cp % 4) * 32 + row; - if (plane[lane] & (1 << (2 * i))) { - b->qh[2 * cp + 0] |= hbit; - } - if (plane[lane] & (1 << (2 * i + 1))) { - b->qh[2 * cp + 1] |= hbit; - } - } + void reset() { + n_bufs = 0; + n_tens = 0; + n_ops = 0; + b_vmem = 0; - const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]); - const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]); - sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f; - sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f; - } + t_map.clear(); + ops.resize(n_ops_max); + } - float max_scale = 0.0f; - float max_min = 0.0f; - for (int j = 0; j < 8; j++) { - if (sub_scales[j] > max_scale) max_scale = sub_scales[j]; - if (sub_mins[j] > max_min) max_min = sub_mins[j]; - } + ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) { + this->sess = sess; - float inv_scale = 0.0f; - if (max_scale > 0.0f) { - b->d = GGML_FP32_TO_FP16(max_scale / 63.0f); - const float d_actual = GGML_FP16_TO_FP32(b->d); - inv_scale = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f; - } else { - b->d = GGML_FP32_TO_FP16(0.0f); - } + n_bufs_max = HTP_OP_MAX_BUFS; + n_ops_max = batch_size; + n_tens_max = std::min<size_t>(n_ops_max * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS); - float inv_min = 0.0f; - if (max_min > 0.0f) { - b->dmin = GGML_FP32_TO_FP16(max_min / 63.0f); - const float dmin_actual = GGML_FP16_TO_FP32(b->dmin); - inv_min = (dmin_actual > 0.0f) ? (1.0f / dmin_actual) : 0.0f; - } else { - b->dmin = GGML_FP32_TO_FP16(0.0f); - } + b_vmem_max = max_vmem; - for (int j = 0; j < 8; j++) { - uint8_t ls = (uint8_t) roundf(inv_scale * sub_scales[j]); - uint8_t lm = (uint8_t) roundf(inv_min * sub_mins[j]); - ls = (std::min)((uint8_t) 63, ls); - lm = (std::min)((uint8_t) 63, lm); - if (j < 4) { - b->scales[j] = ls; - b->scales[j + 4] = lm; - } else { - b->scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); - b->scales[j - 4] |= ((ls >> 4) << 6); - b->scales[j - 0] |= ((lm >> 4) << 6); - } - } - } - } - } - } + ops.resize(n_ops_max); - GGML_UNUSED(size); -} + h_bufs.resize(n_bufs_max); + h_tens.resize(n_tens_max); + h_ops.resize(n_ops_max); -static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) { - switch (tensor->type) { - case GGML_TYPE_Q4_0: - repack_q4_0_tiled(tensor, data, 0, size); - break; + t_map.reserve(n_tens_max); - case GGML_TYPE_Q4_1: - repack_q4_1_tiled(tensor, data, 0, size); - break; + GGML_LOG_INFO("ggml-hex: %s op batching: n-bufs %u n-tensors %u n-ops %u vmem %zu\n", + sess->c_name(), n_bufs_max, n_tens_max, n_ops_max, b_vmem_max); - case GGML_TYPE_Q4_K: - repack_q4_K_tiled(tensor, data, 0, size); - break; + reset(); + } - case GGML_TYPE_Q8_0: - repack_q8_0_tiled(tensor, data, 0, size); - break; + bool empty() const { return n_ops == 0; } + bool full() const { return n_ops >= n_ops_max; } - case GGML_TYPE_IQ4_NL: - repack_q4_0_tiled(tensor, data, 0, size); - break; + int find_buffer(int fd) const { + for (unsigned int i = 0; i < n_bufs; i++) { + if ((int) h_bufs[i].fd == fd) { + return (int) i; + } + } + return -1; + } - case GGML_TYPE_MXFP4: - repack_mxfp4_tiled(tensor, data, 0, size); - break; + // add buffer and return its index + int add_buffer(ggml_hexagon_shared_buffer * sbuf) { + // Lookup by fd + int bi = find_buffer(sbuf->fd()); + if (bi >= 0) { return bi; } - case GGML_TYPE_Q5_K: - repack_q5_K_tiled(tensor, data, 0, size); - break; + // Add new buffer to the batch + GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS); + bi = n_bufs++; - case GGML_TYPE_Q6_K: - repack_q6_K_tiled(tensor, data, 0, size); - break; + htp_buf_desc &b = h_bufs[bi]; + b.base = (uint64_t) sbuf->base(); + b.fd = sbuf->fd(); + b.size = sbuf->size(); + b.flags = sbuf->extended ? HTP_BUF_EXTENDED : 0; - case GGML_TYPE_Q3_K: - repack_q3_K_tiled(tensor, data, 0, size); - break; + if (!sbuf->extended) { + b_vmem += b.size; + } - case GGML_TYPE_Q2_K: - repack_q2_K_tiled(tensor, data, 0, size); - break; + HEX_VERBOSE("ggml-hex: %s add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", sess->c_name(), bi, b.fd, (void*) sbuf->base(), (size_t) b.size, b_vmem); - default: - break; + return bi; } -} -static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, - ggml_tensor * tensor, - const void * data, - size_t offset, - size_t size) { - auto extra = (ggml_hexagon_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + static void init_tensor_shape(htp_tensor & h, const ggml_tensor * t) { + const uint32_t flags = ggml_hexagon_tensor_get_flags(t); - if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT; - if (ggml_hexagon_is_repack_type(tensor->type)) { - extra->flags |= GGML_HEXAGON_TENSOR_REPACK; + h.type = t->type; + h.ne[0] = t->ne[0]; + h.ne[1] = t->ne[1]; + h.ne[2] = t->ne[2]; + h.ne[3] = t->ne[3]; + + if (flags & HTP_TENSOR_TILED) { + if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16) { + h.type = GGML_TYPE_F16; + h.nb[0] = sizeof(ggml_fp16_t); + } else { + h.nb[0] = t->nb[0]; + } + const int64_t ne0_padded = hex_round_up(t->ne[0], 32); + const int64_t ne1_padded = hex_round_up(t->ne[1], 32); + h.nb[1] = htp_tiled_row_size(t->type, ne0_padded); + h.nb[2] = h.nb[1] * ne1_padded; + h.nb[3] = h.nb[2] * h.ne[2]; + h.size = h.nb[3] * h.ne[3]; + } else { + h.nb[0] = t->nb[0]; + h.nb[1] = t->nb[1]; + h.nb[2] = t->nb[2]; + h.nb[3] = t->nb[3]; + h.size = ggml_nbytes(t); } + + h.flags = flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE); } - HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); + static uint64_t hash_tensor(const ggml_tensor * t, const htp_tensor & shape) { + uint64_t hash = 1469598103934665603ull; + auto hash_value = [&hash](uint64_t value) { + hash ^= value; + hash *= 1099511628211ull; + }; - if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) { - memcpy((char *) tensor->data + offset, data, size); - return; - } + hash_value((uintptr_t) t->data); + hash_value(shape.type); + hash_value(shape.size); + hash_value(shape.flags); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + hash_value(shape.ne[i]); + hash_value(shape.nb[i]); + } - if (offset == 0 && size == ggml_nbytes(tensor) && extra->shadow_buf.empty()) { - repack_tensor_tiled(tensor, data, size); - return; - } + // avalanche high bits down to low bits for power-of-two table indexing + hash ^= hash >> 33; + hash *= 0xff51afd7ed558ccdull; + hash ^= hash >> 33; - if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { - extra->shadow_buf.resize(ggml_nbytes(tensor)); + return hash; } - memcpy(extra->shadow_buf.data() + offset, data, size); - extra->shadow_size += size; - if (extra->shadow_size >= ggml_nbytes(tensor)) { - repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); - extra->shadow_buf.clear(); - extra->shadow_buf.shrink_to_fit(); - extra->shadow_size = 0; - } -} + // add tensor and return its index + int add_tensor(const ggml_tensor * t) { + htp_tensor shape {}; + init_tensor_shape(shape, t); -static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, - const ggml_tensor * tensor, - void * data, - size_t offset, - size_t size) { - auto extra = (ggml_hexagon_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + const uint64_t hash = hash_tensor(t, shape); + auto it = t_map.find(hash); + if (it != t_map.end()) { + return it->second; + } - HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); + auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) { - memcpy(data, (const char *) tensor->data + offset, size); - return; - } + // Add new tensor to the batch + int ti = n_tens++; + GGML_ASSERT(n_tens <= n_tens_max); - switch (tensor->type) { - case GGML_TYPE_Q4_0: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q4_0(data, tensor, offset, size); - break; + t_map[hash] = ti; - case GGML_TYPE_Q4_1: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q4_1(data, tensor, offset, size); - break; + uint64_t t_offset = (uint8_t *) t->data - sbuf->base(); - case GGML_TYPE_Q4_K: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q4_K(data, tensor, offset, size); - break; + htp_tensor & h = h_tens[ti]; + h = shape; + h.bi = add_buffer(sbuf); + h.ti = ti; + h.data = t_offset; - case GGML_TYPE_Q5_K: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q5_K(data, tensor, offset, size); - break; + HEX_VERBOSE("ggml-hex: %s add-tensor #%u %s : bi %d data %p offset %zu size %zu flags 0x%x : %zu:%zu:%zu:%zu\n", sess->c_name(), + ti, t->name, h.bi, (void*) t->data, (size_t) t_offset, (size_t) h.size, h.flags, + (size_t) h.ne[0], (size_t) h.ne[1], (size_t) h.ne[2], (size_t) h.ne[3]); - case GGML_TYPE_Q8_0: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q8_0(data, tensor, offset, size); - break; + return ti; + } - case GGML_TYPE_IQ4_NL: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q4_0(data, tensor, offset, size); - break; + bool fit_tensors(const htp_opnode & node) const { + // check how much extras we will need + size_t extra_bufs = 0; + size_t extra_vmem = 0; + size_t extra_tens = 0; - case GGML_TYPE_MXFP4: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_mxfp4(data, tensor, offset, size); - break; + const bool check_tens = (n_tens + node.inputs.size() + node.outputs.size()) > n_tens_max; - case GGML_TYPE_Q6_K: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q6_K(data, tensor, offset, size); - break; + int seen_bufs[HTP_OP_MAX_BUFS]; + int n_seen_bufs = 0; - case GGML_TYPE_Q3_K: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q3_K(data, tensor, offset, size); - break; + auto fit_tensor = [&](const ggml_tensor * t) { + if (!t) return; - case GGML_TYPE_Q2_K: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_tiled_q2_K(data, tensor, offset, size); - break; + if (check_tens) { + htp_tensor shape {}; + init_tensor_shape(shape, t); + if (t_map.count(hash_tensor(t, shape))) return; + extra_tens++; + } - default: - memcpy(data, (const char *) tensor->data + offset, size); - break; + auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); + int fd = sbuf->fd(); + if (find_buffer(fd) < 0) { + for (int i = 0; i < n_seen_bufs; i++) { + if (seen_bufs[i] == fd) return; + } + if (n_seen_bufs < HTP_OP_MAX_BUFS) { + seen_bufs[n_seen_bufs++] = fd; + } + if (!sbuf->extended) { + extra_vmem += sbuf->size(); + } + extra_bufs += 1; + } + }; + + for (const auto * src : node.get_inputs()) { + fit_tensor(src); + } + for (const auto * output : node.get_outputs()) { + fit_tensor(output); + } + + if ((extra_bufs + n_bufs) > n_bufs_max) return false; + if (check_tens && (extra_tens + n_tens) > n_tens_max) return false; + if ((extra_vmem + b_vmem) > b_vmem_max) return false; + + return true; } -} -static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer, - const struct ggml_tensor * src, - struct ggml_tensor * dst) { - // we might optimize this later, for now take the slow path (ie get/set_tensor) - return false; + void sync_op_desc(unsigned int n) { + const htp_opnode & node = ops[n]; + htp_op_desc & o = h_ops[n]; - GGML_UNUSED(buffer); - GGML_UNUSED(src); - GGML_UNUSED(dst); -} + memcpy(o.params, node.params, sizeof(o.params)); + memcpy(o.kernel_params, node.kernel_params, sizeof(o.kernel_params)); + o.opcode = node.opcode; -static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, - ggml_tensor * tensor, - const void * data, - size_t offset, - size_t size, - size_t n_copies, - size_t stride_tensor, - size_t stride_data) { - auto extra = (ggml_hexagon_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + const auto & inputs = node.get_inputs(); + for (unsigned int i = 0; i < HTP_OP_MAX_INPUTS; i++) { + o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff; + } - if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT; - if (ggml_hexagon_is_repack_type(tensor->type)) { - extra->flags |= GGML_HEXAGON_TENSOR_REPACK; + const auto & outputs = node.get_outputs(); + for (unsigned int i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { + o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff; } } - HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags); + // caller must ensure the op fits + void add_op(const htp_opnode & node) { + unsigned int n = n_ops++; + GGML_ASSERT(n_ops <= n_ops_max); - if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) { - for (size_t i = 0; i < n_copies; i++) { - memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); - } - return; - } + ops[n] = node; + h_ops[n].flags = 0; - if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { - extra->shadow_buf.resize(ggml_nbytes(tensor)); + ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], h_ops[n].flags); + sync_op_desc(n); } - for (size_t i = 0; i < n_copies; i++) { - memcpy(extra->shadow_buf.data() + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); - } - extra->shadow_size += n_copies * size; - if (extra->shadow_size >= ggml_nbytes(tensor)) { - repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); - extra->shadow_buf.clear(); - extra->shadow_buf.shrink_to_fit(); - extra->shadow_size = 0; - } -} + void sort_buffers() { + if (n_bufs <= 1) return; -static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, - const ggml_tensor * tensor, - void * data, - size_t offset, - size_t size, - size_t n_copies, - size_t stride_tensor, - size_t stride_data) { - auto extra = (ggml_hexagon_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + std::vector<int> order(n_bufs); + for (unsigned int i = 0; i < n_bufs; i++) { order[i] = (int) i; } - HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", - sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage); + std::stable_sort(order.begin(), order.end(), [&](int a, int b) { + return h_bufs[a].size > h_bufs[b].size; + }); - if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) { - for (size_t i = 0; i < n_copies; i++) { - memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size); + bool already_sorted = true; + for (unsigned int i = 0; i < n_bufs; i++) { + if (order[i] != (int) i) { + already_sorted = false; + break; + } } - return; - } + if (already_sorted) return; - size_t temp_size = n_copies > 0 ? (n_copies - 1) * stride_tensor + size : 0; - size_t slice_size = tensor->ne[1] * ggml_row_size(tensor->type, tensor->ne[0]); - size_t slice_offset = offset % slice_size; - size_t row_size_bytes = ggml_row_size(tensor->type, tensor->ne[0]); + std::vector<uint16_t> remap(n_bufs); + std::vector<htp_buf_desc> sorted_bufs(n_bufs); + for (unsigned int new_bi = 0; new_bi < n_bufs; new_bi++) { + int old_bi = order[new_bi]; + remap[old_bi] = (uint16_t) new_bi; + sorted_bufs[new_bi] = h_bufs[old_bi]; + } - GGML_ASSERT((slice_offset % row_size_bytes) == 0 && "offset must be aligned to row boundary"); - GGML_ASSERT((temp_size % row_size_bytes) == 0 && "temp_size must be a multiple of row size"); - GGML_ASSERT((slice_offset / row_size_bytes) % 32 == 0 && "offset must be aligned to tile size (32 rows)"); - GGML_ASSERT((offset + temp_size) <= ggml_nbytes(tensor)); + for (unsigned int i = 0; i < n_bufs; i++) { + h_bufs[i] = sorted_bufs[i]; + } - std::vector<uint8_t> temp_buf(temp_size); + for (unsigned int i = 0; i < n_tens; i++) { + h_tens[i].bi = remap[h_tens[i].bi]; + } + } - switch (tensor->type) { - case GGML_TYPE_Q4_0: - repack_tiled_q4_0(temp_buf.data(), tensor, offset, temp_size); - break; + void update_mdev_group(uint32_t mdev_idx) { + if (n_ops > 0 && h_ops[0].opcode == HTP_OP_MDEV_GROUP) { + h_ops[0].params[0] = (int32_t) mdev_idx; + } + } - case GGML_TYPE_Q4_1: - repack_tiled_q4_1(temp_buf.data(), tensor, offset, temp_size); - break; + bool try_fuse(const htp_opnode & node) { + if (n_ops == 0) { + return false; + } - case GGML_TYPE_Q4_K: - repack_tiled_q4_K(temp_buf.data(), tensor, offset, temp_size); - break; + htp_opnode & last_node = ops[n_ops - 1]; + if (!htp_fusion_try_fuse(sess, last_node, node)) { + return false; + } - case GGML_TYPE_Q5_K: - repack_tiled_q5_K(temp_buf.data(), tensor, offset, temp_size); - break; + sync_op_desc(n_ops - 1); + HEX_VERBOSE("ggml-hex: %s fused %s (N=%zu, #%u)\n", + sess->c_name(), last_node.name().c_str(), last_node.fused.size() + 1, n_ops - 1); + return true; + } +}; - case GGML_TYPE_Q8_0: - repack_tiled_q8_0(temp_buf.data(), tensor, offset, temp_size); - break; +struct ggml_hexagon_registry { + ggml_hexagon_registry(ggml_backend_reg_t reg); + ~ggml_hexagon_registry(); - case GGML_TYPE_IQ4_NL: - repack_tiled_q4_0(temp_buf.data(), tensor, offset, temp_size); - break; + ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS]; +}; - case GGML_TYPE_MXFP4: - repack_tiled_mxfp4(temp_buf.data(), tensor, offset, temp_size); - break; +struct ggml_hexagon_opqueue { + // Shared buffer for storing batches + ggml_hexagon_shared_buffer *shm_buf; + size_t shm_blk_size; + size_t depth; - case GGML_TYPE_Q6_K: - repack_tiled_q6_K(temp_buf.data(), tensor, offset, temp_size); - break; + using opvec = std::vector<htp_opnode>; - case GGML_TYPE_Q3_K: - repack_tiled_q3_K(temp_buf.data(), tensor, offset, temp_size); - break; - - case GGML_TYPE_Q2_K: - repack_tiled_q2_K(temp_buf.data(), tensor, offset, temp_size); - break; - - default: - memcpy(temp_buf.data(), (const uint8_t *) tensor->data + offset, temp_size); - break; - } - - for (size_t i = 0; i < n_copies; i++) { - memcpy((uint8_t *) data + i * stride_data, temp_buf.data() + i * stride_tensor, size); - } -} - -static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; - HEX_VERBOSE("ggml-hex: %s clear-buff base %p size %zu\n", sess->c_name(), (void *) sbuf->base(), sbuf->size()); - memset(sbuf->base(), value, sbuf->size()); -} - -static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { - /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, - /* .get_base = */ ggml_backend_hexagon_buffer_get_base, - /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, - /* .memset_tensor = */ NULL, - /* .set_tensor = */ ggml_backend_hexagon_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_hexagon_buffer_get_tensor, - /* .set_tensor_2d = */ ggml_backend_hexagon_buffer_set_tensor_2d, - /* .get_tensor_2d = */ ggml_backend_hexagon_buffer_get_tensor_2d, - /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, - /* .clear = */ ggml_backend_hexagon_buffer_clear, - /* .reset = */ NULL, -}; - -// ** backend buffer type - -static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t buffer, - ggml_tensor * tensor, - const void * data, - size_t offset, - size_t size) { - memcpy((char *) tensor->data + offset, data, size); - GGML_UNUSED(buffer); -} - -static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer, - const ggml_tensor * tensor, - void * data, - size_t offset, - size_t size) { - memcpy(data, (const char *) tensor->data + offset, size); - GGML_UNUSED(buffer); -} - -static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = { - /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, - /* .get_base = */ ggml_backend_hexagon_buffer_get_base, - /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, - /* .memset_tensor = */ NULL, - /* .set_tensor = */ ggml_backend_hexagon_host_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_hexagon_host_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, - /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, - /* .clear = */ ggml_backend_hexagon_buffer_clear, - /* .reset = */ NULL, -}; - -// ** backend buffer type - -static const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_type_t buffer_type) { - return static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->name.c_str(); -} - -static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( - ggml_backend_buffer_type_t buffer_type, size_t size) { - auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx; - auto sess = dev_ctx->session(); - if (sess && sess->max_vmem && size > sess->max_vmem) { - GGML_LOG_ERROR("ggml-hex: %s buffer size %zu exceeds max_vmem %zu\n", - dev_ctx->c_name(), size, sess->max_vmem); - return nullptr; - } - try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); - return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); - } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", dev_ctx->c_name(), exc.what()); - return nullptr; - } -} - -static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( - ggml_backend_buffer_type_t buffer_type, size_t size) { - auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx; - auto sess = dev_ctx->session(); - if (sess && sess->max_vmem && size > sess->max_vmem) { - GGML_LOG_ERROR("ggml-hex: %s host buffer size %zu exceeds max_vmem %zu\n", - dev_ctx->c_name(), size, sess->max_vmem); - return nullptr; - } - try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); - return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size); - } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", dev_ctx->c_name(), exc.what()); - return nullptr; - } -} - -static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return 128; // HVX alignment - GGML_UNUSED(buft); -} - -static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) { - if (ggml_hexagon_is_repack_type(t->type)) { - int64_t ne0 = hex_round_up(t->ne[0], 32); - int64_t ne1 = hex_round_up(t->ne[1], 32); - int64_t ne2 = t->ne[2]; - int64_t ne3 = t->ne[3]; - return ggml_hexagon_tiled_row_size(t->type, ne0) * ne1 * ne2 * ne3; - } - return ggml_nbytes(t); - - GGML_UNUSED(buft); -} - -static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - return opt_mbuf; - GGML_UNUSED(buft); -} - -static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return false; - GGML_UNUSED(buft); -} - -static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return true; - GGML_UNUSED(buft); -} - -static ggml_backend_buffer_type_i ggml_backend_hexagon_buffer_type_interface = { - /* .get_name = */ ggml_backend_hexagon_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_hexagon_buffer_type_alloc_buffer, - /* .alloc_buffer_n = */ NULL, - /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, - /* .get_alloc_size_n = */ NULL, - /* .is_host = */ ggml_backend_hexagon_buffer_type_is_host, -}; - -static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interface = { - /* .get_name = */ ggml_backend_hexagon_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_hexagon_host_buffer_type_alloc_buffer, - /* .alloc_buffer_n = */ NULL, - /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, - /* .get_alloc_size_n = */ NULL, - /* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host, -}; - -ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) - : dev_id(dev_id), config(config), dev(dev) { - buffer_type.device = dev; - buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; - buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name, this); - - host_buffer_type.device = dev; - host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; - host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-HOST", this); - - fence_buffer_type.device = dev; - fence_buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; - fence_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-FENCE", this); -} - -ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() { - delete static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type.context); - delete static_cast<ggml_backend_hexagon_buffer_type_context *>(host_buffer_type.context); - delete static_cast<ggml_backend_hexagon_buffer_type_context *>(fence_buffer_type.context); -} - -static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { - return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment; -} - -struct ggml_hexagon_opbatch { - ggml_hexagon_session* sess; - - std::vector<htp_opnode> ops; // htp_opnode of ops - - std::vector<htp_buf_desc> h_bufs; // htp buffer descriptors - std::vector<htp_tensor> h_tens; // htp tensor descriptors - std::vector<htp_op_desc> h_ops; // htp op descriptors - - std::unordered_map<int, int> b_map; // buffer fd to index - std::unordered_map<const ggml_tensor*, int> t_map; // tensor ptr to index - std::unordered_multimap<void*, int> d_map; // tensor data to index - - unsigned int n_bufs; // num buffers in the batch - unsigned int n_tens; // num tensors ... - unsigned int n_ops; // num ops ... - size_t b_vmem; // sum of non-extended buffer sizes - - unsigned int n_bufs_max; - unsigned int n_tens_max; - unsigned int n_ops_max; - size_t b_vmem_max; - - void reset() { - n_bufs = 0; - n_tens = 0; - n_ops = 0; - b_vmem = 0; - - b_map.clear(); - t_map.clear(); - d_map.clear(); - ops.resize(n_ops_max); - } - - ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) { - this->sess = sess; - - n_bufs_max = HTP_OP_MAX_BUFS; - n_ops_max = batch_size; - n_tens_max = std::min<size_t>(n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS, HTP_OP_MAX_TENSORS); - - b_vmem_max = max_vmem; - - ops.resize(n_ops_max); - - h_bufs.resize(n_bufs_max); - h_tens.resize(n_tens_max); - h_ops.resize(n_ops_max); - - b_map.reserve(n_bufs_max); - t_map.reserve(n_tens_max); - d_map.reserve(n_tens_max); - - GGML_LOG_INFO("ggml-hex: %s op batching: n-bufs %u n-tensors %u n-ops %u vmem %zu\n", - sess->c_name(), n_bufs_max, n_tens_max, n_ops_max, b_vmem_max); - - reset(); - } - - bool empty() const { return n_ops == 0; } - - // add buffer and return its index - int add_buffer(ggml_hexagon_shared_buffer * sbuf) { - // Lookup by fd - auto it = b_map.find(sbuf->fd()); - if (it != b_map.end()) { return it->second; } - - // Add new buffer to the batch - GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS); - int bi = n_bufs++; - - b_map.insert({sbuf->fd(), bi}); - - htp_buf_desc &b = h_bufs[bi]; - b.base = (uint64_t) sbuf->base(); - b.fd = sbuf->fd(); - b.size = sbuf->size(); - b.flags = sbuf->extended ? HTP_BUF_EXTENDED : 0; - - if (!sbuf->extended) { - b_vmem += b.size; - } - - HEX_VERBOSE("ggml-hex: %s add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", sess->c_name(), bi, b.fd, (void*) sbuf->base(), (size_t) b.size, b_vmem); - - return bi; - } - - bool same_shape(const htp_tensor * h, const ggml_tensor * t) const { - auto extra = (ggml_hexagon_tensor_extra *) t->extra; - - int64_t ne0 = t->ne[0]; - int64_t ne1 = t->ne[1]; - const bool is_repack = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0; - if (is_repack) { - ne0 = hex_round_up(ne0, 32); - ne1 = hex_round_up(ne1, 32); - } - int64_t nb1 = is_repack ? (int64_t) ggml_hexagon_tiled_row_size(t->type, ne0) : t->nb[1]; - int64_t nb2 = is_repack ? nb1 * ne1 : t->nb[2]; - int64_t nb3 = is_repack ? nb2 * t->ne[2] : t->nb[3]; - - return (h->type == t->type) && - (h->ne[0] == ne0) && (h->ne[1] == ne1) && (h->ne[2] == t->ne[2]) && (h->ne[3] == t->ne[3]) && - (h->nb[0] == t->nb[0]) && (h->nb[1] == nb1) && (h->nb[2] == nb2) && (h->nb[3] == nb3); - } - - // add tensor and return its index - int add_tensor(const ggml_tensor * t) { - auto extra = (ggml_hexagon_tensor_extra *) t->extra; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - - // First lookup by tensor data - auto range = d_map.equal_range(t->data); - for (auto it = range.first; it != range.second; ++it) { - htp_tensor * h = &h_tens[it->second]; - if (same_shape(h, t)) { return it->second; } - } - - // Lookup by tensor ptr - auto it = t_map.find(t); - if (it != t_map.end()) { return it->second; } - - // Add new tensor to the batch - int ti = n_tens++; - GGML_ASSERT(n_tens <= n_tens_max); - - t_map.insert({t, ti}); - d_map.insert({t->data, ti}); - - uint64_t t_offset = (uint8_t *) t->data - sbuf->base(); - size_t t_size = ggml_nbytes(t); - - htp_tensor &h = h_tens[ti]; - h.bi = add_buffer(sbuf); - h.ti = ti; - h.data = t_offset; - h.type = t->type; - - const bool is_repack = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0; - if (is_repack) { - h.ne[0] = hex_round_up(t->ne[0], 32); - h.ne[1] = hex_round_up(t->ne[1], 32); - h.ne[2] = t->ne[2]; - h.ne[3] = t->ne[3]; - - h.nb[0] = t->nb[0]; - h.nb[1] = ggml_hexagon_tiled_row_size(t->type, h.ne[0]); - h.nb[2] = h.nb[1] * h.ne[1]; - h.nb[3] = h.nb[2] * h.ne[2]; - h.size = h.nb[3] * h.ne[3]; - t_size = h.size; - } else { - h.size = t_size; - h.ne[0] = t->ne[0]; h.ne[1] = t->ne[1]; h.ne[2] = t->ne[2]; h.ne[3] = t->ne[3]; - h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3]; - } - - h.flags = 0; - if ((extra->flags & GGML_HEXAGON_TENSOR_WEIGHT) != 0) { - h.flags |= HTP_TENSOR_WEIGHT; - } - if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0) { - h.flags |= HTP_TENSOR_REPACK; - } - if ((extra->flags & GGML_HEXAGON_TENSOR_FENCE) != 0) { - h.flags |= HTP_TENSOR_FENCE; - } - - HEX_VERBOSE("ggml-hex: %s add-tensor #%u %s : bi %d data %p offset %zu size %zu flags 0x%x : %zu:%zu:%zu:%zu\n", sess->c_name(), - ti, t->name, h.bi, (void*) t->data, (size_t) t_offset, t_size, h.flags, - (size_t) h.ne[0], (size_t) h.ne[1], (size_t) h.ne[2], (size_t) h.ne[3]); - - return ti; - } - - bool fit_op(const htp_opnode & node) const { - if (n_ops >= n_ops_max) return false; - - // check how much extras we will need - size_t extra_bufs = 0; - size_t extra_vmem = 0; - size_t extra_tens = 0; - - int seen_bufs[HTP_OP_MAX_BUFS]; - int n_seen_bufs = 0; - - auto fit_tensor = [&](const ggml_tensor *t) { - if (!t) return; - if (!t_map.count(t)) { - extra_tens++; - - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - int fd = sbuf->fd(); - if (!b_map.count(fd)) { - for (int i = 0; i < n_seen_bufs; i++) { - if (seen_bufs[i] == fd) return; - } - if (n_seen_bufs < HTP_OP_MAX_BUFS) { - seen_bufs[n_seen_bufs++] = fd; - } - if (!sbuf->extended) { - extra_vmem += sbuf->size(); - } - extra_bufs += 1; - } - } - }; - - for (const auto * src : node.get_inputs()) { - fit_tensor(src); - } - for (const auto * output : node.get_outputs()) { - fit_tensor(output); - } - - if ((extra_bufs + n_bufs) > n_bufs_max) return false; - if ((extra_tens + n_tens) > n_tens_max) return false; - if ((extra_vmem + b_vmem) > b_vmem_max) return false; - - return true; - } - - // assumes that fit_op() was called first and returned true - void add_op(const htp_opnode & node) { - // Add new op - - unsigned int n = n_ops++; - GGML_ASSERT(n_ops <= n_ops_max); - - ops[n] = node; - - htp_op_desc &o = h_ops[n]; - memcpy(o.params, node.node->op_params, sizeof(node.node->op_params)); - memcpy(o.kernel_params, node.kernel_params, sizeof(o.kernel_params)); - o.opcode = node.opcode; - o.flags = 0; - - ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], o.flags); - - auto inputs = node.get_inputs(); - for (unsigned int i=0; i < HTP_OP_MAX_INPUTS; i++) { - o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff; - } - - auto outputs = node.get_outputs(); - for (unsigned int i=0; i < HTP_OP_MAX_OUTPUTS; i++) { - o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff; - } - } - - void sort_buffers() { - if (n_bufs <= 1) return; - - std::vector<int> order(n_bufs); - for (unsigned int i = 0; i < n_bufs; i++) { order[i] = (int) i; } - - std::stable_sort(order.begin(), order.end(), [&](int a, int b) { - return h_bufs[a].size > h_bufs[b].size; - }); - - bool already_sorted = true; - for (unsigned int i = 0; i < n_bufs; i++) { - if (order[i] != (int) i) { - already_sorted = false; - break; - } - } - if (already_sorted) return; - - std::vector<uint16_t> remap(n_bufs); - std::vector<htp_buf_desc> sorted_bufs(n_bufs); - for (unsigned int new_bi = 0; new_bi < n_bufs; new_bi++) { - int old_bi = order[new_bi]; - remap[old_bi] = (uint16_t) new_bi; - sorted_bufs[new_bi] = h_bufs[old_bi]; - } - - for (unsigned int i = 0; i < n_bufs; i++) { - h_bufs[i] = sorted_bufs[i]; - } - - for (unsigned int i = 0; i < n_tens; i++) { - h_tens[i].bi = remap[h_tens[i].bi]; - } - } - - void update_mdev_group(uint32_t mdev_idx) { - if (n_ops > 0 && h_ops[0].opcode == HTP_OP_MDEV_GROUP) { - h_ops[0].params[0] = (int32_t) mdev_idx; - } - } - - bool try_fuse_common(std::initializer_list<const ggml_tensor *> tensors) const { - size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; - - int seen_bufs[HTP_OP_MAX_BUFS]; - int n_seen_bufs = 0; - - for (const auto * t : tensors) { - if (!t || t_map.count(t)) { - continue; - } - extra_tens++; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - int fd = sbuf->fd(); - if (!b_map.count(fd)) { - bool found = false; - for (int i = 0; i < n_seen_bufs; i++) { - if (seen_bufs[i] == fd) { - found = true; - break; - } - } - if (!found) { - if (n_seen_bufs < HTP_OP_MAX_BUFS) { - seen_bufs[n_seen_bufs++] = fd; - } - if (!sbuf->extended) { - extra_vmem += sbuf->size(); - } - extra_bufs += 1; - } - } - } - - if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { - return false; - } - - return true; - } - - bool try_fuse_common(const ggml_tensor * t1, const ggml_tensor * t2) const { - return try_fuse_common({t1, t2}); - } - - bool try_fuse_allreduce_add(const htp_opnode & node) { - if (n_ops == 0 || opt_ar_select != 2) return false; - if (node.opcode != HTP_OP_ADD) return false; - - htp_opnode & last_node = ops[n_ops - 1]; - if (last_node.opcode != HTP_OP_ALLREDUCE) return false; - - auto * ar_kparams = (struct htp_allreduce_kernel_params *) last_node.kernel_params; - const uint32_t rank = (uint32_t) ar_kparams->rank; - const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; - const ggml_tensor * ar_local = last_node.inputs[rank]; - const ggml_tensor * add_src0 = node.src0(); - const ggml_tensor * add_src1 = node.src1(); - const ggml_tensor * add_dst = node.dst(); - - if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false; - - const ggml_tensor * res_tensor; - if (add_src0 == ar_local || add_src0->data == ar_local->data) { - res_tensor = add_src1; - } else if (add_src1 == ar_local || add_src1->data == ar_local->data) { - res_tensor = add_src0; - } else { - return false; - } - - if (ar_local->type != res_tensor->type) return false; - - const bool is_same_shape = (ar_local->ne[0] == res_tensor->ne[0] && ar_local->ne[1] == res_tensor->ne[1] && - ar_local->ne[2] == res_tensor->ne[2] && ar_local->ne[3] == res_tensor->ne[3]); - const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] && res_tensor->ne[1] == 1 && - res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); - - if (!is_same_shape && !is_row_bcast) return false; - - if (is_same_shape) { - if (ar_local->nb[1] != res_tensor->nb[1] || ar_local->nb[2] != res_tensor->nb[2] || - ar_local->nb[3] != res_tensor->nb[3]) { - return false; - } - if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(res_tensor)) { - return false; - } - } - if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) { - return false; - } - - // scatter is only valid for in-place add within max outputs - const bool is_shard_ok = n_ranks <= HTP_OP_MAX_OUTPUTS && add_dst->data == ar_local->data; - - struct htp_allreduce_kernel_params new_kparams; - if (!ggml_hexagon_precompute_allreduce_params( - sess, add_dst, (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, is_shard_ok, &new_kparams - )) { - HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", sess->c_name()); - return false; - } - - const bool scatter_ok = new_kparams.n_dsts > 1; - new_kparams.mode = scatter_ok ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL; - - for (uint32_t r = 0; r < n_ranks; r++) { - const ggml_tensor * ar_src = last_node.inputs[r]; - if (!ggml_hexagon_tensors_overlap(add_dst, ar_src)) { - continue; - } - // in-place aliasing is safe under reduce-scatter since each rank writes disjoint shards - if (scatter_ok && r == rank) { - continue; - } - HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", sess->c_name(), r); - return false; - } - - if (!try_fuse_common(res_tensor, add_dst)) { - return false; - } - - last_node.opcode = HTP_OP_ALLREDUCE_ADD; - last_node.name = "ALLREDUCE+ADD"; - last_node.inputs.push_back(res_tensor); - last_node.outputs.clear(); - last_node.outputs.push_back(add_dst); - last_node.fused.push_back(node.node); - memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams)); - - htp_op_desc & o = h_ops[n_ops - 1]; - o.opcode = HTP_OP_ALLREDUCE_ADD; - memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams)); - - o.src[2 * n_ranks] = add_tensor(res_tensor); - if (new_kparams.mode == HTP_ALLREDUCE_SHARDED_FANOUT) { - // fan out shard to all per-rank partial buffers - GGML_ASSERT((uint32_t) new_kparams.n_dsts == n_ranks); - for (uint32_t d = 0; d < n_ranks; d++) { - GGML_ASSERT(o.src[d] != 0xffff); - o.dst[d] = o.src[d]; - } - for (uint32_t d = n_ranks; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - } else { - o.dst[0] = add_tensor(add_dst); - for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - } - - HEX_VERBOSE("ggml-hex: %s fused ALLREDUCE+ADD (#%u) mode=%d n_dsts=%d\n", - sess->c_name(), n_ops - 1, (int) new_kparams.mode, (int) new_kparams.n_dsts); - return true; - } - - bool try_fuse_rms_norm_mul(const htp_opnode & node) { - if (n_ops == 0) return false; - if (node.opcode != HTP_OP_MUL) return false; - - htp_opnode & last_node = ops[n_ops - 1]; - if (last_node.opcode != HTP_OP_RMS_NORM) return false; - - const ggml_tensor * mul_src0 = node.src0(); - const ggml_tensor * mul_src1 = node.src1(); - const ggml_tensor * rms_out = last_node.dst(); - - if (!ggml_hexagon_tensor_is_fuseable(rms_out)) return false; - - const ggml_tensor * weight; - if (mul_src0 == rms_out || mul_src0->data == rms_out->data) { - weight = mul_src1; - } else if (mul_src1 == rms_out || mul_src1->data == rms_out->data) { - weight = mul_src0; - } else { - return false; - } - - const ggml_tensor * src0 = last_node.src0(); - - if (src0->ne[0] != weight->ne[0] || src0->ne[0] != node.dst()->ne[0]) { - return false; - } - - const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1); - const bool is_same_shape = (src0->ne[0] == weight->ne[0] && src0->ne[1] == weight->ne[1] && - src0->ne[2] == weight->ne[2] && src0->ne[3] == weight->ne[3]); - if (!is_row_bcast && !is_same_shape) return false; - - if (!ggml_are_same_shape(src0, node.dst())) { - return false; - } - if (ggml_is_contiguous(src0) != ggml_is_contiguous(node.dst())) { - return false; - } - - struct htp_unary_kernel_params new_kparams; - ggml_hexagon_precompute_unary_params( - sess, HTP_OP_RMS_NORM_MUL, src0, weight, node.dst(), &new_kparams - ); - - if ((size_t) new_kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n", - sess->c_name(), new_kparams.vtcm_size, sess->vtcm_size); - return false; - } - - if (!try_fuse_common(weight, node.dst())) { - return false; - } - - last_node.opcode = HTP_OP_RMS_NORM_MUL; - last_node.name = "RMS_NORM+MUL"; - last_node.inputs.clear(); - last_node.inputs.push_back(src0); - last_node.inputs.push_back(weight); - last_node.outputs.clear(); - last_node.outputs.push_back(node.dst()); - last_node.fused.push_back(node.node); - memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams)); - - htp_op_desc & o = h_ops[n_ops - 1]; - o.opcode = HTP_OP_RMS_NORM_MUL; - memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams)); - - o.src[0] = add_tensor(src0); - o.src[1] = add_tensor(weight); - for (uint32_t s = 2; s < HTP_OP_MAX_INPUTS; s++) { - o.src[s] = 0xffff; - } - o.dst[0] = add_tensor(node.dst()); - for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - - HEX_VERBOSE("ggml-hex: %s fused RMS_NORM+MUL (#%u)\n", sess->c_name(), n_ops - 1); - return true; - } - - bool try_fuse_mul_mat_add(const htp_opnode & node) { - if (n_ops == 0) return false; - if (node.opcode != HTP_OP_ADD) return false; - - htp_opnode & last_node = ops[n_ops - 1]; - if (last_node.opcode != HTP_OP_MUL_MAT) return false; - - const ggml_tensor * add_src0 = node.src0(); - const ggml_tensor * add_src1 = node.src1(); - const ggml_tensor * mm_out = last_node.dst(); - - if (!ggml_hexagon_tensor_is_fuseable(mm_out)) return false; - - const ggml_tensor * src2; - if (add_src0 == mm_out || add_src0->data == mm_out->data) { - src2 = add_src1; - } else if (add_src1 == mm_out || add_src1->data == mm_out->data) { - src2 = add_src0; - } else { - return false; - } - - const ggml_tensor * src0 = last_node.src0(); - const ggml_tensor * src1 = last_node.src1(); - - if (src2->type != GGML_TYPE_F32) return false; - - const struct htp_mm_kernel_params * orig_kparams = (const struct htp_mm_kernel_params *) last_node.kernel_params; - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_fused_matmul_add_params(sess, src0, src1, src2, node.dst(), &kparams); - if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED) { - return false; - } - - const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; - const bool can_fuse = (kparams.n_hmx > 0) || (src1_nrows == 1); - if (!can_fuse) return false; - - if ((size_t) kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: VTCM needed (%d) > budget (%zu)\n", - sess->c_name(), kparams.vtcm_size, sess->vtcm_size); - return false; - } - - if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) { - if (kparams.m_chunk < orig_kparams->m_chunk || - kparams.n_chunk < orig_kparams->n_chunk || - kparams.n_act_threads < orig_kparams->n_act_threads) { - HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n", - sess->c_name(), orig_kparams->m_chunk, kparams.m_chunk, - orig_kparams->n_chunk, kparams.n_chunk, - orig_kparams->n_act_threads, kparams.n_act_threads); - return false; - } - } - - if (!try_fuse_common(src2, node.dst())) { - return false; - } - - last_node.opcode = HTP_OP_MUL_MAT_ADD; - last_node.name = "MUL_MAT+ADD"; - last_node.inputs.clear(); - last_node.inputs.push_back(src0); - last_node.inputs.push_back(src1); - last_node.inputs.push_back(src2); - last_node.outputs.clear(); - last_node.outputs.push_back(node.dst()); - last_node.fused.push_back(node.node); - memcpy(last_node.kernel_params, &kparams, sizeof(kparams)); - - htp_op_desc & o = h_ops[n_ops - 1]; - o.opcode = HTP_OP_MUL_MAT_ADD; - memcpy(o.kernel_params, &kparams, sizeof(kparams)); - - o.src[0] = add_tensor(src0); - o.src[1] = add_tensor(src1); - o.src[2] = add_tensor(src2); - for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) { - o.src[s] = 0xffff; - } - o.dst[0] = add_tensor(node.dst()); - for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - - HEX_VERBOSE("ggml-hex: %s fused MUL_MAT+ADD (#%u)\n", sess->c_name(), n_ops - 1); - return true; - } - - bool try_fuse_mul_mat_nx(const htp_opnode & node) { - if (n_ops == 0 || node.opcode != HTP_OP_MUL_MAT) return false; - if (!is_mergeable_mul_mat(node.node)) return false; - - const ggml_tensor * w_in = node.src0(); - const ggml_tensor * x_in = node.src1(); - const ggml_tensor * d_in = node.dst(); - - htp_opnode & last_node = ops[n_ops - 1]; - - // Case 1: last_node is already MUL_MAT_NX - if (last_node.opcode == HTP_OP_MUL_MAT_NX) { - const uint32_t curr_n = (uint32_t) last_node.outputs.size(); - if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) { - return false; - } - - const ggml_tensor * w0 = last_node.inputs[0]; - const ggml_tensor * x = last_node.inputs[curr_n]; - - if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0]) { - return false; - } - if (!last_node.fused.empty() && (mm_is_hmx_eligible(last_node.fused[0]) != mm_is_hmx_eligible(node.node))) { - return false; - } - - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, curr_n + 1, &kparams); - if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) { - return false; - } - if ((size_t) kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n", - sess->c_name(), kparams.vtcm_size, sess->vtcm_size); - return false; - } - - if (!try_fuse_common(w_in, d_in)) { - return false; - } - - last_node.inputs[curr_n] = w_in; - last_node.inputs.push_back(x); - last_node.outputs.push_back(d_in); - last_node.fused.push_back(node.node); - memcpy(last_node.kernel_params, &kparams, sizeof(kparams)); - - htp_op_desc & o = h_ops[n_ops - 1]; - memcpy(o.kernel_params, &kparams, sizeof(kparams)); - - for (uint32_t s = 0; s <= curr_n + 1; s++) { - o.src[s] = add_tensor(last_node.inputs[s]); - } - for (uint32_t s = curr_n + 2; s < HTP_OP_MAX_INPUTS; s++) { - o.src[s] = 0xffff; - } - for (uint32_t d = 0; d <= curr_n; d++) { - o.dst[d] = add_tensor(last_node.outputs[d]); - } - for (uint32_t d = curr_n + 1; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - - HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=%u, #%u)\n", sess->c_name(), curr_n + 1, n_ops - 1); - return true; - } - - // Case 2: last_node is single MUL_MAT - if (last_node.opcode == HTP_OP_MUL_MAT) { - if (!is_mergeable_mul_mat_pair(last_node.node, node.node)) { - return false; - } - - const ggml_tensor * w0 = last_node.src0(); - const ggml_tensor * x = last_node.src1(); - const ggml_tensor * w1 = node.src0(); - - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, 2, &kparams); - if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) { - return false; - } - if ((size_t) kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n", - sess->c_name(), kparams.vtcm_size, sess->vtcm_size); - return false; - } - - if (!try_fuse_common(w1, node.dst())) { - return false; - } - - const ggml_tensor * dst_0 = last_node.dst(); - const ggml_tensor * dst_1 = node.dst(); - - last_node.opcode = HTP_OP_MUL_MAT_NX; - last_node.name = "MUL_MAT_NX"; - last_node.inputs.clear(); - last_node.inputs.push_back(w0); - last_node.inputs.push_back(w1); - last_node.inputs.push_back(x); - last_node.outputs.clear(); - last_node.outputs.push_back(dst_0); - last_node.outputs.push_back(dst_1); - last_node.fused.push_back(node.node); - memcpy(last_node.kernel_params, &kparams, sizeof(kparams)); - - htp_op_desc & o = h_ops[n_ops - 1]; - o.opcode = HTP_OP_MUL_MAT_NX; - memcpy(o.kernel_params, &kparams, sizeof(kparams)); - - o.src[0] = add_tensor(w0); - o.src[1] = add_tensor(w1); - o.src[2] = add_tensor(x); - for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) { - o.src[s] = 0xffff; - } - o.dst[0] = add_tensor(dst_0); - o.dst[1] = add_tensor(dst_1); - for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - - HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=2, #%u)\n", sess->c_name(), n_ops - 1); - return true; - } - - return false; - } - - bool try_fuse_mul_mat_id_nx(const htp_opnode & node) { - if (n_ops == 0 || node.opcode != HTP_OP_MUL_MAT_ID) return false; - if (!is_mergeable_mul_mat_id(node.node)) return false; - - const ggml_tensor * w_in = node.src0(); - const ggml_tensor * x_in = node.src1(); - const ggml_tensor * ids_in = node.node->src[2]; - const ggml_tensor * d_in = node.dst(); - - htp_opnode & last_node = ops[n_ops - 1]; - - // Case 1: last_node is already MUL_MAT_ID_NX - if (last_node.opcode == HTP_OP_MUL_MAT_ID_NX) { - const uint32_t curr_n = (uint32_t) last_node.outputs.size(); - if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 2 >= HTP_OP_MAX_INPUTS) { - return false; - } - - const ggml_tensor * w0 = last_node.inputs[0]; - const ggml_tensor * x = last_node.inputs[curr_n]; - const ggml_tensor * ids = last_node.inputs[curr_n + 1]; - - if (x_in != x || ids_in != ids || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2]) { - return false; - } - if (!last_node.fused.empty() && (mm_is_hmx_eligible(last_node.fused[0]) != mm_is_hmx_eligible(node.node))) { - return false; - } - - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, d_in, curr_n + 1, &kparams); - if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) { - return false; - } - if ((size_t) kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: %s skip ID NX fusion: VTCM needed (%d) > budget (%zu)\n", - sess->c_name(), kparams.vtcm_size, sess->vtcm_size); - return false; - } - - if (!try_fuse_common(w_in, d_in)) { - return false; - } - - last_node.inputs[curr_n] = w_in; - last_node.inputs[curr_n + 1] = x; - last_node.inputs.push_back(ids); - last_node.outputs.push_back(d_in); - last_node.fused.push_back(node.node); - memcpy(last_node.kernel_params, &kparams, sizeof(kparams)); - - htp_op_desc & o = h_ops[n_ops - 1]; - memcpy(o.kernel_params, &kparams, sizeof(kparams)); - - for (uint32_t s = 0; s <= curr_n + 2; s++) { - o.src[s] = add_tensor(last_node.inputs[s]); - } - for (uint32_t s = curr_n + 3; s < HTP_OP_MAX_INPUTS; s++) { - o.src[s] = 0xffff; - } - for (uint32_t d = 0; d <= curr_n; d++) { - o.dst[d] = add_tensor(last_node.outputs[d]); - } - for (uint32_t d = curr_n + 1; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - - HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_ID_NX (N=%u, #%u)\n", sess->c_name(), curr_n + 1, n_ops - 1); - return true; - } - - // Case 2: last_node is single MUL_MAT_ID - if (last_node.opcode == HTP_OP_MUL_MAT_ID) { - if (!is_mergeable_mul_mat_id_pair(last_node.node, node.node)) { - return false; - } - - const ggml_tensor * w0 = last_node.src0(); - const ggml_tensor * x = last_node.src1(); - const ggml_tensor * ids = last_node.node->src[2]; - const ggml_tensor * w1 = node.src0(); - - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, node.dst(), 2, &kparams); - if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) { - return false; - } - if ((size_t) kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: %s skip ID NX fusion: VTCM needed (%d) > budget (%zu)\n", - sess->c_name(), kparams.vtcm_size, sess->vtcm_size); - return false; - } - - if (!try_fuse_common(w1, node.dst())) { - return false; - } - - const ggml_tensor * dst_0 = last_node.dst(); - const ggml_tensor * dst_1 = node.dst(); - - last_node.opcode = HTP_OP_MUL_MAT_ID_NX; - last_node.name = "MUL_MAT_ID_NX"; - last_node.inputs.clear(); - last_node.inputs.push_back(w0); - last_node.inputs.push_back(w1); - last_node.inputs.push_back(x); - last_node.inputs.push_back(ids); - last_node.outputs.clear(); - last_node.outputs.push_back(dst_0); - last_node.outputs.push_back(dst_1); - last_node.fused.push_back(node.node); - memcpy(last_node.kernel_params, &kparams, sizeof(kparams)); - - htp_op_desc & o = h_ops[n_ops - 1]; - o.opcode = HTP_OP_MUL_MAT_ID_NX; - memcpy(o.kernel_params, &kparams, sizeof(kparams)); - - o.src[0] = add_tensor(w0); - o.src[1] = add_tensor(w1); - o.src[2] = add_tensor(x); - o.src[3] = add_tensor(ids); - for (uint32_t s = 4; s < HTP_OP_MAX_INPUTS; s++) { - o.src[s] = 0xffff; - } - o.dst[0] = add_tensor(dst_0); - o.dst[1] = add_tensor(dst_1); - for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - - HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_ID_NX (N=2, #%u)\n", sess->c_name(), n_ops - 1); - return true; - } - - return false; - } - - bool try_fuse_gdn_cpy(const htp_opnode & node) { - if (n_ops == 0 || node.opcode != HTP_OP_CPY) return false; - - htp_opnode & last_node = ops[n_ops - 1]; - if (last_node.opcode != HTP_OP_GATED_DELTA_NET) return false; - if (last_node.outputs.size() != 1) return false; - - const ggml_tensor * gdn_out = last_node.dst(); - const ggml_tensor * cpy_node = node.node; - const ggml_tensor * cpy_src = node.src0(); - const ggml_tensor * cpy_dst = node.dst(); - - if (!cpy_src || !cpy_dst || !cpy_dst->data) return false; - if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false; - if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false; - - const ggml_tensor * v = last_node.node->src[2]; - if (!v) return false; - - const int64_t S_v = v->ne[0]; - const int64_t H = v->ne[1]; - const int64_t n_tokens = v->ne[2]; - const int64_t n_seqs = v->ne[3]; - const int64_t K = ggml_get_op_params_i32(last_node.node, 0); - const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float); - - const int64_t D = S_v * S_v * H; - const int64_t n_written = std::min<int64_t>(n_tokens, K); - - if (cpy_src->op != GGML_OP_VIEW || (cpy_src->view_src != gdn_out && cpy_src->view_src->data != gdn_out->data) || - cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) { - return false; - } - - if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->nb[0] != sizeof(float)) { - return false; - } - if (n_seqs > 1 && cpy_dst->nb[1] != (size_t) D * sizeof(float)) { - return false; - } - if (n_written > 1) { - if (cpy_dst->ne[2] != n_written || cpy_dst->nb[2] != (size_t) D * n_seqs * sizeof(float)) { - return false; - } - } - - if (!try_fuse_common({cpy_dst})) { - return false; - } - - last_node.name += "+CPY"; - last_node.outputs.push_back(cpy_dst); - last_node.fused.push_back(node.node); - - htp_op_desc & o = h_ops[n_ops - 1]; - o.dst[1] = add_tensor(cpy_dst); - for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) { - o.dst[d] = 0xffff; - } - - HEX_VERBOSE("ggml-hex: %s fused GATED_DELTA_NET+CPY (#%u)\n", sess->c_name(), n_ops - 1); - return true; - } - - bool try_fuse(const htp_opnode & node) { - if (!opt_opfusion) return false; - if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_ALLREDUCE_ADD) && try_fuse_allreduce_add(node)) return true; - if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_RMS_NORM_MUL) && try_fuse_rms_norm_mul(node)) return true; - if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ADD) && try_fuse_mul_mat_add(node)) return true; - if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_NX) && try_fuse_mul_mat_nx(node)) return true; - if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ID_NX) && try_fuse_mul_mat_id_nx(node)) return true; - if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_GDN_CPY) && try_fuse_gdn_cpy(node)) return true; - return false; - } -}; - -struct ggml_hexagon_registry { - ggml_hexagon_registry(ggml_backend_reg_t reg); - ~ggml_hexagon_registry(); - - ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS]; -}; - -struct ggml_hexagon_opqueue { - // Shared buffer for storing batches - ggml_hexagon_shared_buffer *shm_buf; - size_t shm_blk_size; - size_t depth; - - using opvec = std::vector<htp_opnode>; - - std::vector<opvec> op_cache; // per batch op cache - std::vector<uint64_t> start_usec; // per batch start time - - ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) { - size_t n_bufs = HTP_OP_MAX_BUFS; - size_t n_ops = batch_size; - size_t n_tensors = n_ops * HTP_OP_MAX_OUTPUTS + n_ops * HTP_OP_MAX_INPUTS; - - size_t tr_size = 0; - if (opt_profile == 3) { - tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc); - } - - shm_blk_size = sizeof(htp_buf_desc) * n_bufs + - sizeof(htp_tensor) * n_tensors + - sizeof(htp_op_desc) * n_ops + - sizeof(htp_prof_desc) * n_ops + - tr_size; - - shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */); - - op_cache.resize(depth); - start_usec.resize(depth, 0); - - if (opt_verbose) { - GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", - sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size); - } - } - - ~ggml_hexagon_opqueue() { - delete shm_buf; - } - - size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; } - - // push new batch - bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) { - static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8"); - static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8"); - static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8"); - static_assert(sizeof(htp_tensor) % 8 == 0, "sizeof(htp_tensor) must be multiple of 8"); - static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8"); - static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8"); - - if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; } - - const uint32_t slot = (uint32_t) ((seq - 1) % depth); - - req.seq = seq; - req.n_bufs = op_batch->n_bufs; - req.n_tensors = op_batch->n_tens; - req.n_ops = op_batch->n_ops; - - op_cache[slot] = op_batch->ops; - start_usec[slot] = ggml_time_us(); - - const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs; - const size_t t_size = sizeof(htp_tensor) * req.n_tensors; - const size_t o_size = sizeof(htp_op_desc) * req.n_ops; - const size_t p_size = sizeof(htp_prof_desc) * req.n_ops; - - size_t tr_size = 0; - if (opt_profile == 3) { - req.n_traces = opt_optrace; - tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc); - } else { - req.n_traces = 0; - } - - dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size); - dbuf.fd = shm_buf->fd(); - dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; - dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base(); - dbuf.size = b_size + t_size + o_size + p_size + tr_size; - - GGML_ASSERT(dbuf.size <= shm_blk_size); - - uint8_t * m_ptr = (uint8_t*) dbuf.ptr; - uint8_t * b_ptr = m_ptr; m_ptr += b_size; - uint8_t * t_ptr = m_ptr; m_ptr += t_size; - uint8_t * o_ptr = m_ptr; - - memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size); - memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size); - memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size); - - HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", - shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, - b_size, t_size, o_size, (size_t) dbuf.size); - - if (opt_verbose > 1) { - htp_buf_desc *b = (htp_buf_desc*) b_ptr; - for (unsigned int i=0; i < req.n_bufs; i++) { - GGML_LOG_DEBUG("ggml-hex: %s htp-buf #%u : fd %d base %p size %zu\n", shm_buf->sess->c_name(), i, - b[i].fd, (void *) b[i].base, (size_t) b[i].size); - } - htp_tensor *t = (htp_tensor*) t_ptr; - for (unsigned int i=0; i < req.n_tensors; i++) { - GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %llu size %u : %zu:%zu:%zu:%zu\n", - shm_buf->sess->c_name(), i, t[i].bi, (unsigned long long) t[i].data, t[i].size, - (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]); - } - } - - return true; - } - - void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) { - const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth); - - const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs; - const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors; - const size_t o_size = sizeof(htp_op_desc) * rsp.n_ops; - const size_t p_size = sizeof(htp_prof_desc) * rsp.n_ops; - - size_t tr_size = 0; - uint32_t n_traces = 0; - if (opt_profile == 3) { - n_traces = opt_optrace; - tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc); - } - - const size_t m_size = b_size + t_size + o_size + p_size + tr_size; - GGML_ASSERT(m_size <= shm_blk_size); - - HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", - shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, - (size_t) dbuf.size, b_size, t_size, o_size); - - uint8_t * m_ptr = (uint8_t*) dbuf.ptr; - uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size); - - if (rsp.n_ops > 0) { - auto & ops = op_cache[slot]; - GGML_ASSERT(rsp.n_ops <= ops.size()); - - const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr; - const htp_trace_desc * trace_events = nullptr; - if (opt_profile == 3) { - trace_events = (const htp_trace_desc *) (p_ptr + p_size); - } - - if (opt_profile) { - ggml_hexagon_dump_batch_prof(shm_buf->sess->name, rsp); - } - - for (uint32_t i = 0; i < rsp.n_ops; i++) { - if (opt_profile) { - ggml_hexagon_dump_op_prof(shm_buf->sess->name, ops[i], pd[i]); - } - } - - if (opt_profile) { - ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces); - } - } - } -}; - -void ggml_hexagon_session::flush_peers() { - auto vpeers = std::move(virt_peers); - virt_peers.clear(); - for (auto * peer : vpeers) { - peer->flush_sync(); - } - - auto ppeers = std::move(phys_peers); - phys_peers.clear(); - for (auto * peer : ppeers) { - peer->flush_async(); - } - - for (auto & sub : this->mdev.sessions) { - sub->flush_peers(); - } -} - -void ggml_hexagon_session::flush_async() { - flush_peers(); - flush_batch(); -} - -void ggml_hexagon_session::flush_pending(bool all) { - for (auto & sub : this->mdev.sessions) { - sub->flush_pending(all); - if (sub->last_error > HTP_STATUS_OK) { - this->last_error = sub->last_error.load(); - } - } - - while (this->batch_rsp_seq < this->batch_req_seq) { - struct htp_opbatch_rsp rsp; - uint32_t rsp_size; - uint32_t flags; - - struct dspqueue_buffer dbuf; - uint32_t n_dbufs; - - // Read response packet from queue - const uint32_t timeo = opt_oppoll ? 0 : DSPQUEUE_TIMEOUT; - - int err = dspqueue_read(this->queue, &flags, 1, &n_dbufs, &dbuf, sizeof(rsp), &rsp_size, (uint8_t *) &rsp, timeo); - if (err == AEE_EEXPIRED || err == AEE_EWOULDBLOCK) { - continue; - } - - if (err != 0) { - GGML_ABORT("ggml-hex: dspqueue_read failed: 0x%08x\n", (unsigned) err); - } - - // Basic sanity checks - if (rsp_size != sizeof(rsp) || n_dbufs != 1) { - GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs); - } - - if (rsp.status > HTP_STATUS_OK) { - GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status)); - this->last_error = rsp.status; - for (auto & sub : this->mdev.sessions) { - sub->last_error = rsp.status; - } - } - - op_queue->pop(rsp, dbuf); - - GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1); - this->batch_rsp_seq = rsp.seq; - - if (!all) break; - } -} - -void ggml_hexagon_session::flush_sync(bool all) { - flush_async(); - flush_pending(all); -} - -void ggml_hexagon_session::start_batch() { - if (this->mdev.count > 1) { - enqueue_mdev_group(); - } -} - -void ggml_hexagon_session::flush_batch(size_t min_ops) { - if (op_batch->n_ops < min_ops) { return; } - - op_batch->sort_buffers(); - - htp_opbatch_req req {}; - dspqueue_buffer dbuf{}; - - const uint64_t seq = ++this->batch_req_seq; - - op_batch->update_mdev_group(this->mdev.idx); - - if (!op_queue->push(req, dbuf, op_batch, seq)) { - flush_pending(false); - op_queue->push(req, dbuf, op_batch, seq); - } - - for (auto & sub : this->mdev.sessions) { - htp_opbatch_req sub_req {}; - dspqueue_buffer sub_dbuf{}; - - sub->batch_req_seq = seq; - op_batch->update_mdev_group(sub->mdev.idx); - - if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) { - sub->flush_pending(false); - sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq); - } - - HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size); - - int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT); - if (err != 0) { - GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err); - } - } - - HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size); - - int err = dspqueue_write(this->queue, 0, 1, &dbuf, sizeof(req), (const uint8_t*) &req, DSPQUEUE_TIMEOUT); - if (err != 0) { - GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err); - } - - op_batch->reset(); -} - -void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { - auto clone_tensor_buffer = [this](const ggml_tensor * t) { - auto sbuf = this->mmap_tensor(t); - if (!sbuf) return; - if (sbuf->sess != this) { - this->clone_buffer(sbuf); - } - for (auto & sub : this->mdev.sessions) { - sub->clone_buffer(sbuf); - } - }; - - for (auto t : node.get_inputs()) { - clone_tensor_buffer(t); - } - for (auto t : node.get_outputs()) { - clone_tensor_buffer(t); - } - - if (opt_opfusion && op_batch->try_fuse(node)) { - return; - } - - if (!op_batch->fit_op(node)) { - flush_async(); - } - - if (op_batch->empty()) { - start_batch(); - } - - if (!op_batch->fit_op(node)) { - GGML_ABORT("ggml-hex: %s op does not fit into empty batch (vmem/tensor/buffer limit exceeded)\n", - c_name()); - } - - op_batch->add_op(node); -} - -void ggml_hexagon_session::enqueue_mdev_group() { - htp_opnode group_node(HTP_OP_MDEV_GROUP); - - uint8_t * fence_slot = this->mdev_fence_slot; - - static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; - ggml_tensor dummy_t {}; - dummy_t.buffer = &this->fence_buf->backend_buffer; - dummy_t.extra = &fence_extra; - dummy_t.data = (void *) fence_slot; - dummy_t.type = GGML_TYPE_I8; - dummy_t.ne[0] = HTP_FENCE_SLOT_SIZE; - dummy_t.ne[1] = (int64_t) this->mdev.count; - dummy_t.ne[2] = 1; - dummy_t.ne[3] = 1; - dummy_t.nb[0] = 1; - dummy_t.nb[1] = HTP_FENCE_SLOT_SIZE; - dummy_t.nb[2] = dummy_t.nb[1] * dummy_t.ne[1]; - dummy_t.nb[3] = dummy_t.nb[2]; - dummy_t.op = GGML_OP_NONE; - dummy_t.op_params[0] = (int32_t) this->mdev.idx; - - ggml_tensor * node = group_node.add_dummy(dummy_t); - node->src[0] = node; - group_node.init(node); - group_node.outputs.clear(); - group_node.name = "MDEV_GROUP"; - - for (auto & sub : this->mdev.sessions) { - sub->clone_buffer(this->fence_buf); - } - - op_batch->add_op(group_node); -} - -void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) { - const bool with_fence = sync_tensor != nullptr; - htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY); - - ggml_tensor* node = cpy_node.add_dummy(*dst); - node->op = GGML_OP_CPY; - node->src[0] = const_cast<ggml_tensor *>(src); - node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr; - if (with_fence) { - node->op_params[0] = (int32_t) fence_seq; - } - - cpy_node.init(node); - if (with_fence) { - cpy_node.name = "CPY+FENCE"; - } - this->enqueue_op(cpy_node); -} - -void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) { - htp_opnode sync_node(HTP_OP_FENCE); - - ggml_tensor* node = sync_node.add_dummy(*sync_tensor); - node->op = GGML_OP_NONE; - node->src[0] = node; - node->op_params[0] = (int32_t) fence_seq; - node->op_params[1] = wait ? 0 : 1; - - sync_node.init(node); - sync_node.name = wait ? "FENCE_WAIT" : "FENCE_SIGNAL"; - this->enqueue_op(sync_node); -} - -static bool ggml_hexagon_precompute_allreduce_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * dst, - uint32_t rank, - uint32_t n_ranks, - bool has_add, - bool is_row_bcast, - bool is_shard_ok, - struct htp_allreduce_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - kparams->rank = (int32_t) rank; - kparams->n_ranks = (int32_t) n_ranks; - kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; - - const uint32_t nelem = (uint32_t) ggml_nelements(dst); - const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); - const bool is_contiguous = ggml_is_contiguous(dst); - - const uint32_t ne0 = (uint32_t) dst->ne[0]; - const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); - kparams->ne0 = (int32_t) ne0; - kparams->ne1 = (int32_t) ne1; - - const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1); - - if (has_add) { - // sharded reduce-scatter for contiguous in-place add - if (opt_ar_scatter && is_shard_ok && use_1d && !is_row_bcast) { - const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128); - const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem); - const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem); - kparams->n_dsts = (int32_t) n_ranks; - kparams->rank_elem_start = (int32_t) rank_elem_start; - kparams->rank_nelem = (int32_t) (rank_elem_end - rank_elem_start); - } else if (use_1d) { - kparams->n_dsts = 1; - kparams->rank_elem_start = 0; - kparams->rank_nelem = (int32_t) nelem; - } else { - kparams->n_dsts = 1; - kparams->rank_elem_start = 0; - kparams->rank_nelem = (int32_t) ne1; - } - } else { - kparams->n_dsts = (int32_t) n_ranks; - if (use_1d) { - const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128); - const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem); - const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem); - const uint32_t rank_nelem = rank_elem_end - rank_elem_start; - kparams->rank_elem_start = (int32_t) rank_elem_start; - kparams->rank_nelem = (int32_t) rank_nelem; - } else { - const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks; - const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1); - const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1); - const uint32_t rank_nrows = rank_r1 - rank_r0; - kparams->rank_elem_start = (int32_t) rank_r0; - kparams->rank_nelem = (int32_t) rank_nrows; - } - } - - kparams->mode = (kparams->n_dsts > 1) ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL; - - if (use_1d) { - const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nelem / 128)); - kparams->n_threads = n_threads; - const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); - - uint32_t block_elems = 65536; - if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) { - block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128); - } - block_elems = (std::max)(128u, block_elems); - - kparams->block_elems = block_elems; - kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; - - while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_elems > 128) { - const size_t max_bytes_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2); - block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); - if (block_elems < 128) break; - kparams->block_elems = block_elems; - kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; - } - - if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { - HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n", - sess->c_name(), kparams->vtcm_size, sess->vtcm_size); - return false; - } - - kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems); - kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D; - return true; - } else { - const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nrows)); - kparams->n_threads = n_threads; - const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); - - const uint32_t row_bytes = ne0 * elem_size; - const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); - kparams->row_size_aligned = row_size_aligned; - - const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads; - uint32_t block_rows = (std::min)(128u, nrows_per_thread); - block_rows = (std::max)(1u, block_rows); - kparams->block_elems = block_rows; - - kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; - - while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_rows > 1) { - const size_t max_rows_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned); - block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); - kparams->block_elems = block_rows; - kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; - if (max_rows_per_buf == 0) break; - } - - if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) { - HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n", - sess->c_name(), kparams->vtcm_size, sess->vtcm_size); - return false; - } - - kparams->elems_per_thread = nrows_per_thread; - kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D; - return true; - } -} - -void ggml_hexagon_session::enqueue_allreduce( - const ggml_tensor * dst, - const std::vector<const ggml_tensor *> & src_tensors, - const std::vector<const ggml_tensor *> & sync_tensors, - uint32_t rank, - uint32_t n_ranks, - uint32_t fence_seq_entry, - uint32_t fence_seq_exit -) { - htp_opnode ar_node(HTP_OP_ALLREDUCE); - - ggml_tensor* node = ar_node.add_dummy(*dst); - node->op = GGML_OP_NONE; - node->op_params[0] = (int32_t) fence_seq_entry; - node->op_params[1] = (int32_t) fence_seq_exit; - - ar_node.init(node); - - ar_node.inputs.clear(); - for (size_t i = 0; i < src_tensors.size(); i++) { - ar_node.inputs.push_back(src_tensors[i]); - } - for (size_t i = 0; i < sync_tensors.size(); i++) { - ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i])); - } - - ar_node.outputs.clear(); - for (size_t i = 0; i < src_tensors.size(); i++) { - ar_node.outputs.push_back(src_tensors[i]); - } - - ggml_hexagon_precompute_allreduce_params( - this, dst, rank, n_ranks, false, false, /*is_shard_ok=*/ false, - (struct htp_allreduce_kernel_params *) ar_node.kernel_params); - - ar_node.name = "ALLREDUCE"; - this->enqueue_op(ar_node); -} - -ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) { - if (!t) return nullptr; - - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!sbuf->mapped) { - const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; - const bool extended = opt_dma64 && is_weight; - sbuf->mmap(extended); - } - return sbuf; -} - -bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf) -{ - GGML_ASSERT(sbuf && sbuf->mem); - if (sbuf->sess == this) return true; - - auto mem = sbuf->mem; - int fd = mem->fd; - - GGML_ASSERT(fd >= 0); - - if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true; - - GGML_ASSERT(sbuf->mapped); - - HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(), - sbuf->c_name(), sbuf->base(), sbuf->size(), fd); - - auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf); - try { - clone->mmap(sbuf->extended); - } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what()); - return false; - } - - this->cloned_buffers[fd] = std::move(clone); - mem->mapped_clones.insert(this); - return true; -} - -void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) { - GGML_ASSERT(sbuf && sbuf->mem); - - auto mem = sbuf->mem; - int fd = mem->fd; - - GGML_ASSERT(fd >= 0); - - auto it = this->cloned_buffers.find(fd); - if (it != this->cloned_buffers.end()) { - auto clone = std::move(it->second); - this->cloned_buffers.erase(it); - } - mem->mapped_clones.erase(this); -} - -void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) { - GGML_ASSERT(sbuf && sbuf->mem); - - auto mem = sbuf->mem; - std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end()); - - for (auto * sess : sessions) { - sess->release_buffer(sbuf); - } -} - -static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { - // Allocate a bunch pinned buffers till failure. - // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device. - // Typically we're going to allocate all/most of these buffers anyway for the model weights. - - std::vector<ggml_hexagon_shared_buffer *> sbufs; - - const size_t MiB = 1024 * 1024; - const size_t GiB = MiB * 1024; - - size_t vmem = 0; - size_t step = 256u * MiB; - - try { - sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; - sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; - sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; - - while (1) { - sbufs.push_back(new ggml_hexagon_shared_buffer(sess, step, true)); - vmem += step; - } - } catch (...) { } - - for (auto b : sbufs) { delete b; } - - return vmem - step; // backoff to account for overhead from internal mappings -} - -void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) { - int phys_idx = config.physical_idx; - int virt_idx = config.virtual_idx; - - this->valid_session = false; - this->valid_handle = false; - this->valid_queue = false; - this->valid_iface = false; - - this->name = config.name; - this->phys_idx = phys_idx; - this->virt_idx = virt_idx; - this->domain_id = config.domain_id; - this->session_id = 0; - this->batch_req_seq = 0; - this->batch_rsp_seq = 0; - this->last_error = HTP_STATUS_OK; - - GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx); - - if (config.domain_id < 0 || config.domain_name.empty()) { - GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx); - throw std::runtime_error("ggml-hex: invalid physical CDSP core"); - } - - const std::string & dom_name = config.domain_name; - - // Create new session if virtual_idx > 0 - if (virt_idx > 0) { - struct remote_rpc_reserve_new_session n {}; - n.domain_name_len = dom_name.size(); - n.domain_name = const_cast<char *>(dom_name.c_str()); - n.session_name = const_cast<char *>(this->name.c_str()); - n.session_name_len = this->name.size(); - n.session_id = virt_idx; - - int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); - if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n", - this->c_name(), phys_idx, virt_idx, err); - throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)"); - } - - // Save the IDs - this->session_id = n.session_id; - this->domain_id = n.effective_domain_id; - this->valid_session = true; - } else { - struct remote_rpc_effective_domain_id eff {}; - eff.domain_name = const_cast<char *>(dom_name.c_str()); - eff.domain_name_len = dom_name.size(); - eff.session_id = 0; - - int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff)); - if (err == AEE_SUCCESS) { - this->domain_id = eff.effective_domain_id; - } else { - GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n", - this->name.c_str(), err, this->domain_id); - } - } - - // Enable unsigned modules - { - struct remote_rpc_control_unsigned_module u; - u.domain = this->domain_id; - u.enable = 1; - int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); - if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); - throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); - } - } - - char session_uri[256]; - { - char htp_uri[256]; - snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch); - - struct remote_rpc_get_uri u = {}; - u.session_id = this->session_id; - u.domain_name = const_cast<char *>(dom_name.c_str()); - u.domain_name_len = dom_name.size(); - u.module_uri = const_cast<char *>(htp_uri); - u.module_uri_len = strlen(htp_uri); - u.uri = session_uri; - u.uri_len = sizeof(session_uri); - - int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u)); - if (err != AEE_SUCCESS) { - snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u", - htp_uri, dom_name.c_str(), this->session_id); - - GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", - this->c_name(), phys_idx, virt_idx, err, session_uri); - } - } - - // Open session - int err = htp_iface_open(session_uri, &this->handle); - if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err); - throw std::runtime_error("ggml-hex: failed to open session (see log for details)"); - } - - this->valid_handle = true; - - // Query HW info and resolve session options - { - unsigned int hw_n_threads = 0; - unsigned int hw_n_hvx = 0; - unsigned int hw_n_hmx = 0; - unsigned long long hw_vtcm_size = 0; - int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size); - if (hw_err == 0) { - const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads); - this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads; - this->n_hvx = this->n_threads; - this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; - this->vtcm_size = (uint64_t)hw_vtcm_size; - GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n", - this->c_name(), this->n_threads, this->n_hvx, this->n_hmx, - (unsigned long long)(this->vtcm_size / (1024 * 1024))); - } else { - GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err); - const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS); - this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads; - this->n_hvx = this->n_threads; - this->n_hmx = (opt_nhmx != 0) ? 1 : 0; - this->vtcm_size = 8 * 1024 * 1024; - } - } - - // Enable FastRPC QoS mode - { - struct remote_rpc_control_latency l; - l.enable = 1; - - int err = remote_handle64_control(this->handle, DSPRPC_CONTROL_LATENCY, (void *) &l, sizeof(l)); - if (err != 0) { - GGML_LOG_WARN("ggml-hex: failed to enable fastrpc QOS mode: 0x%08x\n", (unsigned) err); - } - } - - GGML_LOG_INFO("ggml-hex: %s new session : session-id %d domain-id %d uri %s handle 0x%lx\n", this->c_name(), - this->session_id, this->domain_id, session_uri, (unsigned long) this->handle); - - const size_t req_q_size = (sizeof(htp_opbatch_req) * opt_opqueue * 2) + 1024; - const size_t rsp_q_size = (sizeof(htp_opbatch_rsp) * opt_opqueue * 2) + 1024; - - // Now let's setup the DSP queue - err = dspqueue_create(this->domain_id, - 0, // Flags - req_q_size, // Request queue size (in bytes) - rsp_q_size, // Response queue size (in bytes) - nullptr, // Read packet callback (we handle reads explicitly) - nullptr, // Error callback (we handle errors during reads) - (void *) this, // Callback context - &queue); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: %s dspqueue_create failed: 0x%08x\n", this->name.c_str(), (unsigned) err); - throw std::runtime_error("ggml-hex: failed to create dspqueue (see log for details)"); - } - - this->valid_queue = true; - - // Export queue for use on the DSP - err = dspqueue_export(queue, &this->queue_id); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: dspqueue_export failed: 0x%08x\n", (unsigned) err); - throw std::runtime_error("ggml-hex: dspqueue export failed (see log for details)"); - } - - if (opt_etm) { - err = htp_iface_etm(this->handle, 1); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: failed to enable ETM tracing: 0x%08x\n", (unsigned) err); - } - } - - // Allocate buffers and state for op batching - this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); - - this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024); - if (this->mdev.count > 1) { - this->mdev_fence_slot = this->alloc_fence(this->mdev.count); - } - - if (!opt_vmem) { - opt_vmem = ggml_hexagon_measure_max_vmem(this); - GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem); - } - const size_t shm_size = this->op_queue->shm_size(); - this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem; - - this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); - - // Start dspqueue/opbatch processing - err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); - throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); - } - this->valid_iface = true; - - if (opt_profile) { - htp_iface_pmu_conf pmu_conf{}; - std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events); - - err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err); - } - } -} - -void ggml_hexagon_session::release() noexcept(true) { - GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str()); - - this->mdev.sessions.clear(); - - int err; - - if (this->valid_iface) { - // Stop dspqueue/opbatch processing - err = htp_iface_stop(this->handle); - if (err != 0) { - GGML_ABORT("ggml-hex: htp_iface_stop failed: 0x%08x\n", (unsigned) err); - } - } - - delete this->op_batch; - delete this->op_queue; - for (auto & it : this->cpy_fence_slots) { - free_fence((void *) it.second, 1); - } - this->cpy_fence_slots.clear(); - - if (this->fence_buf) { - unclone_buffer(this->fence_buf); - delete this->fence_buf; - this->fence_buf = nullptr; - } - while (!this->cloned_buffers.empty()) { - release_buffer(this->cloned_buffers.begin()->second.get()); - } - - if (opt_etm) { - err = htp_iface_etm(this->handle, 0); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: warn : failed to disable ETM tracing: 0x%08x\n", (unsigned) err); - } - } - - if (opt_profile) { - htp_iface_pmu_conf pmu_conf{}; - err = htp_iface_profiler(this->handle, 0, &pmu_conf); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: warn : failed to disable profiling: 0x%08x\n", (unsigned) err); - } - } - - if (this->valid_queue) { - err = dspqueue_close(queue); - if (err != 0) { - GGML_ABORT("ggml-hex: dspqueue_close failed: 0x%08x\n", (unsigned) err); - } - } - - if (this->valid_handle) { - htp_iface_close(this->handle); - } -} - -ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) { - this->dev = dev; - this->dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context); - this->mdev.idx = mdev_idx; - this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size()); - op_batch = nullptr; - op_queue = nullptr; - fence_buf = nullptr; - fence_seq = ((uintptr_t)this) & 0xFFFF; - - try { - allocate(config); - if (this->mdev.idx == 0 && !config.mdev_group.empty()) { - for (size_t i = 0; i < config.mdev_group.size(); i++) { - this->mdev.sessions.push_back(std::make_unique<ggml_hexagon_session>( - config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count)); - } - } - } catch (const std::exception & exc) { - release(); - throw; - } -} - -ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { - release(); -} - -// ** backend interface - -static bool ggml_hexagon_flash_attn_is_hmx_eligible( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * q, - const struct ggml_tensor * k, - const struct ggml_tensor * v, - const struct ggml_tensor * sinks -) { - if (sess->n_hmx == 0) { - return false; - } - - if (opt_fa_select < 2) { - return false; - } - - if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) || - (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) { - return false; - } - - const uint32_t DK = q->ne[0]; - const uint32_t DV = v->ne[0]; - - // Head dims that are not multiples of 64 are handled by internally padding to - // DK_pad/DV_pad = round_up(.,64) and zero-filling the tail lanes. - if (DK % 8 != 0 || DV % 8 != 0) { - return false; - } - - // Fall back to HVX for small token counts if head dimension is small (DK <= 128) - const uint32_t neq1 = q->ne[1]; - if (DK <= 128 && neq1 < 5) { - return false; - } - - return true; - - GGML_UNUSED(sinks); -} - -static bool ggml_hexagon_precompute_flash_attn_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - struct htp_fa_kernel_params * kparams -) { - if (opt_fa_select < 1) { - return false; - } - - memset(kparams, 0, sizeof(*kparams)); - - const struct ggml_tensor * q = op->src[0]; - const struct ggml_tensor * k = op->src[1]; - const struct ggml_tensor * v = op->src[2]; - const struct ggml_tensor * mask = op->src[3]; - const struct ggml_tensor * dst = op; - - const uint32_t neq0 = q->ne[0]; // head_dim (DK) - const uint32_t neq1 = q->ne[1]; // n_tokens - const uint32_t neq2 = q->ne[2]; // n_heads - - const uint32_t nek1 = k->ne[1]; // kv_len - - const uint32_t nev0 = v->ne[0]; // head_dim (DV) - - const uint32_t DK = neq0; - const uint32_t DV = nev0; - - const uint32_t n_kv_heads = k->ne[2]; - const uint32_t G = neq2 / n_kv_heads; - - float scale = 1.0f; - float max_bias = 0.0f; - float logit_softcap = 0.0f; - memcpy(&scale, &op->op_params[0], sizeof(float)); - memcpy(&max_bias, &op->op_params[1], sizeof(float)); - memcpy(&logit_softcap, &op->op_params[2], sizeof(float)); - - if (logit_softcap != 0.0f) { - scale /= logit_softcap; - } - - kparams->scale = scale; - kparams->max_bias = max_bias; - kparams->logit_softcap = logit_softcap; - - kparams->is_q_fp32 = (q->type == GGML_TYPE_F32) ? 1 : 0; - kparams->is_dst_fp32 = (dst->type == GGML_TYPE_F32) ? 1 : 0; - kparams->G = G; - - const uint32_t n_head = q->ne[2]; - kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head)); - kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2); - kparams->m1 = std::pow(2.0f, -(max_bias / 2.0f) / kparams->n_head_log2); - - // Check HMX eligibility - const struct ggml_tensor * sinks = op->src[4]; - if (ggml_hexagon_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) { - // HMX tiles head_dim in units of 64; when DK/DV are not 64-aligned the kernel - // operates on padded dims with zero-filled tail lanes. VTCM budget and chunk-size - // are sized for the padded tiles. - const uint32_t DK_pad = hex_round_up(DK, 64); - const uint32_t DV_pad = hex_round_up(DV, 64); - size_t Br = 0, Bc = 0; - int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, sess->vtcm_size, sess->n_threads, kparams->is_q_fp32 != 0, sinks != nullptr, n_head); - if (ret == 0) { - kparams->kernel_type = HTP_FA_KERNEL_HMX; - kparams->Br = Br; - kparams->Bc = Bc; - kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc; - kparams->n_threads = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? sess->n_threads : 1; - - kparams->u.hmx.g_br = hex_align_up(G * Br, 32); - kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? 1 : 0; - kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, kparams->is_q_fp32 != 0, sinks != nullptr, n_head); - - const size_t row_vec_bytes = hex_align_up(Bc * sizeof(uint16_t), 256); - kparams->u.hmx.row_buf_stride = row_vec_bytes / 128; // HVX vector is 128 bytes - - const size_t m_line_bytes = hex_align_up(Bc * sizeof(uint16_t), 128); - kparams->u.hmx.mask_buf_row_stride = m_line_bytes / sizeof(uint16_t); - kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0; - kparams->u.hmx.div_G = init_fastdiv_values(G); - if (mask) { - kparams->src3_div2 = init_fastdiv_values(mask->ne[2]); - kparams->src3_div3 = init_fastdiv_values(mask->ne[3]); - } - - kparams->qrows = 0; - kparams->qrows_per_thread = 0; - return true; - } - } - - // Fallback to HVX - kparams->kernel_type = HTP_FA_KERNEL_HVX; - kparams->Br = 1; - kparams->Bc = 64; // FLASH_ATTN_BLOCK_SIZE - kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64; - kparams->n_threads = sess->n_threads; - - const size_t size_q_row_padded = hex_round_up(q->ne[0] * (kparams->is_q_fp32 ? 4 : 2), 128); - const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128); - const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128); - - kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, kparams->is_q_fp32 != 0, mask != nullptr, sinks != nullptr, n_head, sess->n_threads); - - kparams->u.hvx.size_q_row_padded = size_q_row_padded; - kparams->u.hvx.size_k_row_padded = size_k_row_padded; - kparams->u.hvx.size_v_row_padded = size_v_row_padded; - kparams->u.hvx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]); - kparams->u.hvx.src0_div1 = init_fastdiv_values(q->ne[1]); - kparams->broadcast_rk2 = init_fastdiv_values(q->ne[2]/k->ne[2]); - kparams->broadcast_rk3 = init_fastdiv_values(q->ne[3]/k->ne[3]); - kparams->broadcast_rv2 = init_fastdiv_values(q->ne[2]/v->ne[2]); - kparams->broadcast_rv3 = init_fastdiv_values(q->ne[3]/v->ne[3]); - if (mask) { - kparams->src3_div2 = init_fastdiv_values(mask->ne[2]); - kparams->src3_div3 = init_fastdiv_values(mask->ne[3]); - } - - kparams->qrows = q->ne[1] * q->ne[2] * q->ne[3]; - kparams->qrows_per_thread = (kparams->qrows + sess->n_threads - 1) / sess->n_threads; - - return true; -} - -static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * src3 = op->src[3]; - const struct ggml_tensor * src4 = op->src[4]; - const struct ggml_tensor * dst = op; - - // Check for F16/Q8_0 support - if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || - (src1->type != GGML_TYPE_F16 && src1->type != GGML_TYPE_Q8_0) || - (src2->type != GGML_TYPE_F16 && src2->type != GGML_TYPE_Q8_0)) { - return false; - } - - if (src3 && src3->type != GGML_TYPE_F16) { // mask - return false; - } - - if (src4 && src4->type != GGML_TYPE_F32) { // sinks - return false; - } - - // For now we support F32 or F16 output as htp backend often converts output on the fly if needed, - // but the op implementation writes to F16 or F32. - // Let's assume dst can be F32 or F16. - if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { - return false; - } - - if (dst->ne[3] != 1) { - return false; - } - - struct htp_fa_kernel_params kparams; - if (!ggml_hexagon_precompute_flash_attn_params(sess, op, &kparams)) { - return false; - } - - if ((size_t) kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n", - kparams.vtcm_size, sess->vtcm_size); - return false; - } - - return true; -} - -static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - if (opt_gdn_select < 1) { - return false; - } - - const struct ggml_tensor * q = op->src[0]; - const struct ggml_tensor * k = op->src[1]; - const struct ggml_tensor * v = op->src[2]; - const struct ggml_tensor * g = op->src[3]; - const struct ggml_tensor * beta = op->src[4]; - const struct ggml_tensor * state = op->src[5]; - const struct ggml_tensor * dst = op; - - if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 || - g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 || - dst->type != GGML_TYPE_F32) { - return false; - } - - if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) || - !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) || - !ggml_is_contiguous(dst)) { - return false; - } - - const int64_t S_v = v->ne[0]; - const int64_t H = v->ne[1]; - const int64_t n_tokens = v->ne[2]; - const int64_t n_seqs = v->ne[3]; - const int64_t K = ggml_get_op_params_i32(op, 0); - - if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) { - return false; - } - if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 || - q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 || - (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) { - return false; - } - if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) { - return false; - } - // state holds s0 only [S_v, S_v, H, n_seqs]; K is op param 0. - if (ggml_nelements(state) != S_v * S_v * H * n_seqs) { - return false; - } - if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K) { - return false; - } - - const uint32_t total_rows = (uint32_t) (H * n_seqs); - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, total_rows); + std::vector<opvec> op_cache; // per batch op cache - const bool can_use_hmx = (opt_gdn_select >= 2) && - (sess->n_hmx > 0) && - (S_v % 64 == 0) && - (n_tokens >= HTP_GDN_MIN_TOKENS) && - (g->ne[0] == 1) && - (K == 1); + ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) { + size_t n_bufs = HTP_OP_MAX_BUFS; + size_t n_ops = batch_size; + size_t n_tensors = std::min<size_t>(n_ops * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS); - if (can_use_hmx) { - struct htp_gdn_hmx_vtcm_layout layout; - uint32_t n_heads_batch = 0; - if (!htp_gdn_hmx_solve_layout(&layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, sess->vtcm_size, n_threads, true, &n_heads_batch)) { - return false; - } - } else { - struct htp_gdn_vtcm_layout layout; - htp_gdn_vtcm_layout_build(&layout, (uint32_t) S_v, n_threads); - if (layout.total_bytes > sess->vtcm_size) { - return false; + size_t tr_size = 0; + if (opt_profile == 3) { + tr_size = sess->n_threads * opt_optrace * sizeof(htp_trace_desc); } - } - - return true; -} - -static bool ggml_hexagon_matmul_is_hmx_eligible( - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - int ne01_padded, - bool is_matmul_id, - bool is_batched -) { - if (src1->type != GGML_TYPE_F32) { - return false; - } - - const int ne00 = src0->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int wtype = src0->type; - - // HMX weight tile requires N to be 32-aligned. - if (ne01_padded % 32 != 0) { - return false; - } - - // HMX supports F16, F32, and repack quantized types. - if (!ggml_hexagon_is_hmx_weight_type((ggml_type) wtype)) { - return false; - } - - // HMX paths require K aligned to 32. - if (ne00 % 32 != 0) { - return false; - } - - // Quantized HMX kernels only handle flat 2D matmul (or matmul_id wrapping flat 2D matmuls). - if (!is_matmul_id && is_batched && wtype != GGML_TYPE_F16) { - return false; - } - - // HMX assumes contiguous row-major layout. - if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) { - return false; - } - // M alignment: Use HMX when M > HTP_MM_HMX_MIN_NROWS. - // For MUL_MAT_ID, src1 shape is [K, n_expert_used, n_tokens, 1], so n_tokens is ne12. - const int m = is_matmul_id ? ne12 : ne11; - if (m <= HTP_MM_HMX_MIN_NROWS) { - return false; - } - - return true; + shm_blk_size = sizeof(htp_buf_desc) * n_bufs + + sizeof(htp_tensor) * n_tensors + + sizeof(htp_op_desc) * n_ops + + sizeof(htp_prof_desc) * n_ops + + tr_size; - GGML_UNUSED(dst); -} + shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */); -static bool ggml_hexagon_precompute_hmx_mm_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - int wtype, - int ne00_padded, - int ne01_padded, - int ne02, - int ne11, - int ne12, - int ne11_padded, - bool is_matmul_id, - bool is_batched, - size_t src2_size, - size_t vtcm_budget, - struct htp_mm_kernel_params * kparams -) { - const int aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - const bool pipeline = is_matmul_id ? false : htp_mm_hmx_pipeline(ne11); - const int n_threads = (int)sess->n_threads; - const int ne10 = src1->ne[0]; - - const bool is_batched_val = is_matmul_id ? false : is_batched; - const int group_size = (ne02 > 0 ? ne12 / ne02 : 1); - - size_t m_chunk = 0; - size_t n_chunk = 0; - size_t vtcm_size = 0; - bool use_grouped = false; - int act_threads_selected = 0; - - if (is_batched_val && wtype == GGML_TYPE_F16 && group_size > 1) { - // Try grouped path first - if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, ne11, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { - use_grouped = true; - } - } + op_cache.resize(depth); - if (!use_grouped) { - // Fallback to simple 2D path (group_size = 1) - const int m_id_rows = (dst && is_matmul_id) ? (int) ((size_t) dst->ne[1] * dst->ne[2]) : 0; - if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, ne11_padded, ne11, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { - return false; + if (opt_verbose) { + GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", + sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size); } } - kparams->n_hmx = 1; - kparams->pipeline = pipeline ? 1 : 0; - kparams->m_chunk = m_chunk; - kparams->n_chunk = n_chunk; - kparams->n_threads = n_threads; - kparams->n_act_threads = act_threads_selected; - kparams->tile_size = htp_mm_get_weight_tile_size(wtype); - kparams->aligned_tile_size = aligned_tile_size; - kparams->src1_row_size = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - kparams->vtcm_size = vtcm_size; - kparams->vtcm_src0_size = 0; - kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected); - kparams->div_ne00_padded = init_fastdiv_values(ne00_padded); - kparams->vtcm_src1_size = 0; - kparams->vtcm_src2_size = (int32_t) src2_size; - kparams->vtcm_dst_size = 0; - - if (is_batched && !is_matmul_id) { - kparams->kernel_type = HTP_MM_KERNEL_HMX_F16_BATCHED; - } else { - kparams->kernel_type = HTP_MM_KERNEL_HMX_2D; - } - return true; - - GGML_UNUSED(src0); -} - -static void ggml_hexagon_precompute_hvx_mm_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - int wtype, - int ne02, - int ne03, - int ne10, - int ne11, - int ne12, - int ne13, - bool is_matmul_id, - const size_t src2_row_size, - size_t vtcm_budget, - struct htp_mm_kernel_params * kparams -) { - if (opt_mm_select < 1) { - kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; - return; - } - - kparams->n_hmx = 0; - kparams->n_threads = sess->n_threads; - - const bool is_quant = (wtype != GGML_TYPE_F16 && wtype != GGML_TYPE_F32); - const int src1_nrows = ne11 * ne12 * ne13; - - if (is_quant) { - // Quantized HVX - kparams->tile_size = htp_mm_get_weight_tile_size(wtype); - kparams->aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - - const bool k_align = (ne10 % 32 == 0); - - if (is_matmul_id) { - kparams->kernel_type = (src1_nrows < (int) sess->n_threads) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; - kparams->src1_row_size = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - - struct htp_mm_hvx_vtcm_layout L; - uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; - uint32_t best_n_prefetch = 2; - for (uint32_t d = max_prefetch; d >= 2; d /= 2) { - htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0->nb[1], kparams->src1_row_size, 0, d, true, false - ); - if (L.total_bytes <= vtcm_budget) { - best_n_prefetch = d; - break; - } - } - if (L.total_bytes > vtcm_budget) { - kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; - return; - } - kparams->n_prefetch = best_n_prefetch; - kparams->vtcm_size = L.total_bytes; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - goto done_quant; - } else { - bool try_tiled = (k_align && opt_mm_select >= 1); - if (try_tiled) { - kparams->src1_row_size = htp_mm_weight_has_offset(wtype) - ? htp_mm_q8_1_tiled_row_size(ne10) - : htp_mm_q8_0_tiled_row_size(ne10); - if (src1_nrows < (int) sess->n_threads) { - kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_BLOCK; - } else { - kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; - } - - struct htp_mm_hvx_vtcm_layout L; - uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; - uint32_t best_n_prefetch = 2; - for (uint32_t d = max_prefetch; d >= 2; d /= 2) { - htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false - ); - if (L.total_bytes <= vtcm_budget) { - best_n_prefetch = d; - break; - } - } - - uint32_t m_chunk = 0; - if (htp_mm_hvx_solve_vtcm_params( - kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget, - &L, &m_chunk)) { - kparams->n_prefetch = best_n_prefetch; - kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; - kparams->vtcm_size = L.total_bytes; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_src2_size = L.src2_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - goto done_quant; - } - } - - kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; - return; - } - - done_quant:; - } else if (wtype == GGML_TYPE_F16) { - // F16 HVX - struct htp_mm_hvx_vtcm_layout L; - uint32_t m_chunk = 0; - if (htp_mm_hvx_solve_vtcm_params( - HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget, - &L, &m_chunk)) { - kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_VTCM; - kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; - kparams->src1_row_size = hex_round_up(ne10 * 2, 128); - kparams->vtcm_size = L.total_bytes; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_src2_size = L.src2_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - kparams->n_prefetch = 16; - return; - } - - kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; - return; - } else { - // F32 HVX - struct htp_mm_hvx_vtcm_layout L; - uint32_t m_chunk = 0; - if (htp_mm_hvx_solve_vtcm_params( - HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget, - &L, &m_chunk)) { - kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_VTCM; - kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; - kparams->src1_row_size = hex_round_up(ne10 * 4, 128); - kparams->vtcm_size = L.total_bytes; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_src2_size = L.src2_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - kparams->n_prefetch = 16; - return; - } - - kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; - return; + ~ggml_hexagon_opqueue() { + delete shm_buf; } -} -static void ggml_hexagon_precompute_matmul_params_impl( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - const size_t src2_row_size, - const size_t src2_size, - struct htp_mm_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; - - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; - - const int wtype = src0->type; - const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype); - const int ne00_padded = is_repack ? hex_round_up(ne00, 32) : ne00; - const int ne01_padded = is_repack ? hex_round_up(ne01, 32) : ne01; - const int ne11_padded = hex_round_up(ne11, 32); - - const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID); - const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); - - const size_t vtcm_budget = sess->vtcm_size; - - // Check HMX eligibility and try precomputing HMX parameters - bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 2); - if (hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible(src0, src1, dst, ne01_padded, is_matmul_id, is_batched)) { - if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, src1, dst, wtype, ne00_padded, ne01_padded, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, src2_size, vtcm_budget, kparams)) { - goto finalize; - } - } + size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; } - // Fallback to HVX parameter computation - ggml_hexagon_precompute_hvx_mm_params(sess, src0, src1, dst, wtype, ne02, ne03, ne10, ne11, ne12, ne13, is_matmul_id, src2_row_size, vtcm_budget, kparams); + // push new batch + bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) { + static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8"); + static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8"); + static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8"); + static_assert(sizeof(htp_tensor) % 8 == 0, "sizeof(htp_tensor) must be multiple of 8"); + static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8"); + static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8"); -finalize: - kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11); - kparams->div_ne1 = init_fastdiv_values(ne11); - kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); - kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); - kparams->div_ne12 = init_fastdiv_values(ne12); -} + if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; } -static void ggml_hexagon_precompute_matmul_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_mm_kernel_params * kparams -) { - ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, 0, 0, kparams); -} + const uint32_t slot = (uint32_t) ((seq - 1) % depth); -static void ggml_hexagon_precompute_fused_matmul_add_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * src2, - const struct ggml_tensor * dst, - struct htp_mm_kernel_params * kparams -) { - const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0; - ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams); -} + req.seq = seq; + req.n_bufs = op_batch->n_bufs; + req.n_tensors = op_batch->n_tens; + req.n_ops = op_batch->n_ops; -static bool ggml_hexagon_precompute_binary_params( - const struct ggml_hexagon_session * sess, - uint32_t op, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_binary_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const size_t elem_size = ggml_type_size(src0->type); - const size_t src0_row_size = src0->ne[0] * elem_size; - const size_t src1_row_size = src1->ne[0] * elem_size; - const size_t dst_row_size = dst->ne[0] * elem_size; - - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128); - const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128); - - const bool is_add_id = op == HTP_OP_ADD_ID; - const bool is_scalar = !is_add_id && src1->ne[0] == 1; - const bool is_transposed = src0->nb[1] < src0_row_size || src1->nb[1] < src1_row_size || dst->nb[1] < dst_row_size; - const bool is_row_bcast = !is_add_id && !is_scalar && !is_transposed && - src1->ne[0] == src0->ne[0] && - (src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) && - src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1; - const bool is_same_shape = !is_add_id && !is_scalar && !is_transposed && - src1->ne[0] == src0->ne[0] && - src1->ne[1] == src0->ne[1] && - (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) && - (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1); - const bool is_complex = !is_add_id && !is_scalar && !is_same_shape && !is_row_bcast && (src1->ne[0] == src0->ne[0]); - const bool is_contig = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst); - const bool is_scalar_broadcast = !is_add_id && (ggml_nelements(src1) == 1); - - if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) { - const uint32_t total_elems = (uint32_t) ggml_nelements(src0); - const uint32_t n_threads = sess->n_threads; - const uint32_t max_chunk_elems = 32768 / elem_size; - const uint32_t min_chunk_elems = 256; - const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * n_threads) - 1) / (2 * n_threads), 32); - const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems)); - const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); - - kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; - kparams->n_threads = n_threads; - kparams->rows_per_buffer = 1; - kparams->src0_row_size_aligned = src0_row_size_aligned; - kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned; - kparams->dst_row_size_aligned = dst_row_size_aligned; - kparams->src1_size = 0; - kparams->chunk_size = chunk_size; - kparams->chunk_bytes = chunk_bytes; - kparams->is_scalar = is_scalar_broadcast ? 1 : 0; - - struct htp_binary_vtcm_layout L; - htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size); - if (L.total_bytes == 0 || L.total_bytes > sess->vtcm_size) { - return false; + if (opt_profile) { + op_cache[slot].assign(op_batch->ops.begin(), op_batch->ops.begin() + req.n_ops); } - kparams->vtcm_size = L.total_bytes; - return true; - } + const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs; + const size_t t_size = sizeof(htp_tensor) * req.n_tensors; + const size_t o_size = sizeof(htp_op_desc) * req.n_ops; + const size_t p_size = sizeof(htp_prof_desc) * req.n_ops; - enum htp_binary_kernel_type kernel_type; - size_t src1_size = 0; - - if (is_add_id) { - kernel_type = HTP_BINARY_KERNEL_ADD_ID; - src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128); - } else if (is_row_bcast) { - kernel_type = HTP_BINARY_KERNEL_ROW_BCAST; - src1_size = src1_row_size_aligned; - } else if (is_scalar) { - const bool is_scalar_static = (src1->ne[2] == 1 && src1->ne[3] == 1) && - (src1->ne[1] == 1 || src1->nb[1] == elem_size); - if (is_scalar_static) { - kernel_type = HTP_BINARY_KERNEL_SCALAR_DMA; - src1_size = hex_round_up(src1->ne[1] * elem_size, 128); + size_t tr_size = 0; + if (opt_profile == 3) { + req.n_traces = opt_optrace; + tr_size = shm_buf->sess->n_threads * req.n_traces * sizeof(htp_trace_desc); } else { - kernel_type = HTP_BINARY_KERNEL_SCALAR; - } - } else if (is_same_shape) { - kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE; - } else if (is_complex) { - kernel_type = HTP_BINARY_KERNEL_COMPLEX; - } else { - kernel_type = HTP_BINARY_KERNEL_REPEAT; - } - - kparams->kernel_type = kernel_type; - kparams->n_threads = sess->n_threads; - kparams->src0_row_size_aligned = src0_row_size_aligned; - kparams->src1_row_size_aligned = src1_row_size_aligned; - kparams->dst_row_size_aligned = dst_row_size_aligned; - kparams->src1_size = src1_size; - - struct htp_binary_vtcm_layout L; - htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size); - if (L.rows_per_buffer == 0 || L.total_bytes > sess->vtcm_size) { - if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) { - const uint32_t total_elems = (uint32_t) ggml_nelements(src0); - const uint32_t n_threads = sess->n_threads; - const uint32_t max_chunk_elems = 32768 / elem_size; - const uint32_t min_chunk_elems = 256; - const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * n_threads) - 1) / (2 * n_threads), 32); - const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems)); - const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); - - kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; - kparams->n_threads = n_threads; - kparams->rows_per_buffer = 1; - kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned; - kparams->src1_size = 0; - kparams->chunk_size = chunk_size; - kparams->chunk_bytes = chunk_bytes; - kparams->is_scalar = is_scalar_broadcast ? 1 : 0; - - htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size); - if (L.total_bytes == 0 || L.total_bytes > sess->vtcm_size) { - return false; - } - - kparams->vtcm_size = L.total_bytes; - return true; + req.n_traces = 0; } - return false; - } - - kparams->rows_per_buffer = L.rows_per_buffer; - kparams->vtcm_size = L.total_bytes; - - return true; -} - -static void ggml_hexagon_precompute_unary_params( - const struct ggml_hexagon_session * sess, - uint32_t op, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_unary_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = (std::min)((uint32_t)sess->n_threads, src0_nrows); - - kparams->n_threads = n_threads; - - const size_t elem_size = ggml_type_size(src0->type); - - const size_t src0_data_row_size = src0->ne[0] * elem_size; - const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); - - const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); - const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); - - kparams->src0_row_size_aligned = src0_row_size_aligned; - kparams->dst_row_size_aligned = dst_row_size_aligned; - - size_t src1_data_row_size = 0; - size_t src1_row_size_aligned = 0; - bool broadcast_weight = false; - - if (op == HTP_OP_RMS_NORM_MUL) { - GGML_ASSERT(src1 != nullptr); - src1_data_row_size = src1->ne[0] * ggml_type_size(src1->type); - src1_row_size_aligned = hex_round_up(src1_data_row_size, 128); - broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); - } - - kparams->src1_row_size_aligned = src1_row_size_aligned; - kparams->broadcast_weight = broadcast_weight; - - struct htp_unary_vtcm_layout L; - uint32_t col_tile = 0; - uint32_t vtcm_row_per_thread = 0; - htp_unary_vtcm_layout_build(&L, op, src0->ne[0], dst->ne[0], - op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0, - broadcast_weight, n_threads, sess->vtcm_size, elem_size, - &col_tile, &vtcm_row_per_thread); - - kparams->col_tile = col_tile; - kparams->vtcm_row_per_thread = vtcm_row_per_thread; - kparams->vtcm_size = L.total_bytes; - - kparams->vtcm_src0_size_per_thread = L.src0_bytes; - kparams->vtcm_src1_size_per_thread = L.src1_bytes; - kparams->vtcm_dst_size_per_thread = L.dst_bytes; - - kparams->vtcm_src0_size = L.src0_bytes * n_threads; - kparams->vtcm_src1_size = L.src1_bytes * n_threads; - kparams->vtcm_dst_size = L.dst_bytes * n_threads; - - kparams->block = col_tile ? 0 : ((L.src0_bytes / 2) / src0_row_size_aligned); + dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size); + dbuf.fd = shm_buf->fd(); + dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; + dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base(); + dbuf.size = b_size + t_size + o_size + p_size + tr_size; - const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1; - kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); - kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); - kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]); - kparams->div_tpr = init_fastdiv_values(tiles_per_row); -} + GGML_ASSERT(dbuf.size <= shm_blk_size); -static void ggml_hexagon_precompute_get_rows_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_get_rows_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const uint32_t ne00 = src0->ne[0]; - const uint32_t ne02 = src0->ne[2]; - const uint32_t ne03 = src0->ne[3]; - - const uint32_t ne10 = src1->ne[0]; - const uint32_t ne11 = src1->ne[1]; - const uint32_t ne12 = src1->ne[2]; - const uint32_t nr = ne10 * ne11 * ne12; - - const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0; - const auto * extra = src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) ? - (const ggml_hexagon_tensor_extra *) src0_base->extra : nullptr; - const bool tiled = src0->type == GGML_TYPE_Q4_0 || (extra && (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0) || - sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0); - - if (src0->type == dst->type) { - kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE; - } else if (tiled) { - kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED; - } else { - kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT; - } + uint8_t * m_ptr = (uint8_t*) dbuf.ptr; + uint8_t * b_ptr = m_ptr; m_ptr += b_size; + uint8_t * t_ptr = m_ptr; m_ptr += t_size; + uint8_t * o_ptr = m_ptr; - const uint32_t chunks_per_row = 1; - const uint32_t chunk_size = ne00; - const uint32_t total_tasks = nr; + memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size); + memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size); + memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size); - kparams->n_threads = (std::min)((uint32_t)sess->n_threads, total_tasks); + HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, + b_size, t_size, o_size, (size_t) dbuf.size); - struct htp_get_rows_vtcm_layout vtcm_layout = {}; - while (kparams->n_threads > 0) { - htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, kparams->n_threads); - if (vtcm_layout.total_bytes <= sess->vtcm_size) { - break; + if (opt_verbose > 1) { + htp_buf_desc *b = (htp_buf_desc*) b_ptr; + for (unsigned int i=0; i < req.n_bufs; i++) { + GGML_LOG_DEBUG("ggml-hex: %s htp-buf #%u : fd %d base %p size %zu\n", shm_buf->sess->c_name(), i, + b[i].fd, (void *) b[i].base, (size_t) b[i].size); + } + htp_tensor *t = (htp_tensor*) t_ptr; + for (unsigned int i=0; i < req.n_tensors; i++) { + GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %llu size %u : %zu:%zu:%zu:%zu\n", + shm_buf->sess->c_name(), i, t[i].bi, (unsigned long long) t[i].data, t[i].size, + (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]); + } } - --kparams->n_threads; - } - - if (kparams->n_threads == 0 && total_tasks > 0) { - htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, 1); - } - - kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes; - kparams->tasks_per_thread = kparams->n_threads > 0 ? (total_tasks + kparams->n_threads - 1) / kparams->n_threads : 0; - - kparams->chunks_per_row = chunks_per_row; - kparams->chunk_size = chunk_size; - kparams->total_tasks = total_tasks; - - kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0}; - kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0}; - kparams->div_chunks_per_row = chunks_per_row > 0 ? init_fastdiv_values(chunks_per_row) : fastdiv_values{0, 0}; - kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0}; - kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0}; -} - -static void ggml_hexagon_precompute_set_rows_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, // values - const struct ggml_tensor * src1, // indices - const struct ggml_tensor * dst, // destination - struct htp_set_rows_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const uint32_t nr = src0->ne[1]; - - kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr); - kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads; - kparams->total_tasks = nr; - - kparams->div_ne11 = init_fastdiv_values(src1->ne[1]); - kparams->div_ne12 = init_fastdiv_values(src1->ne[2]); - kparams->div_tasks_per_thread = init_fastdiv_values(kparams->tasks_per_thread); - kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); - - struct htp_set_rows_vtcm_layout vtcm_layout; - htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads); - kparams->vtcm_size = vtcm_layout.total_bytes; -} - -static void ggml_hexagon_precompute_softmax_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - struct htp_softmax_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); - - float scale = 1.0f; - float max_bias = 0.0f; - memcpy(&scale, &op->op_params[0], sizeof(float)); - memcpy(&max_bias, &op->op_params[1], sizeof(float)); - - kparams->scale = scale; - kparams->max_bias = max_bias; - - const uint32_t n_head = src0->ne[2]; - const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); - kparams->n_head = n_head; - kparams->n_head_log2 = n_head_log2; - - if (max_bias > 0.0f && n_head_log2 > 0) { - kparams->m0 = powf(2.0f, -(max_bias) / n_head_log2); - kparams->m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - } else { - kparams->m0 = 1.0f; - kparams->m1 = 1.0f; - } - - kparams->use_src1 = (src1 != nullptr) ? 1 : 0; - kparams->use_f16 = (src1 != nullptr && src1->type == GGML_TYPE_F16) ? 1 : 0; - - const uint32_t ne00 = src0->ne[0]; - const uint32_t ne10 = src1 ? src1->ne[0] : 1; - - struct htp_softmax_vtcm_layout layout; - htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads); - - kparams->n_threads = n_threads; - kparams->src0_nrows = src0_nrows; - kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - kparams->vtcm_size = (uint32_t) layout.total_bytes; - kparams->vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread; - kparams->vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread; - kparams->vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread; - kparams->src0_row_size_aligned = (uint32_t) layout.src0_spad_half_size; - kparams->src1_row_size_aligned = (uint32_t) layout.src1_spad_half_size; - kparams->dst_row_size_aligned = (uint32_t) layout.dst_spad_half_size; - kparams->src0_spad_half_size = (uint32_t) layout.src0_spad_half_size; - kparams->src1_spad_half_size = (uint32_t) layout.src1_spad_half_size; - kparams->dst_spad_half_size = (uint32_t) layout.dst_spad_half_size; - if (!kparams->use_src1) { - kparams->kernel_id = HTP_SOFTMAX_KERNEL_NOMASK; - } else if (kparams->use_f16) { - kparams->kernel_id = HTP_SOFTMAX_KERNEL_MASK_F16; - } else { - kparams->kernel_id = HTP_SOFTMAX_KERNEL_MASK_F32; - } - - if (src0->ne[1] > 0) kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); - if (src0->ne[2] > 0) kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); - const uint32_t ne12 = src1 ? src1->ne[2] : 1; - const uint32_t ne13 = src1 ? src1->ne[3] : 1; - if (ne12 > 0) kparams->div_ne12 = init_fastdiv_values(ne12); - if (ne13 > 0) kparams->div_ne13 = init_fastdiv_values(ne13); -} -static void ggml_hexagon_precompute_rope_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - struct htp_rope_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; - - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); - const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0; - - struct htp_rope_vtcm_layout layout; - htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors); - - kparams->n_threads = n_threads; - kparams->src0_nrows = src0_nrows; - kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - kparams->vtcm_size = (uint32_t) layout.total_bytes; - kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread; - kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned; - kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned; - kparams->freq_factors_offset = (uint32_t) (layout.bytes_per_thread * n_threads); - kparams->freq_factors_size = (uint32_t) layout.freq_factors_size_aligned; - - if (src0_nrows > 0) { - kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); - kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); + return true; } -} - -static void ggml_hexagon_precompute_ssm_conv_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * dst, - struct htp_ssm_conv_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const uint32_t d_conv = (uint32_t) src1->ne[0]; - const uint32_t d_inner = (uint32_t) src0->ne[1]; - const uint32_t n_t = (uint32_t) dst->ne[1]; - const uint32_t n_s = (uint32_t) dst->ne[2]; - const uint32_t ncs = (uint32_t) src0->ne[0]; - - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (d_inner + 31) / 32); - - kparams->n_threads = n_threads; - kparams->d_conv = d_conv; - kparams->d_inner = d_inner; - kparams->n_t = n_t; - kparams->n_s = n_s; - - const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads; - const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32); - kparams->d_inner_per_thread = d_inner_per_thread; - - kparams->src0_row_size_aligned = hex_round_up(ncs * sizeof(float), 128); - kparams->src1_row_size_aligned = hex_round_up(d_conv * sizeof(float), 128); - kparams->dst_row_size_aligned = hex_round_up(d_inner * sizeof(float), 128); - - if (n_t == 1) { - kparams->d_inner_tile = d_inner_per_thread; - const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; - const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); - const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes; - - const uint32_t src0_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; - const uint32_t src0_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); - const uint32_t vtcm_src0_per_thread = src0_raw_bytes + src0_T_bytes; - - const uint32_t vtcm_dst_per_thread = hex_round_up(d_inner_per_thread * sizeof(float), 128); - - kparams->vtcm_src0_size_per_thread = vtcm_src0_per_thread; - kparams->vtcm_src1_size_per_thread = vtcm_src1_per_thread; - kparams->vtcm_dst_size_per_thread = vtcm_dst_per_thread; - - kparams->vtcm_src0_size = vtcm_src0_per_thread * n_threads; - kparams->vtcm_src1_size = vtcm_src1_per_thread * n_threads; - kparams->vtcm_dst_size = vtcm_dst_per_thread * n_threads; - kparams->vtcm_size = kparams->vtcm_src0_size + kparams->vtcm_src1_size + kparams->vtcm_dst_size; - } else { - const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; - const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); - const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes; + void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) { + const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth); - const size_t vtcm_budget = (sess->vtcm_size > 0 ? sess->vtcm_size / n_threads : (1024 * 1024)); - const size_t avail_for_src0 = vtcm_budget > vtcm_src1_per_thread ? vtcm_budget - vtcm_src1_per_thread : (128 * 1024); + const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs; + const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors; + const size_t o_size = sizeof(htp_op_desc) * rsp.n_ops; + const size_t p_size = sizeof(htp_prof_desc) * rsp.n_ops; - uint32_t d_inner_tile = (uint32_t)((avail_for_src0 / 2) / (ncs * sizeof(float) + n_t * sizeof(float) + 1)); - d_inner_tile = (d_inner_tile / 32) * 32; - if (d_inner_tile == 0) { - d_inner_tile = 32; - } - if (d_inner_tile > d_inner_per_thread) { - d_inner_tile = d_inner_per_thread; + size_t tr_size = 0; + uint32_t n_traces = 0; + if (opt_profile == 3) { + n_traces = opt_optrace; + tr_size = shm_buf->sess->n_threads * n_traces * sizeof(htp_trace_desc); } - kparams->d_inner_tile = d_inner_tile; - const uint32_t src0_tile_raw = hex_round_up(d_inner_tile * ncs * sizeof(float), 128) + 128; - const uint32_t src0_tile_T = hex_round_up(ncs * d_inner_tile * sizeof(float), 128); - const uint32_t vtcm_src0_per_thread = src0_tile_raw + src0_tile_T; + const size_t m_size = b_size + t_size + o_size + p_size + tr_size; + GGML_ASSERT(m_size <= shm_blk_size); - const uint32_t vtcm_dst_per_thread = hex_round_up(d_inner_tile * n_t * sizeof(float), 128); + HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, + (size_t) dbuf.size, b_size, t_size, o_size); - kparams->vtcm_src0_size_per_thread = vtcm_src0_per_thread; - kparams->vtcm_src1_size_per_thread = vtcm_src1_per_thread; - kparams->vtcm_dst_size_per_thread = vtcm_dst_per_thread; + uint8_t * m_ptr = (uint8_t*) dbuf.ptr; + uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size); - kparams->vtcm_src0_size = vtcm_src0_per_thread * n_threads; - kparams->vtcm_src1_size = vtcm_src1_per_thread * n_threads; - kparams->vtcm_dst_size = vtcm_dst_per_thread * n_threads; - kparams->vtcm_size = kparams->vtcm_src0_size + kparams->vtcm_src1_size + kparams->vtcm_dst_size; - } + if (rsp.n_ops > 0 && opt_profile) { + auto & ops = op_cache[slot]; + GGML_ASSERT(rsp.n_ops <= ops.size()); - kparams->div_n_threads = init_fastdiv_values(n_threads); -} + const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr; + const htp_trace_desc * trace_events = nullptr; + if (opt_profile == 3) { + trace_events = (const htp_trace_desc *) (p_ptr + p_size); + } -static void ggml_hexagon_precompute_gated_delta_net_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - struct htp_gdn_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const struct ggml_tensor * q = op->src[0]; - const struct ggml_tensor * k = op->src[1]; - const struct ggml_tensor * v = op->src[2]; - const struct ggml_tensor * g = op->src[3]; - const struct ggml_tensor * state = op->src[5]; - - const uint32_t S_v = (uint32_t) v->ne[0]; - const uint32_t H = (uint32_t) v->ne[1]; - const uint32_t n_tokens = (uint32_t) v->ne[2]; - const uint32_t n_seqs = (uint32_t) v->ne[3]; - const uint32_t K = (uint32_t) ggml_get_op_params_i32(op, 0); - - const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]); - const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]); - const uint32_t total_rows = H * n_seqs; - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, total_rows); - - const bool can_use_hmx = (opt_gdn_select >= 2) && - (sess->n_hmx > 0) && - (S_v % 64 == 0) && - (n_tokens >= HTP_GDN_MIN_TOKENS) && - (g->ne[0] == 1) && - (K == 1); - - struct htp_gdn_hmx_vtcm_layout hmx_layout; - struct htp_gdn_vtcm_layout hvx_layout; - uint32_t n_heads_batch = 1; - - if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, S_v, HTP_GDN_CHUNK_SIZE, total_rows, sess->vtcm_size, n_threads, true, &n_heads_batch)) { - kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED; - kparams->pipeline = hmx_layout.pipeline ? 1 : 0; - kparams->chunk_size = HTP_GDN_CHUNK_SIZE; - kparams->n_chunks = (n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE; - kparams->n_heads_batch = (uint16_t) n_heads_batch; - kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes; - kparams->state_aligned = (uint32_t) hmx_layout.state_f32_bytes; - kparams->vtcm_per_thread = (uint32_t) (hmx_layout.total_bytes / (n_threads > 0 ? n_threads : 1)); - } else { - htp_gdn_vtcm_layout_build(&hvx_layout, S_v, n_threads); - kparams->kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT; - kparams->pipeline = 0; - kparams->n_heads_batch = 1; - kparams->state_aligned = (uint32_t) hvx_layout.state_aligned; - kparams->vtcm_per_thread = (uint32_t) hvx_layout.bytes_per_thread; - kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes; - } - - kparams->n_threads = n_threads; - kparams->S_v = S_v; - kparams->H = H; - kparams->n_tokens = n_tokens; - kparams->n_seqs = n_seqs; - kparams->K = K; - kparams->total_rows = total_rows; - kparams->rows_per_thread = (total_rows + kparams->n_threads - 1) / kparams->n_threads; - kparams->kda = (g->ne[0] == S_v) ? 1 : 0; - kparams->state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float)); - kparams->state_size_per_snap = S_v * S_v * H * n_seqs; - kparams->scale = 1.0f / sqrtf((float) S_v); - - if (H > 0) kparams->div_H = init_fastdiv_values(H); - if (q->ne[1] > 0) kparams->div_q1 = init_fastdiv_values((uint32_t) q->ne[1]); - if (k->ne[1] > 0) kparams->div_k1 = init_fastdiv_values((uint32_t) k->ne[1]); - if (rq3 > 0) kparams->div_rq3 = init_fastdiv_values(rq3); - if (rk3 > 0) kparams->div_rk3 = init_fastdiv_values(rk3); - if (kparams->n_threads > 0) kparams->div_n_threads = init_fastdiv_values(kparams->n_threads); -} + ggml_hexagon_dump_batch_prof(shm_buf->sess->c_name(), rsp); -static void ggml_hexagon_precompute_sort_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op, - bool is_top_k, - struct htp_sort_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t ne00 = src0->ne[0]; - const uint32_t k = dst->ne[0]; - - int32_t order = GGML_SORT_ORDER_DESC; - if (!is_top_k) { - order = ((const int32_t *) op->op_params)[0]; - } - - const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4; - const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024); - - struct htp_sort_vtcm_layout layout; - bool ok = htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k); - GGML_ASSERT(ok); - - kparams->n_threads = (int32_t) layout.n_threads; - kparams->total_rows = (int32_t) total_rows; - kparams->row_start = 0; - kparams->row_end = (int32_t) total_rows; - kparams->ne00 = (int32_t) ne00; - kparams->k = (int32_t) k; - kparams->order = order; - kparams->is_top_k = is_top_k ? 1 : 0; - kparams->use_dma = 1; - kparams->chunk_elems = (int32_t) layout.chunk_elems; - kparams->n_chunks = (int32_t) layout.n_chunks; - kparams->vtcm_size = (int32_t) layout.total_bytes; - kparams->phase1_slot_size = (int32_t) layout.phase1_slot_size; - kparams->merge_values_off = (int32_t) layout.merge_values_off; - kparams->merge_indices_off = (int32_t) layout.merge_indices_off; - kparams->merge_elems = (int32_t) layout.merge_elems; - kparams->n_slots = (int32_t) layout.n_slots; -} + for (uint32_t i = 0; i < rsp.n_ops; i++) { + ggml_hexagon_dump_op_prof(shm_buf->sess->c_name(), ops[i], pd[i]); + } -static void ggml_hexagon_precompute_fused_mmnx_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, // W0 - const struct ggml_tensor * src1, // x - int32_t n_weights, - struct htp_mm_kernel_params * kparams -) { - memset(kparams, 0, sizeof(*kparams)); - kparams->n_threads = sess->n_threads; - - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; - - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; - - const int wtype = src0->type; - const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype); - const int ne00_padded = is_repack ? hex_round_up(ne00, 32) : ne00; - const int ne01_padded = is_repack ? hex_round_up(ne01, 32) : ne01; - const int ne11_padded = hex_round_up(ne11, 32); - - const size_t vtcm_budget = sess->vtcm_size; - const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); - - bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 2); - if (hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible(src0, src1, nullptr, ne01_padded, false, is_batched)) { - if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, src1, nullptr, wtype, ne00_padded, ne01_padded, ne02, ne11, ne12, ne11_padded, false, is_batched, 0, vtcm_budget, kparams)) { - kparams->n_weights = n_weights; - goto finalize; + ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, shm_buf->sess->n_threads, n_traces); } } +}; - if (!is_repack) { - kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; - return; +void ggml_hexagon_session::flush_peers() { + auto vpeers = std::move(virt_peers); + virt_peers.clear(); + for (auto * peer : vpeers) { + peer->flush_sync(); } - { - const int src1_nrows = ne11 * ne12 * ne13; - const size_t src1_row_size = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - const size_t src0_row_size = src0->nb[1]; - - uint32_t best_n_prefetch = 16; - - if (is_repack) { - const uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; - best_n_prefetch = 2; - for (uint32_t d = max_prefetch; d >= 2; d /= 2) { - struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, src1_row_size, 0, d, false, true - ); - if (L.total_bytes <= sess->vtcm_size) { - best_n_prefetch = d; - break; - } - } - } - - struct htp_mm_hvx_vtcm_layout L; - bool try_tiled = (opt_mm_select >= 1); - - // Test tiled first - htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, src1_row_size, 0, best_n_prefetch, false, true - ); - - if (try_tiled && L.total_bytes <= sess->vtcm_size) { - kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - kparams->vtcm_size = L.total_bytes; - kparams->n_prefetch = best_n_prefetch; - kparams->n_weights = n_weights; - } else { - kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; - return; - } + auto ppeers = std::move(phys_peers); + phys_peers.clear(); + for (auto * peer : ppeers) { + peer->flush_async(); } -finalize: - kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11); - kparams->div_ne1 = init_fastdiv_values(ne11); - kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); - kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); - kparams->div_ne12 = init_fastdiv_values(ne12); -} - -static void ggml_hexagon_precompute_fused_mmidnx_params( - const struct ggml_hexagon_session * sess, - const struct ggml_tensor * src0, // W0 - const struct ggml_tensor * src1, // x - const struct ggml_tensor * dst, // dst0 - int32_t n_weights, - struct htp_mm_kernel_params * kparams -) { - ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, 0, 0, kparams); - kparams->n_weights = n_weights; -} - -static bool ggml_hexagon_tensor_is_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) { - return t && t->buffer && ggml_backend_buft_is_host(t->buffer->buft); - GGML_UNUSED(sess); + for (auto & sub : this->mdev.sessions) { + sub->flush_peers(); + } } -static bool ggml_hexagon_tensor_is_non_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) { - return t && t->buffer && !ggml_backend_buft_is_host(t->buffer->buft); - GGML_UNUSED(sess); +void ggml_hexagon_session::flush_async() { + flush_peers(); + flush_batch(); } -static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) { - const struct ggml_tensor * src0 = dst->src[0]; - const struct ggml_tensor * src1 = dst->src[1]; - - if (dst->type != GGML_TYPE_F32) { - return false; +void ggml_hexagon_session::flush_pending(bool all) { + for (auto & sub : this->mdev.sessions) { + sub->flush_pending(all); + if (sub->last_error > HTP_STATUS_OK) { + this->last_error = sub->last_error.load(); + } } - if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { - return false; - } + while (this->batch_rsp_seq < this->batch_req_seq) { + struct htp_opbatch_rsp rsp; + uint32_t rsp_size; + uint32_t flags; - switch (src0->type) { - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q2_K: - if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) { - return false; - } + struct dspqueue_buffer dbuf; + uint32_t n_dbufs; - if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q5_K || src0->type == GGML_TYPE_Q4_K || - src0->type == GGML_TYPE_Q3_K || src0->type == GGML_TYPE_Q2_K) ? QK_K : 32)) { - return false; - } + // Read response packet from queue + const uint32_t timeo = opt_oppoll ? 0 : DSPQUEUE_TIMEOUT; - if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) { - return false; - } - if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) { - return false; - } + int err = dspqueue_read(this->queue, &flags, 1, &n_dbufs, &dbuf, sizeof(rsp), &rsp_size, (uint8_t *) &rsp, timeo); + if (err == AEE_EEXPIRED || err == AEE_EWOULDBLOCK) { + continue; + } - if (!src0->buffer) { - sess->needs_repack.insert(src0); - } - break; + if (err != 0) { + GGML_ABORT("ggml-hex: dspqueue_read failed: 0x%08x\n", (unsigned) err); + } - case GGML_TYPE_F16: - if (src0->nb[1] < src0->nb[0]) { - return false; - } - if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) { - return false; - } - if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) { - return false; - } - break; + // Basic sanity checks + if (rsp_size != sizeof(rsp) || n_dbufs != 1) { + GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs); + } - case GGML_TYPE_F32: - if (src1->type != GGML_TYPE_F32) { - return false; - } - if (src0->nb[1] < src0->nb[0]) { - return false; - } - if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) { - return false; - } - if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) { - return false; + if (rsp.status > HTP_STATUS_OK) { + GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status)); + this->last_error = rsp.status; + for (auto & sub : this->mdev.sessions) { + sub->last_error = rsp.status; } - break; + } - default: - return false; - } + op_queue->pop(rsp, dbuf); - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_matmul_params(sess, src0, src1, dst, &kparams); - if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: %s supported MUL_MAT VTCM size needed (%d) > budget (%zu)\n", sess->c_name(), kparams.vtcm_size, sess->vtcm_size); - return false; - } + GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1); + this->batch_rsp_seq = rsp.seq; - return true; + if (!all) break; + } } -static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; +void ggml_hexagon_session::flush_sync(bool all) { + flush_async(); + flush_pending(all); +} - if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) { - return false; +void ggml_hexagon_session::start_batch() { + if (this->mdev.count > 1) { + enqueue_mdev_group(); } +} - if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { - return false; - } +void ggml_hexagon_session::flush_batch(size_t min_ops) { + if (op_batch->n_ops < min_ops) { return; } - switch (src0->type) { - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q2_K: - if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) { - return false; - } + op_batch->sort_buffers(); - if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q5_K || src0->type == GGML_TYPE_Q4_K || - src0->type == GGML_TYPE_Q3_K || src0->type == GGML_TYPE_Q2_K) ? QK_K : 32)) { - return false; - } + htp_opbatch_req req {}; + dspqueue_buffer dbuf{}; - if (!src0->buffer) { - sess->needs_repack.insert(src0); - } - break; + const uint64_t seq = ++this->batch_req_seq; - default: - return false; - } + op_batch->update_mdev_group(this->mdev.idx); - struct htp_mm_kernel_params kparams; - ggml_hexagon_precompute_matmul_params(sess, src0, src1, dst, &kparams); - if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams.vtcm_size > sess->vtcm_size) { - HEX_VERBOSE("ggml-hex: %s supported MUL_MAT_ID VTCM size needed (%d) > budget (%zu)\n", sess->c_name(), kparams.vtcm_size, sess->vtcm_size); - return false; + if (!op_queue->push(req, dbuf, op_batch, seq)) { + flush_pending(false); + op_queue->push(req, dbuf, op_batch, seq); } - return true; -} + for (auto & sub : this->mdev.sessions) { + htp_opbatch_req sub_req {}; + dspqueue_buffer sub_dbuf{}; -static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * dst = op; + sub->batch_req_seq = seq; + op_batch->update_mdev_group(sub->mdev.idx); - if (src0->type == GGML_TYPE_F32) { - if (src1->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; - } - } - else if (src0->type == GGML_TYPE_F16) { - if (src1->type != GGML_TYPE_F16) { - return false; - } - if (dst->type != GGML_TYPE_F16) { - return false; + if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) { + sub->flush_pending(false); + sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq); } - } - else { - return false; - } - if (ggml_is_permuted(src0) || ggml_is_permuted(dst)) { - return false; - } - if (!ggml_are_same_shape(src0, dst)) { - return false; - } - if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) { - return false; - } - - struct htp_binary_kernel_params kparams; - return ggml_hexagon_precompute_binary_params(sess, op_remap_to_htp(op), src0, src1, dst, &kparams); -} - -static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; + HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size); - if (!src2) { - return false; - } - if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) { - return false; - } - if (!ggml_are_same_shape(src0, dst)) { - return false; - } - if (src1->ne[0] != src0->ne[0] || src1->ne[2] != 1 || src1->ne[3] != 1) { - return false; - } - if (src2->ne[0] != src0->ne[1] || src2->ne[1] != src0->ne[2]) { - return false; - } - if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t)) { - return false; + int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT); + if (err != 0) { + GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err); + } } - // REVISIT: add support for non-contiguous tensors - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { - return false; + HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size); + + int err = dspqueue_write(this->queue, 0, 1, &dbuf, sizeof(req), (const uint8_t*) &req, DSPQUEUE_TIMEOUT); + if (err != 0) { + GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err); } - struct htp_binary_kernel_params kparams; - return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams); + op_batch->reset(); } -static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) { - return false; - } - if (dst->type != src0->type) { - return false; - } - if (!ggml_is_contiguous_rows(src0)) { - return false; - } - - // F16 device kernels only cover this explicit whitelist (must stay in sync with - // the is_f16 whitelist in execute_op_unary(), unary-ops.c). - if (src0->type == GGML_TYPE_F16) { - switch (op->op) { - case GGML_OP_NORM: - case GGML_OP_RMS_NORM: - case GGML_OP_L2_NORM: - case GGML_OP_SCALE: - case GGML_OP_CLAMP: - case GGML_OP_SQR: - case GGML_OP_SQRT: - case GGML_OP_LOG: - break; - case GGML_OP_UNARY: - if (ggml_get_unary_op(op) != GGML_UNARY_OP_ABS && - ggml_get_unary_op(op) != GGML_UNARY_OP_STEP) { - return false; - } - break; - default: - return false; +void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { + auto clone_tensor_buffer = [this](const ggml_tensor * t) { + auto sbuf = this->mmap_tensor(t); + if (!sbuf) return; + if (sbuf->sess != this) { + this->clone_buffer(sbuf); } - } + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(sbuf); + } + }; - if (!ggml_are_same_shape(src0, dst)) { - return false; + for (auto t : node.get_inputs()) { + clone_tensor_buffer(t); } - - // dst must be contiguous; src0 may be non-contiguous - if (!ggml_is_contiguous(dst)) { - return false; + for (auto t : node.get_outputs()) { + clone_tensor_buffer(t); } - return true; - - GGML_UNUSED(sess); -} - -static bool ggml_hexagon_supported_sum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; + const bool fits_tensors = op_batch->fit_tensors(node); + if (opt_opfusion && fits_tensors && op_batch->try_fuse(node)) { + return; } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; + if (op_batch->full() || !fits_tensors) { + flush_async(); } - return true; + if (op_batch->empty()) { + start_batch(); + if (!op_batch->fit_tensors(node)) { + GGML_ABORT("ggml-hex: %s op does not fit into empty batch (vmem/tensor/buffer limit exceeded)\n", c_name()); + } + } - GGML_UNUSED(sess); + op_batch->add_op(node); } -static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; +void ggml_hexagon_session::enqueue_mdev_group() { + htp_opnode group_node(HTP_OP_MDEV_GROUP); - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; - } + ggml_tensor dummy_t = make_fence_tensor(this->mdev_fence_slot, this->mdev.count); + dummy_t.op_params[0] = (int32_t) this->mdev.idx; - // TODO: add support for non-contigiuos tensors - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; - } + ggml_tensor * node = group_node.add_dummy(dummy_t); + node->src[0] = node; + group_node.init(node); + group_node.outputs.clear(); - return true; + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(this->fence_buf); + } - GGML_UNUSED(sess); + op_batch->add_op(group_node); } -static bool ggml_hexagon_supported_argmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; +void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) { + const bool with_fence = sync_tensor != nullptr; + htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY); - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_I32) { - return false; + ggml_tensor* node = cpy_node.add_dummy(*dst); + node->op = GGML_OP_CPY; + node->src[0] = const_cast<ggml_tensor *>(src); + node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr; + if (with_fence) { + node->op_params[0] = (int32_t) fence_seq; } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; + cpy_node.init(node); + const bool ok = htp_kparams_solve(this, node, cpy_node.kernel_params); + if (ok && !with_fence && cpy_node.kparams<htp_copy_kernel_params>()->total_elems == 0) { + return; } + this->enqueue_op(cpy_node); +} - return true; +void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) { + htp_opnode sync_node(HTP_OP_FENCE); + + ggml_tensor* node = sync_node.add_dummy(*sync_tensor); + node->op = GGML_OP_NONE; + node->src[0] = node; + node->op_params[0] = (int32_t) fence_seq; + node->op_params[1] = wait ? 0 : 1; - GGML_UNUSED(sess); + sync_node.init(node); + this->enqueue_op(sync_node); } -static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * dst = op; +void ggml_hexagon_session::enqueue_allreduce( + const ggml_tensor * dst, + const std::vector<const ggml_tensor *> & src_tensors, + const std::vector<const ggml_tensor *> & sync_tensors, + uint32_t rank, + uint32_t n_ranks, + uint32_t fence_seq_entry, + uint32_t fence_seq_exit +) { + htp_opnode ar_node(HTP_OP_ALLREDUCE); - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) { - return false; - } - if (dst->type != src0->type) { - return false; - } + ggml_tensor* node = ar_node.add_dummy(*dst); + node->op = GGML_OP_NONE; + node->op_params[0] = (int32_t) fence_seq_entry; + node->op_params[1] = (int32_t) fence_seq_exit; - if (!ggml_is_contiguous_1(src0)) { - return false; + ar_node.init(node); + + ar_node.inputs.clear(); + for (size_t i = 0; i < src_tensors.size(); i++) { + ar_node.inputs.push_back(src_tensors[i]); } - if (!ggml_is_contiguous(dst)) { - return false; + for (size_t i = 0; i < sync_tensors.size(); i++) { + ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i])); } - if (src1) { - if (src1->type != src0->type) { - return false; - } - if (!ggml_are_same_shape(src0, src1)) { - return false; - } - if (!ggml_is_contiguous_1(src1)) { - return false; - } + ar_node.outputs.clear(); + for (size_t i = 0; i < src_tensors.size(); i++) { + ar_node.outputs.push_back(src_tensors[i]); } - return true; + htp_kparams_solve_allreduce( + this, dst, rank, n_ranks, false, false, /*is_shard_ok=*/ false, + ar_node.kparams<htp_allreduce_kernel_params>()); - GGML_UNUSED(sess); + this->enqueue_op(ar_node); } -static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; - - if (src2) { - return false; // FIXME: add support for sinks - } +ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) { + if (!t) return nullptr; - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; + auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); + if (!sbuf->mapped) { + const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + const bool extended = opt_dma64 && is_weight; + sbuf->mmap(extended); } + return sbuf; +} - if (src0->ne[2] > 512) { - return false; - } +bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf) +{ + GGML_ASSERT(sbuf && sbuf->mem); + if (sbuf->sess == this) return true; - if (src1) { - if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { - return false; - } - if (src0->ne[0] != src1->ne[0]) { - return false; - } - if (src1->ne[1] < src0->ne[1]) { - return false; - } - if (src0->ne[2] % src1->ne[2] != 0) { - return false; - } - if (src0->ne[3] % src1->ne[3] != 0) { - return false; - } - } + auto mem = sbuf->mem; + int fd = mem->fd; - if (src1) { - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { - return false; - } - } else { - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; - } - } + GGML_ASSERT(fd >= 0); - // Reject non-HVX-aligned sizes when ne[0] > HVX_F32_LANES - // The HVX softmax implementation has issues with tail handling for larger non-aligned sizes - // Small sizes (ne[0] <= 32) work correctly with tail-only processing - const int64_t ne0 = src0->ne[0]; - if (ne0 > 32 && (ne0 & (32 - 1)) != 0) { - return false; - } + if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true; - // HVX vector size constraints for softmax - #define SOFTMAX_MAX_ROW_SIZE 131072 // 128K elements max for numerical precision + GGML_ASSERT(sbuf->mapped); - // Reject very large row sizes to avoid numerical precision issues - // Softmax accumulation over many elements can lead to precision loss - if (ne0 > SOFTMAX_MAX_ROW_SIZE) { - return false; - } + HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(), + sbuf->c_name(), sbuf->base(), sbuf->size(), fd); - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); - struct htp_softmax_vtcm_layout layout; - htp_softmax_vtcm_layout_build(&layout, src0->ne[0], src1 ? src1->ne[0] : 1, src1 != nullptr, src1 && src1->type == GGML_TYPE_F16, n_threads); - if (layout.total_bytes > sess->vtcm_size) { + auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf); + try { + clone->mmap(sbuf->extended); + } catch (const std::exception & exc) { + GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what()); return false; } + this->cloned_buffers[fd] = std::move(clone); + mem->mapped_clones.insert(this); return true; - - GGML_UNUSED(sess); } -static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; // values - const struct ggml_tensor * src1 = op->src[1]; // indices - const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op; +void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); - if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) { - return false; - } + auto mem = sbuf->mem; + int fd = mem->fd; - if (src0->type != GGML_TYPE_F32) { - return false; - } + GGML_ASSERT(fd >= 0); - if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) { - return false; + auto it = this->cloned_buffers.find(fd); + if (it != this->cloned_buffers.end()) { + auto clone = std::move(it->second); + this->cloned_buffers.erase(it); } + mem->mapped_clones.erase(this); +} - if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) { - return false; - } +void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); - return true; + auto mem = sbuf->mem; + std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end()); - GGML_UNUSED(sess); + for (auto * sess : sessions) { + sess->release_buffer(sbuf); + } } -static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; // values - const struct ggml_tensor * src1 = op->src[1]; // indices - const struct ggml_tensor * dst = op; +static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { + // Allocate a bunch pinned buffers till failure. + // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device. + // Typically we're going to allocate all/most of these buffers anyway for the model weights. - if (src0->type == GGML_TYPE_Q4_0 && src0->view_src) { - return false; - } + std::vector<ggml_hexagon_shared_buffer *> sbufs; - const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0; - bool is_repacked = false; - if (src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) && src0_base->extra) { - const auto * extra = (const ggml_hexagon_tensor_extra *) src0_base->extra; - is_repacked = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0; - if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0) { - return false; - } - } - is_repacked = is_repacked || sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0); + const size_t MiB = 1024 * 1024; + const size_t GiB = MiB * 1024; - // View offsets use the raw quantized layout and cannot address a tiled allocation. - if (src0->view_src && is_repacked) { - return false; - } + size_t vmem = 0; + size_t step = 256u * MiB; - if (src0->type == GGML_TYPE_Q4_0 && src0->buffer && !is_repacked) { - return false; - } + try { + sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; + sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; + sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; - if (src0->type != dst->type && src0->ne[0] < 32) { - return false; - } + while (1) { + sbufs.push_back(new ggml_hexagon_shared_buffer(sess, step, true)); + vmem += step; + } + } catch (...) { } - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && - src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) { - return false; - } + for (auto b : sbufs) { delete b; } - if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) { - return false; - } + return vmem - step; // backoff to account for overhead from internal mappings +} - if (src0->type == dst->type) { - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) { - return false; - } - } else if (src0->type == GGML_TYPE_I32) { - return false; - } else if (dst->type != GGML_TYPE_F32) { - return false; - } +void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) { + int phys_idx = config.physical_idx; + int virt_idx = config.virtual_idx; - // Empty recurrent-state gathers are skipped at execution; do not split the graph for them. - if (ggml_is_empty(op)) { - return true; - } + this->valid_session = false; + this->valid_handle = false; + this->valid_queue = false; + this->valid_iface = false; - struct htp_get_rows_kernel_params kparams; - ggml_hexagon_precompute_get_rows_params(sess, src0, src1, dst, &kparams); - if (kparams.n_threads == 0 || (size_t) kparams.vtcm_size > sess->vtcm_size) { - return false; - } + this->name = config.name; + this->phys_idx = phys_idx; + this->virt_idx = virt_idx; + this->domain_id = config.domain_id; + this->session_id = 0; + this->batch_req_seq = 0; + this->batch_rsp_seq = 0; + this->last_error = HTP_STATUS_OK; - // Q4_0 has no raw fallback. Mark only accepted tensors for repacking. - if (src0->type == GGML_TYPE_Q4_0 && !src0->buffer) { - sess->needs_repack.insert(src0); + GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx); + + if (config.domain_id < 0 || config.domain_name.empty()) { + GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx); + throw std::runtime_error("ggml-hex: invalid physical CDSP core"); } - return true; + const std::string & dom_name = config.domain_name; - GGML_UNUSED(sess); -} + // Create new session if virtual_idx > 0 + if (virt_idx > 0) { + struct remote_rpc_reserve_new_session n {}; + n.domain_name_len = dom_name.size(); + n.domain_name = const_cast<char *>(dom_name.c_str()); + n.session_name = const_cast<char *>(this->name.c_str()); + n.session_name_len = this->name.size(); + n.session_id = virt_idx; + + int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); + if (err != AEE_SUCCESS) { + GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n", + this->c_name(), phys_idx, virt_idx, err); + throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)"); + } -static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; // values - const struct ggml_tensor * dst = op; // indices + // Save the IDs + this->session_id = n.session_id; + this->domain_id = n.effective_domain_id; + this->valid_session = true; + } else { + struct remote_rpc_effective_domain_id eff {}; + eff.domain_name = const_cast<char *>(dom_name.c_str()); + eff.domain_name_len = dom_name.size(); + eff.session_id = 0; - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) { - return false; + int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff)); + if (err == AEE_SUCCESS) { + this->domain_id = eff.effective_domain_id; + } else { + GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n", + this->name.c_str(), err, this->domain_id); + } } - const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4; - const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024); - - struct htp_sort_vtcm_layout layout; - if (!htp_sort_solve_layout(&layout, src0->ne[0], total_rows, dst->ne[0], n_threads_max, vtcm_budget, false)) { - return false; + // Enable unsigned modules + { + struct remote_rpc_control_unsigned_module u; + u.domain = this->domain_id; + u.enable = 1; + int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); + if (err != AEE_SUCCESS) { + GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); + throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); + } } - return true; -} + auto open_session = [&](const std::string & prefix, std::string & uri_out) -> int { + const std::string htp_uri = prefix + "libggml-htp-v" + std::to_string(opt_arch) + + ".so?htp_iface_skel_handle_invoke&_modver=1.0"; -static bool ggml_hexagon_supported_top_k(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; // values - const struct ggml_tensor * dst = op; // indices - - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) { - return false; - } + std::vector<char> session_uri(htp_uri.size() + 256); - const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4; - const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024); + struct remote_rpc_get_uri u = {}; + u.session_id = this->session_id; + u.domain_name = const_cast<char *>(dom_name.c_str()); + u.domain_name_len = dom_name.size(); + u.module_uri = const_cast<char *>(htp_uri.c_str()); + u.module_uri_len = htp_uri.size(); + u.uri = session_uri.data(); + u.uri_len = session_uri.size(); - struct htp_sort_vtcm_layout layout; - if (!htp_sort_solve_layout(&layout, src0->ne[0], total_rows, dst->ne[0], n_threads_max, vtcm_budget, true)) { - return false; - } + int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u)); + if (err != AEE_SUCCESS) { + snprintf(session_uri.data(), session_uri.size(), "%s&_dom=%s&_session=%u", + htp_uri.c_str(), dom_name.c_str(), this->session_id); - return true; -} + HEX_VERBOSE("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", + this->c_name(), phys_idx, virt_idx, err, session_uri.data()); + } -static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; + uri_out = session_uri.data(); + return htp_iface_open(session_uri.data(), &this->handle); + }; - if (!ggml_are_same_shape(src0, dst)) { - return false; - } + const std::string lib_dir = htp_lib_dir(); - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) { - return false; - } + std::string session_uri; + int err = AEE_EFAILED; - if (src0->ne[0] <= 0) { - return false; - } + // 1. Try opening session with absolute path in URI + if (!lib_dir.empty()) { + std::string abs_prefix = "file:///" + lib_dir; + if (abs_prefix.back() != '/' && abs_prefix.back() != '\\') { + abs_prefix += "/"; + } - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - if (src0_nrows == 0) { - return false; + HEX_VERBOSE("ggml-hex: %s trying absolute path URI: %s\n", this->c_name(), abs_prefix.c_str()); + err = open_session(abs_prefix, session_uri); + if (err != AEE_SUCCESS) { + HEX_VERBOSE("ggml-hex: %s failed to open session with absolute path : uri %s error 0x%x\n", + this->c_name(), session_uri.c_str(), err); + } } - const int32_t * op_params = &op->op_params[0]; - const int n_dims = op_params[1]; - const int mode = op_params[2]; - const int n_offs = op_params[15]; - - // llama probes weight placement with a dummy rope where every param is 0 (llama-model-loader.cpp). - // Rejecting it puts rope_freqs on the CPU, which then splits the graph at every full-attention layer. - if (n_dims < 0 || n_dims % 2 != 0) { - return false; + // 2. Fallback to DSP search path via FASTRPC_MANAGE_DSP_LIBRARY_PATH + if (err != AEE_SUCCESS && !lib_dir.empty()) { + int sp_err = htpdrv_set_dsp_search_path(this->domain_id, lib_dir, this->c_name()); + if (sp_err == AEE_SUCCESS) { + err = open_session("file:///", session_uri); + if (err != AEE_SUCCESS) { + HEX_VERBOSE("ggml-hex: %s failed to open session with search path : uri %s error 0x%x\n", + this->c_name(), session_uri.c_str(), err); + } + } } - // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) - if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) { - return false; + // 3. Fallback to process environment variables + if (err != AEE_SUCCESS && !lib_dir.empty()) { + HEX_VERBOSE("ggml-hex: %s falling back to process env for DSP search path\n", this->c_name()); + htpdrv_set_dsp_env_path(lib_dir, this->c_name()); + err = open_session("file:///", session_uri); } - float freq_base; - memcpy(&freq_base, op_params + 5, sizeof(float)); - if (freq_base < 0.0f) { - return false; + // Bare name attempt if lib_dir is empty and nothing was tried yet + if (err != AEE_SUCCESS && lib_dir.empty()) { + err = open_session("file:///", session_uri); } - if (mode != GGML_ROPE_TYPE_NORMAL && - mode != GGML_ROPE_TYPE_NEOX && - mode != GGML_ROPE_TYPE_MROPE && - mode != GGML_ROPE_TYPE_VISION && - mode != GGML_ROPE_TYPE_IMROPE) { - return false; + if (err != AEE_SUCCESS) { + GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri.c_str(), err); + throw std::runtime_error("ggml-hex: failed to open session (see log for details)"); } - const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0; - - // n_dims == ne0/2, so the rotation spans the full row - if (mode == GGML_ROPE_TYPE_VISION) { - if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) { - return false; - } - } + this->valid_handle = true; - if (is_mrope) { - const int32_t * sections = op_params + 11; - if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) { - return false; + // Query HW info and resolve session options + { + unsigned int hw_n_threads = 0; + unsigned int hw_n_hvx = 0; + unsigned int hw_n_hmx = 0; + unsigned long long hw_vtcm_size = 0; + int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size); + if (hw_err == 0) { + this->n_threads = (std::min)((uint32_t) HTP_OP_MAX_NTHREADS, (uint32_t) hw_n_threads); + const uint32_t max_n_hvx = (std::min)({this->n_threads - 1, (uint32_t) hw_n_hvx, (uint32_t) HTP_OP_MAX_NHVX}); + this->n_hvx = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_hvx) : max_n_hvx; + this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; + this->vtcm_size = (uint64_t)hw_vtcm_size; + if (opt_vtcm_size > 0) { + if (opt_vtcm_size > hw_vtcm_size) { + GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size); + } + this->vtcm_size = opt_vtcm_size; + } + GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n", + this->c_name(), this->n_threads, this->n_hvx, this->n_hmx, + (unsigned long long)(this->vtcm_size / (1024 * 1024))); + } else { + GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err); + this->n_threads = (std::min)(8u, (uint32_t) HTP_OP_MAX_NTHREADS); + const uint32_t max_n_hvx = (std::min)(this->n_threads - 1, (uint32_t) HTP_OP_MAX_NHVX); + this->n_hvx = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_hvx) : (std::min)(4u, max_n_hvx); + this->n_hmx = (opt_nhmx != 0) ? 1 : 0; + this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024; } } - const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2]; - if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) { - return false; - } + // Enable FastRPC QoS mode + { + struct remote_rpc_control_latency l; + l.enable = 1; - if (src2) { - if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) { - return false; - } - if (src2->ne[0] < (n_dims / 2)) { - return false; + int err = remote_handle64_control(this->handle, DSPRPC_CONTROL_LATENCY, (void *) &l, sizeof(l)); + if (err != 0) { + GGML_LOG_WARN("ggml-hex: failed to enable fastrpc QOS mode: 0x%08x\n", (unsigned) err); } } - // src0/dst elements within a row must be contiguous (nb[0] == sizeof(float)). - // nb[1] may exceed ne[0]*sizeof(float) when the tensor is a strided view of a larger one - if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { - return false; - } - if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) { - return false; - } - - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); - const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0; - - struct htp_rope_vtcm_layout layout; - htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors); - if (layout.total_bytes > sess->vtcm_size) { - return false; - } - - return true; -} - -static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * dst = op; + GGML_LOG_INFO("ggml-hex: %s new session : session-id %d domain-id %d uri %s handle 0x%lx\n", this->c_name(), + this->session_id, this->domain_id, session_uri.c_str(), (unsigned long) this->handle); - // Only support FP32 for now - if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; - } + const size_t req_q_size = (sizeof(htp_opbatch_req) * opt_opqueue * 2) + 1024; + const size_t rsp_q_size = (sizeof(htp_opbatch_rsp) * opt_opqueue * 2) + 1024; - // Check IO tensor shapes and dims - if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) { - return false; // src0 should be effectively 3D + // Now let's setup the DSP queue + err = dspqueue_create(this->domain_id, + 0, // Flags + req_q_size, // Request queue size (in bytes) + rsp_q_size, // Response queue size (in bytes) + nullptr, // Read packet callback (we handle reads explicitly) + nullptr, // Error callback (we handle errors during reads) + (void *) this, // Callback context + &queue); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: %s dspqueue_create failed: 0x%08x\n", this->name.c_str(), (unsigned) err); + throw std::runtime_error("ggml-hex: failed to create dspqueue (see log for details)"); } - const int d_conv = src1->ne[0]; - const int d_inner = src0->ne[1]; - const int n_t = dst->ne[1]; - const int n_s = dst->ne[2]; + this->valid_queue = true; - if (d_conv == 0 || d_conv > 32 || d_inner == 0) { - return false; - } - if (src0->ne[0] != d_conv - 1 + n_t || src0->ne[1] != d_inner || src0->ne[2] != n_s) { - return false; - } - if (src1->ne[0] != d_conv || src1->ne[1] != d_inner) { - return false; - } - if (dst->ne[0] != d_inner || dst->ne[1] != n_t || dst->ne[2] != n_s) { - return false; - } - if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { - return false; - } - if (src0->nb[1] != src0->ne[0] * sizeof(float) || src1->nb[1] != src1->ne[0] * sizeof(float)) { - return false; + // Export queue for use on the DSP + err = dspqueue_export(queue, &this->queue_id); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: dspqueue_export failed: 0x%08x\n", (unsigned) err); + throw std::runtime_error("ggml-hex: dspqueue export failed (see log for details)"); } - struct htp_ssm_conv_kernel_params kparams; - ggml_hexagon_precompute_ssm_conv_params(sess, src0, src1, dst, &kparams); - if ((size_t) kparams.vtcm_size > sess->vtcm_size) { - return false; + if (opt_etm) { + err = htp_iface_etm(this->handle, 1); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: failed to enable ETM tracing: 0x%08x\n", (unsigned) err); + } } - return true; -} - -static bool ggml_hexagon_supported_im2col(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * dst = op; + // Allocate buffers and state for op batching + this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); - // For now support F32->F32 and F32->F16 only. - if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) { - return false; + this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024); + if (this->mdev.count > 1) { + this->mdev_fence_slot = this->alloc_fence(this->mdev.count); } - if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { - return false; + if (!opt_vmem) { + opt_vmem = ggml_hexagon_measure_max_vmem(this); + GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem); } + const size_t shm_size = this->op_queue->shm_size(); + this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem; - GGML_UNUSED(sess); - return true; -} - -static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; + this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; + // Start dspqueue/opbatch processing + err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_hvx, opt_nhmx, opt_vtcm_size, this->max_vmem, + (uint32_t) opt_power_mode); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); + throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); } + this->valid_iface = true; + GGML_LOG_INFO("ggml-hex: %s power mode: %s\n", this->c_name(), htp_power_mode_to_str(opt_power_mode)); - const int32_t lp0 = ((const int32_t *) op->op_params)[0]; - const int32_t rp0 = ((const int32_t *) op->op_params)[1]; - const int32_t circular = ((const int32_t *) op->op_params)[8]; + if (opt_profile) { + htp_iface_pmu_conf pmu_conf{}; + std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events); - if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) { - return false; + err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err); + } } +} - return true; +void ggml_hexagon_session::release() noexcept(true) { + GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str()); - GGML_UNUSED(sess); -} + this->mdev.sessions.clear(); -static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; + int err; - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; + if (this->valid_iface) { + // Stop dspqueue/opbatch processing + err = htp_iface_stop(this->handle); + if (err != 0) { + GGML_ABORT("ggml-hex: htp_iface_stop failed: 0x%08x\n", (unsigned) err); + } } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; + delete this->op_batch; + delete this->op_queue; + for (auto & it : this->cpy_fence_slots) { + free_fence((void *) it.second, 1); } + this->cpy_fence_slots.clear(); - return true; - - GGML_UNUSED(sess); -} - -static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - // diag only supports F32 currently - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; + if (this->fence_buf) { + unclone_buffer(this->fence_buf); + delete this->fence_buf; + this->fence_buf = nullptr; } - - // Input must have ne[1] == 1 (vector input) - if (src0->ne[1] != 1) { - return false; + while (!this->cloned_buffers.empty()) { + release_buffer(this->cloned_buffers.begin()->second.get()); } - // Output must be square in first two dimensions - if (dst->ne[0] != dst->ne[1] || dst->ne[0] != src0->ne[0]) { - return false; + if (opt_etm) { + err = htp_iface_etm(this->handle, 0); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: warn : failed to disable ETM tracing: 0x%08x\n", (unsigned) err); + } } - return true; - - GGML_UNUSED(sess); -} - -static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; // A - const struct ggml_tensor * src1 = op->src[1]; // B - const struct ggml_tensor * dst = op; // X - - if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; + if (opt_profile) { + htp_iface_pmu_conf pmu_conf{}; + err = htp_iface_profiler(this->handle, 0, &pmu_conf); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: warn : failed to disable profiling: 0x%08x\n", (unsigned) err); + } } - if (src0->ne[0] != src0->ne[1]) { - return false; + if (this->valid_queue) { + err = dspqueue_close(queue); + if (err != 0) { + GGML_ABORT("ggml-hex: dspqueue_close failed: 0x%08x\n", (unsigned) err); + } } - if (src0->ne[1] != src1->ne[1]) { - return false; + if (this->valid_handle) { + htp_iface_close(this->handle); } +} - if (src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3]) { - return false; - } +ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) { + this->dev = dev; + this->dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context); + this->mdev.idx = mdev_idx; + this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size()); + op_batch = nullptr; + op_queue = nullptr; + fence_buf = nullptr; + fence_seq = ((uintptr_t)this) & 0xFFFF; - if (dst->ne[0] != src1->ne[0] || dst->ne[1] != src1->ne[1] || dst->ne[2] != src1->ne[2] || dst->ne[3] != src1->ne[3]) { - return false; + try { + allocate(config); + if (this->mdev.idx == 0 && !config.mdev_group.empty()) { + for (size_t i = 0; i < config.mdev_group.size(); i++) { + this->mdev.sessions.push_back(std::make_unique<ggml_hexagon_session>( + config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count)); + } + } + } catch (const std::exception & exc) { + release(); + throw; } - - return true; - - GGML_UNUSED(sess); } -static bool ggml_hexagon_supported_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - if (src0->type != GGML_TYPE_F32) { return false; } - if (dst->type != GGML_TYPE_F32) { return false; } - if (!ggml_are_same_shape(src0, dst)) { return false; } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { return false; } +ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { + release(); +} - return true; +// ** backend interface - GGML_UNUSED(sess); -} static const char * ggml_backend_hexagon_name(ggml_backend_t backend) { auto sess = static_cast<ggml_hexagon_session *>(backend->context); @@ -7046,195 +2413,31 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) { delete backend; } -static htp_op_code op_remap_to_htp(const ggml_tensor * t) { - switch (t->op) { - case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT; - case GGML_OP_MUL_MAT: return HTP_OP_MUL_MAT; - case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID; - case GGML_OP_MUL: return HTP_OP_MUL; - case GGML_OP_ADD: return HTP_OP_ADD; - case GGML_OP_ADD_ID: return HTP_OP_ADD_ID; - case GGML_OP_SUB: return HTP_OP_SUB; - case GGML_OP_DIV: return HTP_OP_DIV; - case GGML_OP_CPY: return HTP_OP_CPY; - case GGML_OP_CONT: return HTP_OP_CPY; - case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS; - case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS; - case GGML_OP_SUM: return HTP_OP_SUM; - case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS; - case GGML_OP_ARGSORT: return HTP_OP_ARGSORT; - case GGML_OP_TOP_K: return HTP_OP_TOP_K; - case GGML_OP_ARGMAX: return HTP_OP_ARGMAX; - case GGML_OP_NORM: return HTP_OP_NORM; - case GGML_OP_L2_NORM: return HTP_OP_L2_NORM; - case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM; - case GGML_OP_CONCAT: return HTP_OP_CONCAT; - case GGML_OP_SCALE: return HTP_OP_SCALE; - case GGML_OP_CLAMP: return HTP_OP_CLAMP; - case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU; - case GGML_OP_SQR: return HTP_OP_SQR; - case GGML_OP_SQRT: return HTP_OP_SQRT; - case GGML_OP_LOG: return HTP_OP_UNARY_LOG; - case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX; - case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV; - case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET; - case GGML_OP_ROPE: return HTP_OP_ROPE; - case GGML_OP_REPEAT: return HTP_OP_REPEAT; - case GGML_OP_CUMSUM: return HTP_OP_CUMSUM; - case GGML_OP_FILL: return HTP_OP_FILL; - case GGML_OP_DIAG: return HTP_OP_DIAG; - case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI; - case GGML_OP_TRI: return HTP_OP_TRI; - case GGML_OP_PAD: return HTP_OP_PAD; - case GGML_OP_IM2COL: return HTP_OP_IM2COL; - case GGML_OP_ROLL: return HTP_OP_ROLL; - - case GGML_OP_UNARY: - switch (ggml_get_unary_op(t)) { - case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU; - case GGML_UNARY_OP_GELU: return HTP_OP_UNARY_GELU; - case GGML_UNARY_OP_GELU_QUICK: return HTP_OP_UNARY_GELU; - case GGML_UNARY_OP_GELU_ERF: return HTP_OP_UNARY_GELU_ERF; - case GGML_UNARY_OP_SIGMOID: return HTP_OP_UNARY_SIGMOID; - case GGML_UNARY_OP_NEG: return HTP_OP_UNARY_NEG; - case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP; - case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS; - case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH; - case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS; - case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU; - case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP; - default: - break; - } - break; - - case GGML_OP_GLU: - switch (ggml_get_glu_op(t)) { - case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU; - case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI; - case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP; - case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU; - case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK; - case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF; - default: break; - } - break; - - default: - GGML_ABORT("\nggml-hex: graph-compute %s is not supported\n", ggml_op_desc(t)); - } - return HTP_OP_INVALID; -} - -static inline bool op_is_compute(ggml_tensor *node) -{ +static inline bool ggml_hexagon_op_is_compute(const struct ggml_tensor * node) { return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE); } -static bool mm_is_hmx_eligible(const ggml_tensor * t) { - if (opt_nhmx == 0) { return false; } - - const ggml_tensor * src0 = t->src[0]; - const ggml_tensor * src1 = t->src[1]; - - const int wtype = src0->type; - const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype); - const bool is_matmul_id = (t->op == GGML_OP_MUL_MAT_ID); - const bool is_batched = (src0->ne[2] * src0->ne[3] > 1 || src1->ne[2] * src1->ne[3] > 1); - - const int ne01_padded = is_repack ? hex_round_up(src0->ne[1], 32) : src0->ne[1]; - - return ggml_hexagon_matmul_is_hmx_eligible(src0, src1, t, ne01_padded, is_matmul_id, is_batched); -} - -static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { - if (kparams->n_hmx) { - return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; - } - - if (!ggml_hexagon_is_repack_type(src0->type) || src0->type == GGML_TYPE_Q6_K) { - return false; // Q6_K has no fused HVX kernel - } - - return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW; -} - -static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { - if (kparams->n_hmx) { - return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; - } - - if (!ggml_hexagon_is_repack_type(src0->type)) { - return false; - } - - return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK; -} - -static bool is_mergeable_mul_mat(const ggml_tensor * t) { - if (t->op != GGML_OP_MUL_MAT) return false; - - const ggml_tensor * src0 = t->src[0]; - const ggml_tensor * src1 = t->src[1]; - if (src1->type != GGML_TYPE_F32) return false; - if (src0->ne[2] != 1 || src0->ne[3] != 1) return false; - - if (mm_is_hmx_eligible(t)) { - return ggml_hexagon_is_hmx_weight_type(src0->type); - } - - return ggml_hexagon_is_repack_type(src0->type) && src0->type != GGML_TYPE_Q6_K; -} - -static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) { - if (!is_mergeable_mul_mat(n1) || !is_mergeable_mul_mat(n2)) { - return false; - } - if (n1->src[1] != n2->src[1]) { - return false; - } - if (n1->src[0]->ne[0] != n2->src[0]->ne[0]) { - return false; - } - if (n1->src[0]->type != n2->src[0]->type) { - return false; - } - if (mm_is_hmx_eligible(n1) != mm_is_hmx_eligible(n2)) { - return false; +static void ggml_hexagon_tensor_repack(ggml_tensor * tensor, ggml_hexagon_session * sess) { + if (tensor->data == nullptr || ggml_hexagon_tensor_is_tiled(tensor) || !ggml_hexagon_tensor_can_tile(tensor)) { + return; } - return true; -} - -static bool is_mergeable_mul_mat_id(const ggml_tensor * t) { - if (t->op != GGML_OP_MUL_MAT_ID) return false; - - const ggml_tensor * src0 = t->src[0]; - return ggml_hexagon_is_repack_type(src0->type); + HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(tensor, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(tensor)); + std::vector<uint8_t> raw(ggml_nbytes(tensor)); + memcpy(raw.data(), tensor->data, raw.size()); + htp_repack_tensor_tiled(tensor, raw.data(), raw.size()); + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_TILED); } -static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tensor * n2) { - if (!is_mergeable_mul_mat_id(n1) || !is_mergeable_mul_mat_id(n2)) { - return false; - } - if (n1->src[1] != n2->src[1]) { - return false; - } - if (n1->src[2] != n2->src[2]) { - return false; - } - if (n1->src[0]->ne[0] != n2->src[0]->ne[0]) { - return false; - } - if (n1->src[0]->ne[2] != n2->src[0]->ne[2]) { - return false; - } - if (n1->src[0]->type != n2->src[0]->type) { - return false; - } - if (mm_is_hmx_eligible(n1) != mm_is_hmx_eligible(n2)) { - return false; +static void ggml_hexagon_graph_repack(const ggml_cgraph * graph, ggml_hexagon_session * sess) { + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * n = graph->nodes[i]; + if (!ggml_hexagon_op_is_compute(n)) { + continue; + } + if (n->op == GGML_OP_MUL_MAT || n->op == GGML_OP_MUL_MAT_ID) { + ggml_hexagon_tensor_repack(n->src[0], sess); + } } - return true; } static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { @@ -7249,98 +2452,30 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg const std::vector<htp_opnode> * nodes_ptr = nullptr; std::vector<htp_opnode> computed_nodes; - // Check for cache hit bool cache_hit = (graph->uid != 0 && sess->cached_uid == graph->uid); if (cache_hit) { nodes_ptr = &sess->cached_nodes; } else { - // Tag fusable tensors in graph - for (int i = 0; i < graph->n_nodes; i++) { - auto * extra = (ggml_hexagon_tensor_extra *) graph->nodes[i]->extra; - if (!extra) continue; - - extra->flags &= ~GGML_HEXAGON_TENSOR_FUSEABLE; - - if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { - extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; - } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT || graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { - if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { graph->nodes[i]->op, GGML_OP_ADD })) || - ggml_node_has_n_uses(graph, i, 1)) { - extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; - } - } - } + ggml_hexagon_graph_repack(graph, sess); + + htp_fusion_tag_graph(graph); computed_nodes.reserve(graph->n_nodes); for (int i = 0; i < graph->n_nodes; ++i) { ggml_tensor * n = graph->nodes[i]; - if (!op_is_compute(n)) { + if (!ggml_hexagon_op_is_compute(n)) { continue; } - htp_opnode node(HTP_OP_INVALID, n); - node.opcode = op_remap_to_htp(n); - if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID) { - ggml_hexagon_precompute_matmul_params(sess, - node.node->src[0], node.node->src[1], node.node, - (struct htp_mm_kernel_params *)node.kernel_params - ); - } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { - const ggml_tensor * src1 = node.node->src[1]; - GGML_ASSERT(ggml_hexagon_precompute_binary_params(sess, - node.opcode, node.node->src[0], src1, node.node, - (struct htp_binary_kernel_params *) node.kernel_params)); - } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { - ggml_hexagon_precompute_flash_attn_params(sess, - node.node, - (struct htp_fa_kernel_params *)node.kernel_params - ); - } else if (htp_op_is_unary(node.opcode)) { - auto inputs = node.get_inputs(); - const struct ggml_tensor * src0 = inputs[0]; - const struct ggml_tensor * src1 = inputs.size() > 1 ? inputs[1] : nullptr; - ggml_hexagon_precompute_unary_params(sess, - node.opcode, src0, src1, node.dst(), - (struct htp_unary_kernel_params *)node.kernel_params - ); - } else if (node.opcode == HTP_OP_GET_ROWS) { - ggml_hexagon_precompute_get_rows_params(sess, - node.node->src[0], node.node->src[1], node.dst(), - (struct htp_get_rows_kernel_params *)node.kernel_params - ); - } else if (node.opcode == HTP_OP_SET_ROWS) { - ggml_hexagon_precompute_set_rows_params(sess, - node.node->src[0], node.node->src[1], node.dst(), - (struct htp_set_rows_kernel_params *)node.kernel_params - ); - } else if (node.opcode == HTP_OP_ROPE) { - ggml_hexagon_precompute_rope_params(sess, - node.node, - (struct htp_rope_kernel_params *)node.kernel_params - ); - } else if (node.opcode == HTP_OP_SSM_CONV) { - ggml_hexagon_precompute_ssm_conv_params(sess, - node.node->src[0], node.node->src[1], node.dst(), - (struct htp_ssm_conv_kernel_params *)node.kernel_params - ); - } else if (node.opcode == HTP_OP_SOFTMAX) { - ggml_hexagon_precompute_softmax_params(sess, - node.node, - (struct htp_softmax_kernel_params *)node.kernel_params - ); - } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { - ggml_hexagon_precompute_gated_delta_net_params(sess, - node.node, - (struct htp_gdn_kernel_params *)node.kernel_params - ); - } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) { - ggml_hexagon_precompute_sort_params(sess, - node.node, - node.opcode == HTP_OP_TOP_K, - (struct htp_sort_kernel_params *) node.kernel_params - ); + const htp_op_code opcode = htp_op_remap(n); + if (opcode == HTP_OP_INVALID) { + GGML_ABORT("\nggml-hex: graph-compute %s is not supported\n", ggml_op_desc(n)); } + + htp_opnode node(opcode, n); + GGML_ASSERT(htp_kparams_solve(sess, node)); + computed_nodes.push_back(std::move(node)); } @@ -7353,7 +2488,6 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg } } - // Queue and execute for (const auto & node : *nodes_ptr) { sess->enqueue_op(node); } @@ -7437,7 +2571,22 @@ static ggml_hexagon_mem_range ggml_hexagon_mem_range_from_tensor(const ggml_tens return mr; } -static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) { +struct node_info { + ggml_tensor * node; + htp_opvec<ggml_tensor *, HTP_OP_MAX_FUSED> fused; + + const ggml_tensor * dst() const { return fused.empty() ? node : fused.back(); } + + bool is_empty() const { + return ggml_op_is_empty(node->op); + } + + void add_fused(ggml_tensor * t) { + fused.push_back(t); + } +}; + +static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const node_info & node) { if (node.is_empty()) return; for (int i = 0; i < GGML_MAX_SRC; i++) { @@ -7455,7 +2604,7 @@ static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, cons mrs.add(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST)); } -static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) { +static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const node_info & node) { if (node.is_empty()) return true; for (int i = 0; i < GGML_MAX_SRC; i++) { @@ -7477,7 +2626,7 @@ static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & m return mrs.check(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST)); } -static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<htp_opnode> & nodes) { +static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<node_info> & nodes) { const int n = nodes.size(); std::vector<int> res; @@ -7497,16 +2646,16 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<ht const auto & node0 = nodes[i0]; - if (!node0.stackable()) { + if (!htp_fusion_can_stack(node0.node)) { res.push_back(i0); used[i0] = true; continue; } // that many nodes forward to search for stackable nodes that can reuse VTCM - constexpr int N_FORWARD = 16; + constexpr int N_FORWARD = 64; - std::vector<int> stack; + htp_opvec<int, N_FORWARD> stack; stack.push_back(i0); mrs.reset(); @@ -7518,7 +2667,7 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<ht const auto & node1 = nodes[i1]; - if (node1.stackable() && node1.same_input(node0) && ggml_hexagon_mem_ranges_check_node(mrs, node1)) { + if (htp_fusion_can_stack(node0.node, node1.node) && ggml_hexagon_mem_ranges_check_node(mrs, node1)) { stack.push_back(i1); } else { ggml_hexagon_mem_ranges_add_node(mrs, node1); @@ -7535,56 +2684,24 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<ht } static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgraph * gf, ggml_backend_graph_optimize_params * params) { - GGML_UNUSED(params); + auto sess = static_cast<ggml_hexagon_session *>(backend->context); - const int n = gf->n_nodes; + HEX_VERBOSE("ggml-hex: %s graph-optimize n_nodes %d\n", sess->c_name(), gf->n_nodes); - constexpr int MAX_FUSE = 16; + ggml_hexagon_graph_repack(gf, sess); - enum ggml_op ops[MAX_FUSE]; + const int n = gf->n_nodes; - std::vector<htp_opnode> nodes; + std::vector<node_info> nodes; nodes.reserve(gf->n_nodes); // Pack nodes for reordering for (int i = 0; i < n; i++) { - htp_opnode node(HTP_OP_INVALID, gf->nodes[i]); - - // fuse only ops that start with these operations - // can be expanded when needed - if (node.op() == GGML_OP_ADD || - node.op() == GGML_OP_NORM || - node.op() == GGML_OP_RMS_NORM) { - ops[0] = node.op(); - - int f = i + 1; - while (f < n && f < i + MAX_FUSE) { - // conservatively allow fusing only these ops - // can be expanded when needed - if (gf->nodes[f]->op != GGML_OP_ADD && - gf->nodes[f]->op != GGML_OP_MUL && - gf->nodes[f]->op != GGML_OP_NORM && - gf->nodes[f]->op != GGML_OP_RMS_NORM) { - break; - } - ops[f - i] = gf->nodes[f]->op; - f++; - } - - f -= i; - for (; f > 1; f--) { - if (ggml_can_fuse(gf, i, ops, f)) { - break; - } - } - - // add the fused tensors into the node info so we can unfuse them later - for (int k = 1; k < f; k++) { - ++i; + node_info node = { gf->nodes[i], {} }; - // the .dst() becomes the last fused tensor - node.add_fused(gf->nodes[i]); - } + const int f = htp_fusion_max(gf, i); + for (int k = 1; k < f; k++) { + node.add_fused(gf->nodes[++i]); } nodes.push_back(std::move(node)); @@ -7606,7 +2723,7 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr } } - GGML_UNUSED(backend); + GGML_UNUSED(params); } static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) { @@ -7630,29 +2747,12 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_ if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; } - if (++sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; - uint32_t fence_seq = sess_dst->fence_seq; + uint32_t fence_seq = sess_dst->next_fence_seq(); HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq); - // dummy fence extra (must be static) - static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; - - ggml_tensor fence_tensor {}; - fence_tensor.buffer = &sess_dst->fence_buf->backend_buffer; - fence_tensor.extra = &fence_extra; - fence_tensor.data = (void *) fence_slot; - fence_tensor.type = GGML_TYPE_I32; - fence_tensor.ne[0] = 1; - fence_tensor.ne[1] = 1; - fence_tensor.ne[2] = 1; - fence_tensor.ne[3] = 1; - fence_tensor.nb[0] = sizeof(int32_t); - fence_tensor.nb[1] = sizeof(int32_t); - fence_tensor.nb[2] = sizeof(int32_t); - fence_tensor.nb[3] = sizeof(int32_t); - fence_tensor.op = GGML_OP_NONE; + ggml_tensor fence_tensor = sess_dst->make_fence_tensor(fence_slot); sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq); sess_dst->enqueue_fence(&fence_tensor, fence_seq, /* wait = */ true); @@ -7685,14 +2785,7 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg return false; } - // FIXME: ggml-meta needs to call init_tensor on auxiliary tensors - if (!dst->extra) { - ggml_backend_buffer_init_tensor(dst->buffer, dst); - } - - auto * dst_extra = static_cast<ggml_hexagon_tensor_extra *>(dst->extra); - const auto * src_extra = static_cast<const ggml_hexagon_tensor_extra *>(src->extra); - dst_extra->flags = src_extra->flags & ~GGML_HEXAGON_TENSOR_FUSEABLE; + dst->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(src) & ~HTP_TENSOR_FUSEABLE); auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context); auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context); @@ -7714,24 +2807,10 @@ static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_d auto sess = dev_ctx->session(); ggml_hexagon_event * hex_event = new ggml_hexagon_event(); - hex_event->fence_sess = sess; - hex_event->sess = sess; - hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1); - - static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; - hex_event->fence_tensor.buffer = &sess->fence_buf->backend_buffer; - hex_event->fence_tensor.extra = &fence_extra; - hex_event->fence_tensor.data = (void *) hex_event->fence_slot; - hex_event->fence_tensor.type = GGML_TYPE_I32; - hex_event->fence_tensor.ne[0] = 1; - hex_event->fence_tensor.ne[1] = 1; - hex_event->fence_tensor.ne[2] = 1; - hex_event->fence_tensor.ne[3] = 1; - hex_event->fence_tensor.nb[0] = sizeof(int32_t); - hex_event->fence_tensor.nb[1] = sizeof(int32_t); - hex_event->fence_tensor.nb[2] = sizeof(int32_t); - hex_event->fence_tensor.nb[3] = sizeof(int32_t); - hex_event->fence_tensor.op = GGML_OP_NONE; + hex_event->fence_sess = sess; + hex_event->sess = sess; + hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1); + hex_event->fence_tensor = sess->make_fence_tensor(hex_event->fence_slot); HEX_VERBOSE("ggml-hex: %s event-new : event %p fence %p\n", ggml_backend_dev_name(dev), (void *)hex_event, (void *)hex_event->fence_slot); @@ -7786,9 +2865,8 @@ static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backe auto sess = static_cast<ggml_hexagon_session *>(backend->context); auto hex_event = static_cast<ggml_hexagon_event *>(event->context); - if (++sess->fence_seq == 0) sess->fence_seq = 1; hex_event->sess = sess; - hex_event->seq = sess->fence_seq; + hex_event->seq = sess->next_fence_seq(); sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ false); @@ -7928,8 +3006,8 @@ static const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev } static void ggml_backend_hexagon_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - *free = 0; - *total = *free; + *free = opt_mbuf_total; + *total = opt_mbuf_total; GGML_UNUSED(dev); } @@ -7967,131 +3045,9 @@ static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_ty return &dev_ctx->host_buffer_type; } -static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - GGML_UNUSED(sess); - - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && - src0->type != GGML_TYPE_I32) return false; - if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && - dst->type != GGML_TYPE_I32) return false; - - const bool is_scalar = (ggml_nelements(src0) == 1 && ggml_nelements(dst) == 1); - const bool sametype = (src0->type == dst->type); - const bool transposed = !is_scalar && (ggml_is_transposed(src0) || ggml_is_transposed(dst)); - const bool sameshape = is_scalar || (!transposed && ggml_are_same_shape(src0, dst)); - - if (src0->type == GGML_TYPE_I32 || dst->type == GGML_TYPE_I32) { - if (!sameshape) return false; - if (sametype) return true; - if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) || - (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32)) { - return true; - } - return false; - } - - // can handle any shape and any same-type (pretty slow if reshaping is required) - if (sametype) return true; - - // cannot handle re-shaping and type conversion at the same time - if (!sameshape) return false; - - return true; -} - -static bool ggml_hexagon_supported_cont(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - GGML_UNUSED(sess); - const struct ggml_tensor * src0 = op->src[0]; - - // CONT is same-type only and supports F32, F16, and I32. - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && - src0->type != GGML_TYPE_I32) return false; - - return true; -} - -static bool ggml_hexagon_supported_repeat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - GGML_UNUSED(sess); - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - // Support f32 and f16 - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) return false; - - // src and dst must be the same type - if (src0->type != dst->type) return false; - - // dst dims must be multiples of src dims - if (dst->ne[0] % src0->ne[0] != 0) return false; - if (dst->ne[1] % src0->ne[1] != 0) return false; - if (dst->ne[2] % src0->ne[2] != 0) return false; - if (dst->ne[3] % src0->ne[3] != 0) return false; - - // require contiguous tensors (no transposition) - if (ggml_is_transposed(src0) || ggml_is_transposed(dst)) return false; - - return true; -} - -static bool ggml_hexagon_supported_concat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - int dim = ((const int32_t *) op->op_params)[0]; - if (dim < 0 || dim >= GGML_MAX_DIMS) { - return false; - } - - for (int i = 0; i < GGML_MAX_SRC; ++i) { - const struct ggml_tensor * src = op->src[i]; - if (!src) { - continue; - } - if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_I32 && src->type != GGML_TYPE_F16) { - return false; - } - } - - return true; - GGML_UNUSED(sess); -} - -static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * dst = op; - - if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { - return false; - } - - return true; - GGML_UNUSED(sess); -} - -static bool ggml_hexagon_supported_roll(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - GGML_UNUSED(sess); - - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; - } - - if (!ggml_are_same_shape(src0, dst)) { - return false; - } - - if (src0->nb[0] != ggml_type_size(src0->type) || dst->nb[0] != ggml_type_size(dst->type)) { - return false; - } - - if (!ggml_is_contiguous(dst)) { - return false; - } - - return true; -} - +// IMPORTANT: supports_op runs before buffer allocation and before tensor data is populated. +// Never check buffer usage, buffer properties, or repacking state (is_tiled) here. +// Support must depend only on static tensor properties (shape, type, strides). static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { auto dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context); auto sess = dev_ctx->session(); @@ -8102,188 +3058,21 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons } bool supp = false; - switch (op->op) { - case GGML_OP_NONE: - case GGML_OP_RESHAPE: - case GGML_OP_VIEW: - case GGML_OP_PERMUTE: - case GGML_OP_TRANSPOSE: - supp = true; - break; - - case GGML_OP_MUL: - case GGML_OP_ADD: - case GGML_OP_SUB: - case GGML_OP_DIV: - supp = ggml_hexagon_supported_binary(sess, op); - break; - - case GGML_OP_MUL_MAT: - supp = ggml_hexagon_supported_mul_mat(sess, op); - break; - - case GGML_OP_MUL_MAT_ID: - supp = ggml_hexagon_supported_mul_mat_id(sess, op); - break; - - case GGML_OP_ADD_ID: - supp = ggml_hexagon_supported_add_id(sess, op); - break; - - case GGML_OP_NORM: - case GGML_OP_L2_NORM: - case GGML_OP_RMS_NORM: - case GGML_OP_SCALE: - case GGML_OP_CLAMP: - case GGML_OP_LEAKY_RELU: - supp = ggml_hexagon_supported_unary(sess, op); - break; - - case GGML_OP_SQR: - case GGML_OP_SQRT: - case GGML_OP_LOG: - supp = ggml_hexagon_supported_unary(sess, op); - break; - - case GGML_OP_SUM: - supp = ggml_hexagon_supported_sum(sess, op); - break; - - case GGML_OP_SUM_ROWS: - supp = ggml_hexagon_supported_sum_rows(sess, op); - break; - - case GGML_OP_ARGMAX: - supp = ggml_hexagon_supported_argmax(sess, op); - break; - - case GGML_OP_SOFT_MAX: - supp = ggml_hexagon_supported_softmax(sess, op); - break; - - case GGML_OP_UNARY: - switch (ggml_get_unary_op(op)) { - case GGML_UNARY_OP_NEG: - case GGML_UNARY_OP_EXP: - case GGML_UNARY_OP_SIGMOID: - case GGML_UNARY_OP_SOFTPLUS: - case GGML_UNARY_OP_TANH: - case GGML_UNARY_OP_ABS: - case GGML_UNARY_OP_SILU: - case GGML_UNARY_OP_GELU: - case GGML_UNARY_OP_GELU_QUICK: - case GGML_UNARY_OP_GELU_ERF: - case GGML_UNARY_OP_RELU: - case GGML_UNARY_OP_STEP: - supp = ggml_hexagon_supported_unary(sess, op); - break; - default: - supp = false; - break; - } - break; - - case GGML_OP_GLU: - switch (ggml_get_glu_op(op)) { - case GGML_GLU_OP_SWIGLU: - case GGML_GLU_OP_SWIGLU_OAI: - case GGML_GLU_OP_SWIGLU_CLAMP: - case GGML_GLU_OP_GEGLU: - case GGML_GLU_OP_GEGLU_QUICK: - case GGML_GLU_OP_GEGLU_ERF: - supp = ggml_hexagon_supported_activations(sess, op); - break; - default: - supp = false; - break; + if (ggml_op_is_empty(op->op) || ggml_is_empty(op)) { + supp = true; + } else { + supp = ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op) <= sess->max_vmem; + for (int i = 0; supp && i < GGML_MAX_SRC; i++) { + if (op->src[i] && ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op->src[i]) > sess->max_vmem) { + supp = false; } - break; - - case GGML_OP_ROPE: - supp = ggml_hexagon_supported_rope(sess, op); - break; - - case GGML_OP_FLASH_ATTN_EXT: - supp = ggml_hexagon_supported_flash_attn_ext(sess, op); - break; - - case GGML_OP_SET_ROWS: - supp = ggml_hexagon_supported_set_rows(sess, op); - break; - - case GGML_OP_GET_ROWS: - supp = ggml_hexagon_supported_get_rows(sess, op); - break; - - case GGML_OP_CPY: - supp = ggml_hexagon_supported_cpy(sess, op); - break; - - case GGML_OP_CONT: - supp = ggml_hexagon_supported_cont(sess, op); - break; - - case GGML_OP_REPEAT: - supp = ggml_hexagon_supported_repeat(sess, op); - break; - - case GGML_OP_ARGSORT: - supp = ggml_hexagon_supported_argsort(sess, op); - break; - - case GGML_OP_TOP_K: - supp = ggml_hexagon_supported_top_k(sess, op); - break; - - case GGML_OP_SSM_CONV: - supp = ggml_hexagon_supported_ssm_conv(sess, op); - break; - - case GGML_OP_IM2COL: - supp = ggml_hexagon_supported_im2col(sess, op); - break; - - case GGML_OP_GATED_DELTA_NET: - supp = ggml_hexagon_supported_gated_delta_net(sess, op); - break; - - case GGML_OP_CUMSUM: - supp = ggml_hexagon_supported_cumsum(sess, op); - break; - - case GGML_OP_CONCAT: - supp = ggml_hexagon_supported_concat(sess, op); - break; - - case GGML_OP_FILL: - supp = ggml_hexagon_supported_fill(sess, op); - break; - - case GGML_OP_DIAG: - supp = ggml_hexagon_supported_diag(sess, op); - break; - - case GGML_OP_SOLVE_TRI: - supp = ggml_hexagon_supported_solve_tri(sess, op); - break; - - case GGML_OP_TRI: - supp = ggml_hexagon_supported_tri(sess, op); - break; - - case GGML_OP_PAD: - supp = ggml_hexagon_supported_pad(sess, op); - break; - - case GGML_OP_ROLL: - supp = ggml_hexagon_supported_roll(sess, op); - break; - - default: - break; + } + if (supp) { + supp = htp_kparams_check(sess, op); + } } - ggml_hexagon_dump_op_supp(sess->name, op, supp); + ggml_hexagon_dump_op_supp(sess->c_name(), op, supp); return supp; } @@ -8400,24 +3189,10 @@ static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t n ctx->backends.assign(backends, backends + n_backends); ctx->n_backends = n_backends; - static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; for (size_t i = 0; i < n_backends; i++) { auto sess_i = static_cast<ggml_hexagon_session *>(backends[i]->context); - ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1); - ctx->fence_tensors[i] = {}; - ctx->fence_tensors[i].buffer = &sess_i->fence_buf->backend_buffer; - ctx->fence_tensors[i].extra = &fence_extra; - ctx->fence_tensors[i].data = (void *) ctx->fence_slots[i]; - ctx->fence_tensors[i].type = GGML_TYPE_I32; - ctx->fence_tensors[i].ne[0] = 4; - ctx->fence_tensors[i].ne[1] = 1; - ctx->fence_tensors[i].ne[2] = 1; - ctx->fence_tensors[i].ne[3] = 1; - ctx->fence_tensors[i].nb[0] = sizeof(int32_t); - ctx->fence_tensors[i].nb[1] = sizeof(int32_t); - ctx->fence_tensors[i].nb[2] = sizeof(int32_t); - ctx->fence_tensors[i].nb[3] = sizeof(int32_t); - ctx->fence_tensors[i].op = GGML_OP_NONE; + ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1); + ctx->fence_tensors[i] = sess_i->make_fence_tensor(ctx->fence_slots[i]); } return ctx; @@ -8471,8 +3246,7 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct for (size_t r = 0; r < n_backends; r++) { auto sess = static_cast<ggml_hexagon_session *>(comm_ctx->backends[r]->context); - struct htp_allreduce_kernel_params kparams; - if (!ggml_hexagon_precompute_allreduce_params(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, /*is_shard_ok=*/ false, &kparams)) { + if (!htp_kparams_check<htp_kparams_solve_allreduce>(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, false)) { return false; } } @@ -8621,8 +3395,8 @@ static void ggml_hexagon_discover_devices() { discovery_supported = true; const int n_domains = std::min(domain_info.sys.num_domains, (int) domains.size()); for (int i = 0; i < n_domains; i++) { - GGML_LOG_INFO("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type %d id %d name '%s' status %d instance-id %d\n", - i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id); + HEX_VERBOSE("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type %d id %d name '%s' status %d instance-id %d\n", + i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id); if (domains[i].type != FASTRPC_NSP) { GGML_LOG_DEBUG("ggml-hex: skipping non-CDSP domain (type=%d)\n", (int) domains[i].type); continue; @@ -8636,10 +3410,10 @@ static void ggml_hexagon_discover_devices() { domains[i].instance_id, domains[i].id, domains[i].name); } } else { - GGML_LOG_WARN("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err); + HEX_VERBOSE("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err); } } else if (err != AEE_SUCCESS) { - GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err); + HEX_VERBOSE("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err); } // Populate domain IDs and names for all configured devices @@ -8654,6 +3428,10 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_Q4_1 == (unsigned int) GGML_TYPE_Q4_1, "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q5_0 == (unsigned int) GGML_TYPE_Q5_0, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q5_1 == (unsigned int) GGML_TYPE_Q5_1, + "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_Q8_0 == (unsigned int) GGML_TYPE_Q8_0, "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_MXFP4 == (unsigned int) GGML_TYPE_MXFP4, @@ -8670,19 +3448,29 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_Q2_K == (unsigned int) GGML_TYPE_Q2_K, "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q1_0 == (unsigned int) GGML_TYPE_Q1_0, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q2_0 == (unsigned int) GGML_TYPE_Q2_0, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_BF16 == (unsigned int) GGML_TYPE_BF16, + "please update hexagon_type to match ggml_type"); const char * str_verbose = getenv("GGML_HEXAGON_VERBOSE"); const char * str_opbatch = getenv("GGML_HEXAGON_OPBATCH"); const char * str_opqueue = getenv("GGML_HEXAGON_OPQUEUE"); const char * str_oppoll = getenv("GGML_HEXAGON_OPPOLL"); const char * str_opfusion = getenv("GGML_HEXAGON_OPFUSION"); + const char * str_rt_repack = getenv("GGML_HEXAGON_RT_REPACK"); const char * str_opfilter = getenv("GGML_HEXAGON_OPFILTER"); const char * str_profile = getenv("GGML_HEXAGON_PROFILE"); const char * str_etm = getenv("GGML_HEXAGON_ETM"); const char * str_nhvx = getenv("GGML_HEXAGON_NHVX"); const char * str_nhmx = getenv("GGML_HEXAGON_NHMX"); + const char * str_vtcm_size = getenv("GGML_HEXAGON_VTCM_SIZE"); const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT"); const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT"); + const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT"); + const char * str_li_select = getenv("GGML_HEXAGON_LI_SELECT"); const char * str_gdn_select = getenv("GGML_HEXAGON_GDN_SELECT"); const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT"); const char * str_ar_scatter = getenv("GGML_HEXAGON_AR_SCATTER"); @@ -8693,6 +3481,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { const char * str_optrace = getenv("GGML_HEXAGON_OPTRACE"); const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF"); const char * str_dma64 = getenv("GGML_HEXAGON_DMA64"); + const char * str_power_mode = getenv("GGML_HEXAGON_POWER_MODE"); // Init Arch first since it affects other defaults if (!str_arch) { @@ -8704,9 +3493,9 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { if (opt_arch < 73) { GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is under supported range, capping at v73\n", opt_arch); opt_arch = 73; - } else if (opt_arch > 81) { - GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v81\n", opt_arch); - opt_arch = 81; + } else if (opt_arch > 85) { + GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v85\n", opt_arch); + opt_arch = 85; } } } else { @@ -8719,7 +3508,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { size_t MiB = 1024 * 1024; // Update vmem default - opt_vmem = opt_arch >= 75 ? HTP_OP_MAX_VMEM_DEFAULT : 3000 * MiB; + opt_vmem = opt_arch >= 75 ? HTP_OP_MAX_VMEM_DEFAULT : 3000 * MiB; opt_dma64 = opt_arch > 79 && (!str_dma64 || atoi(str_dma64) != 0); auto RE_ICASE = std::regex_constants::icase; @@ -8728,21 +3517,50 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_verbose = str_verbose ? atoi(str_verbose) : 0; opt_opbatch = str_opbatch ? strtoul(str_opbatch, NULL, 0) : opt_opbatch; opt_opqueue = str_opqueue ? strtoul(str_opqueue, NULL, 0) : opt_opqueue; - opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * 256); + opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * HTP_TRACE_MAX_EVENTS + 131072); opt_oppoll = str_oppoll ? strtoul(str_oppoll, NULL, 0) : opt_oppoll; opt_opfusion = str_opfusion ? atoi(str_opfusion) : opt_opfusion; + opt_rt_repack = str_rt_repack ? atoi(str_rt_repack) : opt_rt_repack; opt_profile = str_profile ? atoi(str_profile) : 0; opt_etm = str_etm ? atoi(str_etm) : 0; + if (str_power_mode) { + int m = htp_power_mode_from_str(str_power_mode); + if (m < 0) { + GGML_LOG_WARN("ggml-hex: unknown GGML_HEXAGON_POWER_MODE '%s', falling back to 'burst'\n", str_power_mode); + } else { + opt_power_mode = m; + } + } opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx; opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx; + opt_vtcm_size = str_vtcm_size ? parse_size(str_vtcm_size) : opt_vtcm_size; opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select; opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select; - opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select; - opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select; - opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter; - opt_mbuf = str_mbuf ? strtoul(str_mbuf, NULL, 0) * MiB : opt_mbuf; - opt_vmem = str_vmem ? strtoul(str_vmem, NULL, 0) * MiB : opt_vmem; - opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) != 0 : opt_hostbuf; + opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split; + opt_li_select = str_li_select ? atoi(str_li_select) : opt_li_select; + opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select; + opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select; + opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter; + + if (str_mbuf) { + const char * p = str_mbuf; + for (int idx = 0; idx < 3 && p && *p; idx++) { + while (*p == ' ') p++; + const char * comma = strchr(p, ','); + size_t len = comma ? (size_t)(comma - p) : strlen(p); + while (len > 0 && p[len - 1] == ' ') len--; + if (len > 0) { + std::string token(p, len); + if (idx == 0) opt_mbuf_dyn = parse_size(token.c_str()); + if (idx == 1) opt_mbuf_static = parse_size(token.c_str()); + if (idx == 2) opt_mbuf_total = parse_size(token.c_str()); + } + if (!comma) break; + p = comma + 1; + } + } + opt_vmem = str_vmem ? parse_size(str_vmem) : opt_vmem; + opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) != 0 : opt_hostbuf; // Parse device configuration const char * str_devices = getenv("GGML_HEXAGON_DEVICES"); diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.h new file mode 100644 index 000000000..4e9e54415 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-common.h @@ -0,0 +1,106 @@ +#ifndef HTP_COMMON_H +#define HTP_COMMON_H + +#include "htp-opnode.h" +#include "htp-repack.h" + +extern int opt_arch; +extern size_t opt_ndev; +extern size_t opt_nhvx; +extern int opt_nhmx; +extern size_t opt_vmem; +extern int opt_etm; +extern int opt_verbose; +extern int opt_profile; +extern bool opt_hostbuf; +extern bool opt_dma64; +extern int opt_power_mode; + +extern size_t opt_mbuf_dyn; +extern size_t opt_mbuf_static; +extern size_t opt_mbuf_total; + +extern int opt_mm_select; +extern int opt_fa_select; +extern int opt_fa_split; +extern int opt_li_select; +extern int opt_gdn_select; +extern int opt_ar_select; +extern int opt_ar_scatter; + +extern int opt_opbatch; +extern int opt_opqueue; +extern int opt_optrace; +extern int opt_oppoll; +extern int opt_opfusion; +extern int opt_rt_repack; + +#ifndef HEX_VERBOSE +#define HEX_VERBOSE(...) \ + if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__) +#endif + +static inline uint32_t ggml_hexagon_tensor_get_flags(const struct ggml_tensor * t) { + return (uint32_t) (uintptr_t) t->extra; +} + +static inline void ggml_hexagon_tensor_set_flags(struct ggml_tensor * t, uint32_t flags) { + t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) | flags); +} + +static inline void ggml_hexagon_tensor_clear_flags(struct ggml_tensor * t, uint32_t flags) { + t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) & ~flags); +} + +static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) { + return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_FUSEABLE) != 0; +} + +static inline bool ggml_hexagon_tensor_is_tiled(const struct ggml_tensor * t) { + return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_TILED) != 0; +} + +static inline bool ggml_hexagon_tensor_can_tile(const struct ggml_tensor * t) { + if (opt_rt_repack == 0) { + return false; + } + if (t->buffer) { + if (ggml_backend_buft_is_host(t->buffer->buft) || + ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + return false; + } + } + return htp_is_tiled_weight_tensor(t); +} + +static inline bool ggml_hexagon_tensor_will_tile(const struct ggml_tensor * t) { + return ggml_hexagon_tensor_is_tiled(t) || ggml_hexagon_tensor_can_tile(t); +} + +static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) { + const uintptr_t a0 = (uintptr_t) a->data; + const uintptr_t b0 = (uintptr_t) b->data; + const uintptr_t a1 = a0 + ggml_nbytes(a); + const uintptr_t b1 = b0 + ggml_nbytes(b); + + return a0 < b1 && b0 < a1; +} + +static inline bool htp_can_row_partition(const struct ggml_tensor * t) { + if (t->ne[1] > 1 && (t->nb[1] & 127) != 0) return false; + if (t->ne[2] > 1 && (t->nb[2] & 127) != 0) return false; + if (t->ne[3] > 1 && (t->nb[3] & 127) != 0) return false; + return true; +} + +struct ggml_hexagon_session; + +const char * htp_session_name(const struct ggml_hexagon_session * sess); +size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess); +uint32_t htp_session_n_hvx(const struct ggml_hexagon_session * sess); +uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess); +uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess); + +htp_op_code htp_op_remap(const struct ggml_tensor * t); + +#endif // HTP_COMMON_H diff --git a/ggml/src/ggml-hexagon/htp-drv.cpp b/ggml/src/ggml-hexagon/htp-drv.cpp index 437e367c9..9ae4ea727 100644 --- a/ggml/src/ggml-hexagon/htp-drv.cpp +++ b/ggml/src/ggml-hexagon/htp-drv.cpp @@ -1,7 +1,9 @@ +#include <algorithm> #include <filesystem> #include <set> #include <sstream> #include <string> +#include <vector> #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -28,6 +30,13 @@ #include <domain.h> +extern int opt_verbose; + +#ifndef HEX_VERBOSE +#define HEX_VERBOSE(...) \ + if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__) +#endif + // // Driver API types // @@ -69,7 +78,7 @@ typedef int (*fastrpc_munmap_pfn_t)(int domain, int fd, void *addr, size_t lengt typedef int (*remote_handle64_open_pfn_t)(const char* name, remote_handle64 *ph); typedef int (*remote_handle64_invoke_pfn_t)(remote_handle64 h, uint32_t dwScalars, remote_arg *pra); -typedef int (*remote_handle64_close_pfn_t)(remote_handle h); +typedef int (*remote_handle64_close_pfn_t)(remote_handle64 h); typedef int (*remote_handle_control_pfn_t)(uint32_t req, void* data, uint32_t datalen); typedef int (*remote_handle64_control_pfn_t)(remote_handle64 h, uint32_t req, void* data, uint32_t datalen); typedef int (*remote_session_control_pfn_t)(uint32_t req, void *data, uint32_t datalen); @@ -423,3 +432,188 @@ int htpdrv_get_arch(int domain, int * arch) { *arch = (int) ((val >> 4) * 10 + (val & 0x0f)); return 0; } + +// Directory of the HTP library as a path with a trailing slash, or empty if it cannot be resolved. +// Checks GGML_HEXAGON_LIBHTP_DIR first, then the module directory and ../lib. +// On Windows, non-ASCII directories or paths with spaces use the 8.3 short name. +std::string htp_lib_dir() { + namespace fs = std::filesystem; + + std::vector<fs::path> candidates; + + const char * env_dir = getenv("GGML_HEXAGON_LIBHTP_DIR"); + if (env_dir && *env_dir) { + std::error_code ec; + fs::path p = fs::absolute(env_dir, ec); + if (!ec) { + candidates.push_back(p); + } + } + +#ifdef _WIN32 + HMODULE hm = nullptr; + if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR) &htp_lib_dir, &hm)) { + std::wstring buf(32768, L'\0'); + DWORD n = GetModuleFileNameW(hm, buf.data(), (DWORD) buf.size()); + if (n > 0 && n < buf.size()) { + buf.resize(n); + std::error_code ec; + fs::path mod_path = fs::canonical(buf, ec); + if (ec) { + mod_path = fs::absolute(buf, ec); + } + const fs::path mod_dir = mod_path.parent_path(); + candidates.push_back(mod_dir); + candidates.push_back(mod_dir / "lib"); + candidates.push_back(mod_dir / ".." / "lib"); + } + } +#elif defined(__linux__) || defined(__ANDROID__) + Dl_info info; + if (dladdr((void *) &htp_lib_dir, &info) && info.dli_fname) { + std::error_code ec; + fs::path mod_path = fs::canonical(info.dli_fname, ec); + if (ec) { + mod_path = fs::absolute(info.dli_fname, ec); + } + const fs::path mod_dir = mod_path.parent_path(); + candidates.push_back(mod_dir); + candidates.push_back(mod_dir / "lib"); + candidates.push_back(mod_dir / ".." / "lib"); + } +#endif + + for (size_t i = 0; i < candidates.size(); ++i) { + const fs::path & d = candidates[i]; + std::error_code ec; + if (!fs::is_directory(d, ec)) { + continue; + } + + // For auto-discovered directories, verify it contains an HTP library + const bool is_env = (env_dir && i == 0); + if (!is_env) { + bool found = false; + for (const auto & entry : fs::directory_iterator(d, ec)) { + if (ec) { + break; + } + const std::string fn = entry.path().filename().string(); + if (fn.rfind("libggml-htp-", 0) == 0) { + found = true; + break; + } + } + if (!found) { + continue; + } + } + + std::error_code ec_canon; + fs::path abs_d = fs::canonical(d, ec_canon); + if (ec_canon) { + abs_d = fs::absolute(d, ec_canon); + } + +#ifdef _WIN32 + std::wstring dir = abs_d.wstring(); + auto is_ascii = [](const std::wstring & s) { + return std::all_of(s.begin(), s.end(), [](wchar_t c) { return c < 0x80; }); + }; + auto has_spaces = [](const std::wstring & s) { + return std::any_of(s.begin(), s.end(), [](wchar_t c) { return c == L' '; }); + }; + + if (!is_ascii(dir) || has_spaces(dir)) { + DWORD m = GetShortPathNameW(dir.c_str(), nullptr, 0); + if (m > 0) { + std::wstring sp(m, L'\0'); + DWORD res = GetShortPathNameW(dir.c_str(), sp.data(), (DWORD) sp.size()); + if (res > 0 && res < sp.size()) { + sp.resize(res); + if (is_ascii(sp)) { + dir = sp; + } + } + } + if (!is_ascii(dir)) { + continue; + } + } + if (dir.back() != L'\\' && dir.back() != L'/') { + dir += L'\\'; + } + return std::string(dir.begin(), dir.end()); +#else + std::string dir = abs_d.string(); + if (dir.back() != '/') { + dir += '/'; + } + return dir; +#endif + } + + return {}; +} + +int htpdrv_set_dsp_search_path(int domain_id, const std::string & path, const char * dev_name) { + if (path.empty()) { + return AEE_SUCCESS; + } + + std::string clean_path = path; + while (clean_path.size() > 1 && (clean_path.back() == '/' || clean_path.back() == '\\')) { + clean_path.pop_back(); + } + + fastrpc_manage_dsp_lib_path p {}; + p.effec_domain_id = domain_id; + p.type = FASTRPC_SET_PATH; + p.path = const_cast<char *>(clean_path.c_str()); + p.pathlen = clean_path.size() + 1; + + int err = remote_session_control(FASTRPC_MANAGE_DSP_LIBRARY_PATH, (void *) &p, sizeof(p)); + if (err != AEE_SUCCESS && domain_id != -1) { + p.effec_domain_id = -1; + err = remote_session_control(FASTRPC_MANAGE_DSP_LIBRARY_PATH, (void *) &p, sizeof(p)); + } + + const char * name = dev_name ? dev_name : "driver"; + if (err == AEE_SUCCESS) { + GGML_LOG_INFO("ggml-hex: %s set DSP search path (domain %d) to %s\n", name, p.effec_domain_id, clean_path.c_str()); + return AEE_SUCCESS; + } + + HEX_VERBOSE("ggml-hex: %s FASTRPC_MANAGE_DSP_LIBRARY_PATH failed (0x%x)\n", name, err); + return err; +} + +void htpdrv_set_dsp_env_path(const std::string & path, const char * dev_name) { + if (path.empty()) { + return; + } + + std::string clean_path = path; + while (clean_path.size() > 1 && (clean_path.back() == '/' || clean_path.back() == '\\')) { + clean_path.pop_back(); + } + + const char * name = dev_name ? dev_name : "driver"; + HEX_VERBOSE("ggml-hex: %s setting DSP search path via process env to %s\n", name, clean_path.c_str()); + + static const char * const env_names[] = { + "ADSP_LIBRARY_PATH", + "CDSP_LIBRARY_PATH", + "CDSP1_LIBRARY_PATH", + "CDSP2_LIBRARY_PATH", + "CDSP3_LIBRARY_PATH", + }; + for (const char * env_name : env_names) { +#ifdef _WIN32 + SetEnvironmentVariableA(env_name, clean_path.c_str()); +#else + setenv(env_name, clean_path.c_str(), 1); +#endif + } +} diff --git a/ggml/src/ggml-hexagon/htp-drv.h b/ggml/src/ggml-hexagon/htp-drv.h index 8232780e7..c88a72940 100644 --- a/ggml/src/ggml-hexagon/htp-drv.h +++ b/ggml/src/ggml-hexagon/htp-drv.h @@ -120,4 +120,10 @@ HTPDRV_API int remote_system_request(system_req_payload * req); #ifdef __cplusplus } + +#include <string> + +std::string htp_lib_dir(); +int htpdrv_set_dsp_search_path(int domain_id, const std::string & path, const char * dev_name = nullptr); +void htpdrv_set_dsp_env_path(const std::string & path, const char * dev_name = nullptr); #endif diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp new file mode 100644 index 000000000..eb0862669 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-fusion.cpp @@ -0,0 +1,672 @@ +#include "htp-fusion.h" +#include "htp-kparams.h" +#include "ggml-impl.h" + +#include <algorithm> +#include <cstring> + +static inline bool htp_same_tensor_data(const ggml_tensor * a, const ggml_tensor * b) { + return a && b && (a == b || a->data == b->data); +} + +static inline bool htp_same_shape_and_contiguity(const ggml_tensor * a, const ggml_tensor * b) { + return ggml_are_same_shape(a, b) && (ggml_is_contiguous(a) == ggml_is_contiguous(b)); +} + +static inline bool htp_inplace_overlap_unsafe(const ggml_tensor * dst, const ggml_tensor * in) { + return htp_tensors_overlap(dst, in) && (dst->data != in->data || !ggml_are_same_stride(dst, in)); +} + +static inline bool htp_inputs_overlap_unsafe(const htp_opnode & prev, const ggml_tensor * dst, const ggml_tensor * extra = nullptr) { + if (extra && htp_inplace_overlap_unsafe(dst, extra)) return true; + for (const auto * in : prev.inputs) { + if (htp_inplace_overlap_unsafe(dst, in)) return true; + } + return false; +} + +static inline bool htp_match_consumer( + const htp_opnode & prev, + const htp_opnode & cur, + bool commutative, + const ggml_tensor ** other_src = nullptr +) { + if (prev.outputs.size() != 1) { + return false; + } + const ggml_tensor * prev_out = prev.dst(); + if (!ggml_hexagon_tensor_is_fuseable(prev_out)) { + return false; + } + if (htp_same_tensor_data(cur.src0(), prev_out)) { + if (other_src) *other_src = cur.src1(); + return true; + } + if (commutative && htp_same_tensor_data(cur.src1(), prev_out)) { + if (other_src) *other_src = cur.src0(); + return true; + } + return false; +} + +static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { + if (kparams->n_hmx) { + return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; + } + return ggml_hexagon_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW); +} + +static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { + if (kparams->n_hmx) { + return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; + } + return ggml_hexagon_tensor_is_tiled(src0) && + (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK); +} + +// --- Fusion rules --- + +static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (opt_ar_select != 2 || prev.opcode != HTP_OP_ALLREDUCE) return false; + + const auto * ar_kparams = prev.kparams<htp_allreduce_kernel_params>(); + const uint32_t rank = (uint32_t) ar_kparams->rank; + const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; + + const ggml_tensor * ar_local = prev.inputs[rank]; + if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false; + + const ggml_tensor * add_dst = cur.dst(); + const ggml_tensor * res_tensor = nullptr; + if (htp_same_tensor_data(cur.src0(), ar_local)) { + res_tensor = cur.src1(); + } else if (htp_same_tensor_data(cur.src1(), ar_local)) { + res_tensor = cur.src0(); + } else { + return false; + } + + if (ar_local->type != res_tensor->type) return false; + + const bool is_same_shape = ggml_are_same_shape(ar_local, res_tensor); + const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] && + res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); + if (!is_same_shape && !is_row_bcast) return false; + + if (is_same_shape && !ggml_are_same_stride(ar_local, res_tensor)) { + return false; + } + if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) return false; + + const bool is_shard_ok = n_ranks <= HTP_OP_MAX_OUTPUTS && add_dst->data == ar_local->data; + + struct htp_allreduce_kernel_params kparams; + if (!htp_kparams_solve_allreduce(sess, add_dst, rank, n_ranks, true, is_row_bcast, is_shard_ok, &kparams)) { + HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", htp_session_name(sess)); + return false; + } + + const bool scatter_ok = kparams.n_dsts > 1; + kparams.mode = scatter_ok ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL; + + for (uint32_t r = 0; r < n_ranks; r++) { + const ggml_tensor * ar_src = prev.inputs[r]; + if (!htp_tensors_overlap(add_dst, ar_src)) continue; + if (scatter_ok && r == rank) continue; + HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", htp_session_name(sess), r); + return false; + } + + prev.opcode = HTP_OP_ALLREDUCE_ADD; + prev.inputs.push_back(res_tensor); + if (kparams.mode != HTP_ALLREDUCE_SHARDED_FANOUT) { + prev.outputs = { add_dst }; + } + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_rms_norm_mul(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_RMS_NORM) return false; + + const ggml_tensor * weight = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &weight)) return false; + + const ggml_tensor * src0 = prev.src0(); + const ggml_tensor * dst = cur.dst(); + if (src0->type != GGML_TYPE_F32 || src0->ne[0] != weight->ne[0] || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false; + if (htp_inputs_overlap_unsafe(prev, dst, weight)) return false; + + const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1); + const bool is_same_shape = ggml_are_same_shape(src0, weight); + if (!is_row_bcast && !is_same_shape) return false; + + struct htp_unary_kernel_params kparams; + if (!htp_kparams_solve_rms_norm_mul(sess, src0, weight, dst, &kparams)) { + HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n", + htp_session_name(sess), kparams.vtcm_size, htp_session_vtcm_size(sess)); + return false; + } + + prev.opcode = HTP_OP_RMS_NORM_MUL; + prev.inputs = { src0, weight }; + prev.outputs = { dst }; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +// post-norm residual: dst = rms_norm(x) * w + r +static bool fuse_rms_norm_mul_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_RMS_NORM_MUL) return false; + + const ggml_tensor * r = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &r)) return false; + + const ggml_tensor * src0 = prev.src0(); + const ggml_tensor * weight = prev.inputs[1]; + const ggml_tensor * dst = cur.dst(); + if (!ggml_are_same_layout(src0, r)) return false; + if (htp_inputs_overlap_unsafe(prev, dst, r)) return false; + + struct htp_unary_kernel_params kparams; + if (!htp_kparams_solve_rms_norm_mul_add(sess, src0, weight, dst, &kparams)) return false; + + prev.opcode = HTP_OP_RMS_NORM_MUL_ADD; + prev.inputs.push_back(r); + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_rms_norm_scale(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + GGML_UNUSED(sess); + if (prev.opcode != HTP_OP_RMS_NORM && prev.opcode != HTP_OP_RMS_NORM_MUL) return false; + if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false; + + const ggml_tensor * src0 = prev.src0(); + const ggml_tensor * dst = cur.dst(); + if (src0->type != GGML_TYPE_F32 || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false; + if (htp_inputs_overlap_unsafe(prev, dst)) return false; + + float scale = 0.0f; + float bias = 0.0f; + memcpy(&scale, (const float *) cur.params + 0, sizeof(float)); + memcpy(&bias, (const float *) cur.params + 1, sizeof(float)); + if (bias != 0.0f || scale == 0.0f) return false; + + float prev_scale = 0.0f; + memcpy(&prev_scale, &prev.params[1], sizeof(float)); + if (prev_scale != 0.0f) { + scale *= prev_scale; + } + memcpy(&prev.params[1], &scale, sizeof(float)); + + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + return true; +} + +static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_MUL_MAT) return false; + + const ggml_tensor * src2 = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &src2)) return false; + if (src2->type != GGML_TYPE_F32) return false; + + const ggml_tensor * src0 = prev.node->src[0]; + const ggml_tensor * src1 = prev.node->src[1]; + const ggml_tensor * dst = cur.node; + if (!ggml_are_same_shape(prev.node, dst)) return false; + + const bool prev_collapsed = htp_matmul_can_collapse(src0, src1, prev.node); + const bool cur_collapsed = htp_matmul_add_can_collapse(src0, src1, src2, dst); + if (prev_collapsed != cur_collapsed) { + return false; + } + + const auto * orig_kparams = prev.kparams<htp_mm_kernel_params>(); + struct htp_mm_kernel_params kparams; + if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams)) { + return false; + } + + const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; + if (kparams.n_hmx == 0 && src1_nrows != 1) return false; + + if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) { + if (kparams.m_chunk < orig_kparams->m_chunk || + kparams.n_chunk < orig_kparams->n_chunk || + kparams.n_act_hvx < orig_kparams->n_act_hvx) { + HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, hvx %d->%d)\n", + htp_session_name(sess), orig_kparams->m_chunk, kparams.m_chunk, + orig_kparams->n_chunk, kparams.n_chunk, + orig_kparams->n_act_hvx, kparams.n_act_hvx); + return false; + } + } + + prev.opcode = HTP_OP_MUL_MAT_ADD; + prev.inputs = { src0, src1, src2 }; + prev.outputs = { dst }; + htp_opnode_collapse(sess, prev); + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_MUL_MAT && prev.opcode != HTP_OP_MUL_MAT_NX) return false; + + const uint32_t curr_n = (uint32_t) prev.outputs.size(); + if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) return false; + + const ggml_tensor * w0 = prev.inputs[0]; + const ggml_tensor * x_orig = prev.node->src[1]; + const ggml_tensor * w_in = cur.node->src[0]; + const ggml_tensor * x_in = cur.node->src[1]; + const ggml_tensor * d_in = cur.node; + + auto * orig_kparams = prev.kparams<htp_mm_kernel_params>(); + const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>(); + + if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || + w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 || + ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) || + orig_kparams->n_hmx != cur_kparams->n_hmx) { + return false; + } + + const bool collapse = htp_matmul_can_collapse(w0, x_orig, prev.node) && htp_matmul_can_collapse(w_in, x_in, d_in); + if ((x_orig->ne[2] > 1 || x_orig->ne[3] > 1) && !collapse) return false; + + if (prev.opcode == HTP_OP_MUL_MAT_NX) { + orig_kparams->n_weights = curr_n + 1; + } else { + struct htp_mm_kernel_params kparams; + const ggml_tensor x_collapsed = collapse ? htp_tensor_collapse_rows(x_orig) : *x_orig; + if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false; + if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) return false; + + prev.set_kparams(kparams); + prev.opcode = HTP_OP_MUL_MAT_NX; + } + + prev.inputs.insert(prev.inputs.begin() + curr_n, w_in); + prev.outputs.push_back(collapse ? prev.add_dummy(htp_tensor_collapse_rows(d_in)) : d_in); + prev.fused.push_back(cur.node); + return true; +} + +static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_MUL_MAT_ID && prev.opcode != HTP_OP_MUL_MAT_ID_NX) return false; + + const uint32_t curr_n = (uint32_t) prev.outputs.size(); + if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 2 >= HTP_OP_MAX_INPUTS) return false; + + const ggml_tensor * w0 = prev.inputs[0]; + const ggml_tensor * x = prev.inputs[curr_n]; + const ggml_tensor * ids = prev.inputs[curr_n + 1]; + const ggml_tensor * w_in = cur.src0(); + const ggml_tensor * x_in = cur.src1(); + const ggml_tensor * ids_in = cur.node->src[2]; + const ggml_tensor * d_in = cur.dst(); + + auto * orig_kparams = prev.kparams<htp_mm_kernel_params>(); + const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>(); + + if (x_in != x || ids_in != ids || w_in->type != w0->type || + w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] || + ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) || + orig_kparams->n_hmx != cur_kparams->n_hmx) { + return false; + } + + struct htp_mm_kernel_params kparams; + if (d_in->nb[1] == prev.node->nb[1]) { + kparams = *orig_kparams; + kparams.n_weights = curr_n + 1; + } else if (!htp_kparams_solve_mmidnx(sess, w0, x, d_in, curr_n + 1, &kparams)) { + return false; + } + if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) return false; + + prev.opcode = HTP_OP_MUL_MAT_ID_NX; + prev.inputs.insert(prev.inputs.begin() + curr_n, w_in); + prev.outputs.push_back(d_in); + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_gdn_cpy(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + GGML_UNUSED(sess); + if (prev.opcode != HTP_OP_GATED_DELTA_NET || prev.outputs.size() != 1) return false; + + const ggml_tensor * gdn_out = prev.dst(); + const ggml_tensor * cpy_node = cur.node; + const ggml_tensor * cpy_src = cur.src0(); + const ggml_tensor * cpy_dst = cur.dst(); + + if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false; + if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false; + + const ggml_tensor * v = prev.node->src[2]; + if (!v) return false; + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + const int64_t n_tokens = v->ne[2]; + const int64_t n_seqs = v->ne[3]; + const int64_t K = ggml_get_op_params_i32(prev.node, 0); + const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float); + const int64_t D = S_v * S_v * H; + const int64_t n_written = std::min<int64_t>(n_tokens, K); + + if (cpy_src->op != GGML_OP_VIEW || !htp_same_tensor_data(cpy_src->view_src, gdn_out) || + cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) { + return false; + } + + if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->ne[2] != n_written || !ggml_is_contiguous(cpy_dst)) { + return false; + } + + prev.outputs.push_back(cpy_dst); + prev.fused.push_back(cur.node); + return true; +} + +static bool fuse_ssm_conv_silu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + GGML_UNUSED(sess); + if (prev.opcode != HTP_OP_SSM_CONV) return false; + + auto * kparams = prev.kparams<htp_ssm_conv_kernel_params>(); + if (kparams->silu) return false; + if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false; + + const ggml_tensor * conv_dst = prev.dst(); + const ggml_tensor * dst = cur.dst(); + if (!ggml_are_same_layout(conv_dst, dst)) return false; + if (htp_tensors_overlap(dst, prev.src0()) || htp_tensors_overlap(dst, prev.src1())) return false; + + kparams->silu = 1; + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + return true; +} + +static bool fuse_repeat_get_rows(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_REPEAT) return false; + if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false; + + const ggml_tensor * repeat_out = prev.dst(); + const ggml_tensor * repeat_src0 = prev.src0(); + const ggml_tensor * gr_src1 = cur.src1(); + const ggml_tensor * gr_dst = cur.dst(); + + if (repeat_src0->ne[0] != repeat_out->ne[0] || repeat_src0->ne[1] != repeat_out->ne[1]) return false; + if (repeat_src0->type != repeat_out->type) return false; + + struct htp_get_rows_kernel_params kparams; + if (!htp_kparams_solve_repeat_get_rows(sess, repeat_src0, gr_src1, gr_dst, &kparams) || kparams.vtcm_size == 0) return false; + + prev.opcode = HTP_OP_GET_ROWS; + prev.inputs = { repeat_src0, gr_src1 }; + prev.outputs = { gr_dst }; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_unary_mul_glu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + htp_op_code glu_op = HTP_OP_INVALID; + switch (prev.opcode) { + case HTP_OP_UNARY_SILU: glu_op = HTP_OP_GLU_SWIGLU; break; + case HTP_OP_UNARY_GELU: glu_op = HTP_OP_GLU_GEGLU; break; + case HTP_OP_UNARY_GELU_QUICK: glu_op = HTP_OP_GLU_GEGLU_QUICK; break; + case HTP_OP_UNARY_GELU_ERF: glu_op = HTP_OP_GLU_GEGLU_ERF; break; + case HTP_OP_UNARY_RELU: glu_op = HTP_OP_GLU_REGLU; break; + case HTP_OP_UNARY_SIGMOID: glu_op = HTP_OP_GLU_SIGMOID; break; + default: return false; + } + + const ggml_tensor * up = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &up)) return false; + + const ggml_tensor * gate = prev.src0(); + const ggml_tensor * dst = cur.dst(); + if (htp_inputs_overlap_unsafe(prev, dst, up)) return false; + + prev.opcode = glu_op; + prev.inputs = { gate, up }; + prev.outputs = { dst }; + if (!htp_kparams_solve(sess, prev)) return false; + + prev.fused.push_back(cur.node); + memset(prev.params, 0, sizeof(prev.params)); + return true; +} + +static bool fuse_mul_mul_scalar(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_MUL || prev.inputs.size() != 2) return false; + + const auto * k0 = prev.kparams<htp_binary_kernel_params>(); + const auto * k1 = cur.kparams<htp_binary_kernel_params>(); + if (k0->kernel_type != HTP_BINARY_KERNEL_SCALAR || k1->kernel_type != HTP_BINARY_KERNEL_SCALAR) return false; + if (k0->src1_strided || k1->src1_strided) return false; + + const ggml_tensor * src2 = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ false, &src2)) return false; + + const ggml_tensor * src1 = prev.src1(); + const ggml_tensor * dst = cur.dst(); + if (!ggml_are_same_shape(src1, src2) || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false; + if (htp_inputs_overlap_unsafe(prev, dst, src2)) return false; + + const size_t vtcm_size = htp_session_vtcm_size(sess); + struct htp_binary_kernel_params kparams = *k0; + kparams.src2_size = kparams.src1_size; + kparams.rows_per_buffer = 0; + if (kparams.src1_size > 65536 || kparams.src1_size + kparams.src2_size > vtcm_size) return false; + + struct htp_binary_vtcm_layout L; + htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size); + if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) return false; + + kparams.rows_per_buffer = L.rows_per_buffer; + kparams.vtcm_size = L.total_bytes; + + prev.inputs.push_back(src2); + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_binary_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != cur.opcode) return false; + + const uint32_t curr_n = (uint32_t) prev.inputs.size(); + if (curr_n < 2 || curr_n >= HTP_OP_MAX_INPUTS) return false; + + const auto * k0 = prev.kparams<htp_binary_kernel_params>(); + const auto * k1 = cur.kparams<htp_binary_kernel_params>(); + const bool k0_ok = (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) || + (k0->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k0->is_scalar); + const bool k1_ok = (k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) || + (k1->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k1->is_scalar); + if (!k0_ok || !k1_ok) return false; + + const bool is_comm = (cur.opcode == HTP_OP_ADD || cur.opcode == HTP_OP_MUL); + const ggml_tensor * new_src = nullptr; + if (!htp_match_consumer(prev, cur, is_comm, &new_src)) return false; + + const ggml_tensor * src0 = prev.inputs[0]; + const ggml_tensor * dst = cur.dst(); + if (!ggml_are_same_shape(src0, prev.inputs[1]) || + !ggml_are_same_shape(src0, new_src) || + !htp_same_shape_and_contiguity(prev.dst(), dst)) { + return false; + } + if (htp_inputs_overlap_unsafe(prev, dst, new_src)) return false; + + const size_t vtcm_size = htp_session_vtcm_size(sess); + struct htp_binary_kernel_params kparams = *k0; + struct htp_binary_vtcm_layout L; + memset(&L, 0, sizeof(L)); + + const uint32_t n_srcs = curr_n + 1; + if (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE || k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) { + if (k0->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE) { + kparams = *k1; + } + kparams.n_srcs = (uint8_t) n_srcs; + kparams.rows_per_buffer = 0; + + auto check_split = [&](const ggml_tensor * t) { + if (src0->ne[2] > 1 && t->nb[2] != t->ne[1] * t->nb[1]) kparams.split_at_ne01 = 1; + if (src0->ne[3] > 1 && t->nb[3] != t->ne[2] * t->nb[2]) kparams.split_at_ne02 = 1; + }; + for (const auto * in : prev.inputs) check_split(in); + check_split(new_src); + check_split(dst); + + htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size); + } + + if (L.rows_per_buffer == 0 || L.total_bytes == 0 || L.total_bytes > vtcm_size) { + bool all_contig = ggml_is_contiguous(new_src) && ggml_is_contiguous(dst); + for (const auto * in : prev.inputs) { + all_contig = all_contig && ggml_is_contiguous(in); + } + if (!all_contig) return false; + + const uint32_t total_elems = (uint32_t) ggml_nelements(src0); + const uint32_t elem_size = (uint32_t) ggml_type_size(src0->type); + const uint32_t max_hvx = htp_session_n_hvx(sess); + if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_hvx, n_srcs, false, vtcm_size)) { + return false; + } + } + + kparams.rows_per_buffer = L.rows_per_buffer; + kparams.vtcm_size = (uint32_t) L.total_bytes; + + prev.inputs.push_back(new_src); + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +bool htp_fusion_can_stack(const struct ggml_tensor * a) { + if (!a || !a->src[0] || !a->src[1]) return false; + switch (a->op) { + case GGML_OP_MUL_MAT: + if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX)) return false; + if (htp_is_hadamard(a)) return false; + if (a->src[0]->ne[2] != 1 || a->src[0]->ne[3] != 1) return false; + return ggml_is_quantized(a->src[0]->type); + case GGML_OP_MUL_MAT_ID: + if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX)) return false; + return a->src[2] != nullptr && ggml_is_quantized(a->src[0]->type); + default: + return false; + } +} + +bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b) { + if (!htp_fusion_can_stack(b) || a->op != b->op || a->src[1] != b->src[1]) { + return false; + } + if (a->op == GGML_OP_MUL_MAT_ID && a->src[2] != b->src[2]) { + return false; + } + const ggml_tensor * w0 = a->src[0]; + const ggml_tensor * w1 = b->src[0]; + return w0->type == w1->type && w0->ne[0] == w1->ne[0] && w0->ne[2] == w1->ne[2]; +} + +int htp_fusion_max(const struct ggml_cgraph * graph, int idx) { + const int n = graph->n_nodes; + const ggml_op op0 = graph->nodes[idx]->op; + if (op0 != GGML_OP_ADD && op0 != GGML_OP_NORM && op0 != GGML_OP_RMS_NORM) { + return 1; + } + + enum ggml_op ops[HTP_OP_MAX_FUSED]; + ops[0] = op0; + + int f = idx + 1; + while (f < n && f < idx + HTP_OP_MAX_FUSED) { + const ggml_op op = graph->nodes[f]->op; + if (op != GGML_OP_ADD && op != GGML_OP_MUL && op != GGML_OP_SCALE && + op != GGML_OP_NORM && op != GGML_OP_RMS_NORM) { + break; + } + ops[f - idx] = op; + f++; + } + + f -= idx; + for (; f > 1; f--) { + if (ggml_can_fuse(graph, idx, ops, f)) { + break; + } + } + return f; +} + +void htp_fusion_tag_graph(struct ggml_cgraph * graph) { + for (int i = 0; i < graph->n_nodes; i++) { + ggml_tensor * node = graph->nodes[i]; + ggml_hexagon_tensor_clear_flags(node, HTP_TENSOR_FUSEABLE); + + const ggml_op op = node->op; + if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT || + op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB || + op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID || + op == GGML_OP_SSM_CONV || + (op == GGML_OP_MUL_MAT && !htp_is_hadamard(node))) && + ggml_node_has_n_uses(graph, i, 1)) { + ggml_hexagon_tensor_set_flags(node, HTP_TENSOR_FUSEABLE); + } + } +} + +bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur) { + if (!opt_opfusion) return false; + + switch (cur.opcode) { + case HTP_OP_ADD: + return (htp_fusion_enabled(HTP_FUSE_ALLREDUCE_ADD) && fuse_allreduce_add(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_MUL_MAT_ADD) && fuse_mul_mat_add(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_RMS_NORM_MUL_ADD) && fuse_rms_norm_mul_add(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur)); + case HTP_OP_MUL: + return (htp_fusion_enabled(HTP_FUSE_RMS_NORM_MUL) && fuse_rms_norm_mul(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_UNARY_MUL_GLU) && fuse_unary_mul_glu(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_MUL_MUL_SCALAR) && fuse_mul_mul_scalar(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur)); + case HTP_OP_SUB: + case HTP_OP_DIV: + return htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur); + case HTP_OP_SCALE: + return htp_fusion_enabled(HTP_FUSE_RMS_NORM_SCALE) && fuse_rms_norm_scale(sess, prev, cur); + case HTP_OP_UNARY_SILU: + return htp_fusion_enabled(HTP_FUSE_SSM_CONV_SILU) && fuse_ssm_conv_silu(sess, prev, cur); + case HTP_OP_GET_ROWS: + return htp_fusion_enabled(HTP_FUSE_REPEAT_GET_ROWS) && fuse_repeat_get_rows(sess, prev, cur); + case HTP_OP_CPY: + return htp_fusion_enabled(HTP_FUSE_GDN_CPY) && fuse_gdn_cpy(sess, prev, cur); + case HTP_OP_MUL_MAT: + return htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX) && fuse_mul_mat_nx(sess, prev, cur); + case HTP_OP_MUL_MAT_ID: + return htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX) && fuse_mul_mat_id_nx(sess, prev, cur); + default: + return false; + } +} diff --git a/ggml/src/ggml-hexagon/htp-fusion.h b/ggml/src/ggml-hexagon/htp-fusion.h new file mode 100644 index 000000000..e4f711c87 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-fusion.h @@ -0,0 +1,34 @@ +#ifndef HTP_FUSION_H +#define HTP_FUSION_H + +#include "htp-common.h" + +enum htp_fusion_flags { + HTP_FUSE_ALLREDUCE_ADD = (1 << 1), // 2 + HTP_FUSE_RMS_NORM_MUL = (1 << 2), // 4 + HTP_FUSE_MUL_MAT_ADD = (1 << 3), // 8 + HTP_FUSE_MUL_MAT_NX = (1 << 4), // 16 + HTP_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32 + HTP_FUSE_GDN_CPY = (1 << 6), // 64 + HTP_FUSE_REPEAT_GET_ROWS = (1 << 7), // 128 + HTP_FUSE_UNARY_MUL_GLU = (1 << 8), // 256 + HTP_FUSE_MUL_MUL_SCALAR = (1 << 9), // 512 + HTP_FUSE_BINARY_NX = (1 << 10), // 1024 + HTP_FUSE_RMS_NORM_SCALE = (1 << 11), // 2048 + HTP_FUSE_SSM_CONV_SILU = (1 << 12), // 4096 + HTP_FUSE_RMS_NORM_MUL_ADD = (1 << 13), // 8192 +}; + +static inline bool htp_fusion_enabled(int flag) { + if (opt_opfusion <= 0) return false; + if (opt_opfusion == 1) return true; // 1 enables all + return (opt_opfusion & flag) != 0; +} + +bool htp_fusion_can_stack(const struct ggml_tensor * a); +bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b); +int htp_fusion_max(const struct ggml_cgraph * graph, int idx); +void htp_fusion_tag_graph(struct ggml_cgraph * graph); +bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur); + +#endif // HTP_FUSION_H diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp new file mode 100644 index 000000000..8c75be289 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -0,0 +1,5533 @@ +#define _USE_MATH_DEFINES + +#include "htp-kparams.h" +#include "htp-opnode.h" +#include "ggml-impl.h" + +#include <algorithm> +#include <cmath> +#include <cstring> + +#ifndef M_LOG2E +# define M_LOG2E 1.44269504088896340736 +#endif + +htp_op_code htp_op_remap(const struct ggml_tensor * t) { + switch (t->op) { + case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT; + case GGML_OP_MUL_MAT: return htp_is_hadamard(t) ? HTP_OP_HADAMARD : HTP_OP_MUL_MAT; + case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID; + case GGML_OP_MUL: return HTP_OP_MUL; + case GGML_OP_ADD: return HTP_OP_ADD; + case GGML_OP_ADD_ID: return HTP_OP_ADD_ID; + case GGML_OP_SUB: return HTP_OP_SUB; + case GGML_OP_DIV: return HTP_OP_DIV; + case GGML_OP_DUP: + case GGML_OP_CPY: + case GGML_OP_CONT: return HTP_OP_CPY; + case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS; + case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS; + case GGML_OP_SUM: return HTP_OP_SUM; + case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS; + case GGML_OP_POOL_2D: return HTP_OP_POOL_2D; + case GGML_OP_POOL_1D: return HTP_OP_POOL_1D; + case GGML_OP_ARGSORT: return HTP_OP_ARGSORT; + case GGML_OP_TOP_K: return HTP_OP_TOP_K; + case GGML_OP_ARGMAX: return HTP_OP_ARGMAX; + case GGML_OP_NORM: return HTP_OP_NORM; + case GGML_OP_L2_NORM: return HTP_OP_L2_NORM; + case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM; + case GGML_OP_CONCAT: return HTP_OP_CONCAT; + case GGML_OP_SCALE: return HTP_OP_SCALE; + case GGML_OP_CLAMP: return HTP_OP_CLAMP; + case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU; + case GGML_OP_SQR: return HTP_OP_SQR; + case GGML_OP_SQRT: return HTP_OP_SQRT; + case GGML_OP_LOG: return HTP_OP_UNARY_LOG; + case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX; + case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV; + case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET; + case GGML_OP_ROPE: return HTP_OP_ROPE; + case GGML_OP_REPEAT: return HTP_OP_REPEAT; + case GGML_OP_CUMSUM: return HTP_OP_CUMSUM; + case GGML_OP_TOP_K_MASK: return HTP_OP_TOP_K_MASK; + case GGML_OP_FILL: return HTP_OP_FILL; + case GGML_OP_DIAG: return HTP_OP_DIAG; + case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI; + case GGML_OP_DSV4_HC_PRE: return HTP_OP_DSV4_HC_PRE; + case GGML_OP_DSV4_HC_POST: return HTP_OP_DSV4_HC_POST; + case GGML_OP_TRI: return HTP_OP_TRI; + case GGML_OP_PAD: return HTP_OP_PAD; + case GGML_OP_IM2COL: return HTP_OP_IM2COL; + case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D; + case GGML_OP_CONV_2D: return HTP_OP_CONV_2D; + case GGML_OP_CONV_2D_DW: return HTP_OP_CONV_2D_DW; + case GGML_OP_CONV_3D: return HTP_OP_CONV_3D; + case GGML_OP_ROLL: return HTP_OP_ROLL; + case GGML_OP_UPSCALE: return HTP_OP_UPSCALE; + case GGML_OP_LIGHTNING_INDEXER: return HTP_OP_LIGHTNING_INDEXER; + + case GGML_OP_UNARY: + switch (ggml_get_unary_op(t)) { + case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU; + case GGML_UNARY_OP_GELU: return HTP_OP_UNARY_GELU; + case GGML_UNARY_OP_GELU_QUICK: return HTP_OP_UNARY_GELU_QUICK; + case GGML_UNARY_OP_GELU_ERF: return HTP_OP_UNARY_GELU_ERF; + case GGML_UNARY_OP_SIGMOID: return HTP_OP_UNARY_SIGMOID; + case GGML_UNARY_OP_NEG: return HTP_OP_UNARY_NEG; + case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP; + case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS; + case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH; + case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS; + case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU; + case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP; + case GGML_UNARY_OP_CEIL: return HTP_OP_UNARY_CEIL; + case GGML_UNARY_OP_SGN: return HTP_OP_UNARY_SGN; + default: break; + } + break; + + case GGML_OP_GLU: + switch (ggml_get_glu_op(t)) { + case GGML_GLU_OP_REGLU: return HTP_OP_GLU_REGLU; + case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU; + case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI; + case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP; + case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU; + case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK; + case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF; + default: break; + } + break; + + default: + break; + } + return HTP_OP_INVALID; +} + +bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride) { + size_t s = 0, next = 0; + for (int i = 1; i < GGML_MAX_DIMS; i++) { + if (ne[i] == 1) continue; + if (s == 0) { s = nb[i]; next = s * ne[i]; continue; } + if (nb[i] != next) return false; + next *= ne[i]; + } + *stride = s ? s : nb[1]; + return true; +} + +bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst) { + size_t s1, sd; + return (dst->op == GGML_OP_MUL_MAT || dst->op == GGML_OP_ADD) && + src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] * src1->ne[3] > 1 && + src1->nb[0] == ggml_type_size(src1->type) && htp_rows_stride(src1->ne, src1->nb, &s1) && + dst->nb[0] == ggml_type_size(dst->type) && htp_rows_stride(dst->ne, dst->nb, &sd); +} + +ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t) { + size_t stride = 0; + htp_rows_stride(t->ne, t->nb, &stride); + ggml_tensor c = *t; + c.ne[1] = t->ne[1] * t->ne[2] * t->ne[3]; + c.ne[2] = 1; + c.ne[3] = 1; + c.nb[1] = stride; + c.nb[2] = c.nb[1] * c.ne[1]; + c.nb[3] = c.nb[2]; + return c; +} + +bool htp_kparams_solve_allreduce( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * dst, + uint32_t rank, + uint32_t n_ranks, + bool has_add, + bool is_row_bcast, + bool is_shard_ok, + struct htp_allreduce_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->rank = (int32_t) rank; + kparams->n_ranks = (int32_t) n_ranks; + kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; + + const uint32_t max_hvx = htp_session_n_hvx(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const uint32_t nelem = (uint32_t) ggml_nelements(dst); + const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); + const bool is_contiguous = ggml_is_contiguous(dst); + + const uint32_t ne0 = (uint32_t) dst->ne[0]; + const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + kparams->ne0 = (int32_t) ne0; + kparams->ne1 = (int32_t) ne1; + + const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1); + + if (has_add) { + if (opt_ar_scatter && is_shard_ok && use_1d && !is_row_bcast) { + const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128); + const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem); + const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem); + kparams->n_dsts = (int32_t) n_ranks; + kparams->rank_elem_start = (int32_t) rank_elem_start; + kparams->rank_nelem = (int32_t) (rank_elem_end - rank_elem_start); + } else if (use_1d) { + kparams->n_dsts = 1; + kparams->rank_elem_start = 0; + kparams->rank_nelem = (int32_t) nelem; + } else { + kparams->n_dsts = 1; + kparams->rank_elem_start = 0; + kparams->rank_nelem = (int32_t) ne1; + } + } else { + kparams->n_dsts = (int32_t) n_ranks; + if (use_1d) { + const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128); + const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem); + const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem); + const uint32_t rank_nelem = rank_elem_end - rank_elem_start; + kparams->rank_elem_start = (int32_t) rank_elem_start; + kparams->rank_nelem = (int32_t) rank_nelem; + } else { + const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks; + const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1); + const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1); + const uint32_t rank_nrows = rank_r1 - rank_r0; + kparams->rank_elem_start = (int32_t) rank_r0; + kparams->rank_nelem = (int32_t) rank_nrows; + } + } + + kparams->mode = (kparams->n_dsts > 1) ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL; + + if (use_1d) { + const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; + const uint32_t n_hvx = (std::min)(max_hvx, (std::max)(1u, rank_nelem / 128)); + kparams->n_hvx = n_hvx; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_hvx, has_add, is_row_bcast); + + uint32_t block_elems = 65536; + if (block_elems > rank_nelem / n_hvx && rank_nelem / n_hvx > 128) { + block_elems = hex_round_up(rank_nelem / (n_hvx * 2), 128); + } + block_elems = (std::max)(128u, block_elems); + + kparams->block_elems = block_elems; + kparams->vtcm_size_per_hvx = 2 * block_elems * elem_size; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; + + while ((size_t) kparams->vtcm_size > vtcm_size && block_elems > 128) { + const size_t max_bytes_per_buf = vtcm_size / (n_vtcm_buffers * 2); + block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); + if (block_elems < 128) break; + kparams->block_elems = block_elems; + kparams->vtcm_size_per_hvx = 2 * block_elems * elem_size; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; + } + + if (vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { + HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n", + htp_session_name(sess), kparams->vtcm_size, vtcm_size); + return false; + } + + kparams->elems_per_hvx = hex_round_up((rank_nelem + n_hvx - 1) / n_hvx, block_elems); + kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D; + return true; + } else { + const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; + const uint32_t n_hvx = (std::min)(max_hvx, (std::max)(1u, rank_nrows)); + kparams->n_hvx = n_hvx; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_hvx, has_add, is_row_bcast); + + const uint32_t row_bytes = ne0 * elem_size; + const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); + kparams->row_size_aligned = row_size_aligned; + + const uint32_t nrows_per_hvx = (rank_nrows + n_hvx - 1) / n_hvx; + uint32_t block_rows = (std::min)(128u, nrows_per_hvx); + block_rows = (std::max)(1u, block_rows); + kparams->block_elems = block_rows; + + kparams->vtcm_size_per_hvx = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; + + while ((size_t) kparams->vtcm_size > vtcm_size && block_rows > 1) { + const size_t max_rows_per_buf = vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned); + block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); + kparams->block_elems = block_rows; + kparams->vtcm_size_per_hvx = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; + if (max_rows_per_buf == 0) break; + } + + if (vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) { + HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n", + htp_session_name(sess), kparams->vtcm_size, vtcm_size); + return false; + } + + kparams->elems_per_hvx = nrows_per_hvx; + kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D; + return true; + } +} + +static bool htp_flash_attn_is_hmx_eligible( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * v, + const struct ggml_tensor * sinks +) { + if (htp_session_n_hmx(sess) == 0 || opt_fa_select < 2) { + return false; + } + + if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) || + (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) { + return false; + } + + const uint32_t DK = q->ne[0]; + const uint32_t DV = v->ne[0]; + + if (DK % 8 != 0 || DV % 8 != 0) { + return false; + } + + GGML_UNUSED(sinks); + + if (opt_fa_select > 2) { + return true; + } + + const uint32_t M = q->ne[1]; + if (M > 1) { + return true; + } + + const uint32_t n_hvx = htp_session_n_hvx(sess); + const uint32_t n_head = q->ne[2]; + const uint32_t n_kv_heads = k->ne[2]; + const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1; + const uint32_t S = k->ne[1]; + const uint32_t total_threads = n_hvx * htp_session_mdev_count(sess); + + if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) { + return true; + } + + const bool is_tile_aligned = (G > 0 && (32 % G == 0)); + if (!is_tile_aligned) { + if (n_hvx >= 6) { + return false; + } + return S >= 1024; + } + + uint32_t s_cross = 512; + if (DK == 256) { + s_cross = 4096; + } else if (DK <= 64) { + s_cross = (n_hvx >= 8) ? 2048 : ((n_hvx >= 6) ? 768 : 512); + } else { + s_cross = (n_hvx >= 8) ? 1024 : 512; + } + + return S >= s_cross; +} + +static bool htp_kparams_solve_flash_attn( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_fa_kernel_params * kparams +) { + if (opt_fa_select < 1) { + return false; + } + + // K/V row lists (ggml_flash_attn_ext_rows) are not supported + if (op->src[5] != nullptr) { + return false; + } + + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * q = op->src[0]; + const struct ggml_tensor * k = op->src[1]; + const struct ggml_tensor * v = op->src[2]; + const struct ggml_tensor * mask = op->src[3]; + const struct ggml_tensor * sinks = op->src[4]; + const struct ggml_tensor * dst = op; + + if ((q->type != GGML_TYPE_F16 && q->type != GGML_TYPE_F32) || + (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) || + (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) { + return false; + } + if (mask && mask->type != GGML_TYPE_F16) { + return false; + } + if (sinks && sinks->type != GGML_TYPE_F32) { + return false; + } + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { + return false; + } + + const uint32_t n_hvx = htp_session_n_hvx(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const uint32_t neq0 = q->ne[0]; + const uint32_t neq1 = q->ne[1]; + const uint32_t neq2 = q->ne[2]; + const uint32_t nek1 = k->ne[1]; + const uint32_t nev0 = v->ne[0]; + + const uint32_t DK = neq0; + const uint32_t DV = nev0; + + const uint32_t n_kv_heads = k->ne[2]; + const uint32_t G = neq2 / n_kv_heads; + + float scale = 1.0f; + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&scale, &op->op_params[0], sizeof(float)); + memcpy(&max_bias, &op->op_params[1], sizeof(float)); + memcpy(&logit_softcap, &op->op_params[2], sizeof(float)); + + if (logit_softcap != 0.0f) { + scale /= logit_softcap; + } + + kparams->scale = scale; + kparams->max_bias = max_bias; + kparams->logit_softcap = logit_softcap; + kparams->head_split = (opt_fa_split != 0) ? 1 : 0; + kparams->flags = 0; + kparams->G = G; + + // team-split only when a team has >1 member (n_kv_heads < N_cores) + uint32_t fa_teams = 1, fa_team_size = 1; + const uint32_t mdev_count = htp_session_mdev_count(sess); + if (opt_fa_split && mdev_count > 1) { + htp_fa_mdev_teams(n_kv_heads, mdev_count, kparams->head_split != 0, &fa_teams, &fa_team_size); + if (fa_team_size > 1) { + kparams->flags |= HTP_FA_FLAG_TEAM_SPLIT; + } + } + + const uint32_t n_head = q->ne[2]; + kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head)); + kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2); + kparams->m1 = std::pow(2.0f, -(max_bias / 2.0f) / kparams->n_head_log2); + + if (htp_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) { + const uint32_t DK_pad = hex_round_up(DK, 64); + const uint32_t DV_pad = hex_round_up(DV, 64); + size_t Br = 0, Bc = 0; + int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_hvx, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size); + if (ret == 0) { + kparams->kernel_type = HTP_FA_KERNEL_HMX; + if (logit_softcap == 0.0f) { + kparams->scale = scale * (float) M_LOG2E; + kparams->logit_softcap = 0.0f; + } else { + kparams->scale = scale; + kparams->logit_softcap = logit_softcap * (float) M_LOG2E; + } + kparams->Br = Br; + kparams->Bc = Bc; + kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc; + kparams->n_hvx = (kparams->n_kv_blocks >= 3 && n_hvx >= 2) ? n_hvx : 1; + kparams->u.hmx.g_br = hex_align_up(G * Br, 32); + kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_hvx >= 2) ? 1 : 0; + kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_hvx, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head); + kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0; + kparams->u.hmx.div_G = init_fastdiv_values(G); + if (mask) { + kparams->src3_div2 = init_fastdiv_values(mask->ne[2]); + kparams->src3_div3 = init_fastdiv_values(mask->ne[3]); + } + return true; + } + } + + kparams->kernel_type = HTP_FA_KERNEL_HVX; + kparams->Br = 1; + kparams->Bc = 64; + kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64; + kparams->n_hvx = n_hvx; + + const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128); + const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128); + const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128); + + kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_hvx); + if ((size_t) kparams->vtcm_size > vtcm_size) { + HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n", + kparams->vtcm_size, vtcm_size); + return false; + } + + kparams->u.hvx.size_q_row_padded = size_q_row_padded; + kparams->u.hvx.size_k_row_padded = size_k_row_padded; + kparams->u.hvx.size_v_row_padded = size_v_row_padded; + kparams->u.hvx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]); + kparams->u.hvx.src0_div1 = init_fastdiv_values(q->ne[1]); + kparams->broadcast_rk2 = init_fastdiv_values(q->ne[2] / k->ne[2]); + kparams->broadcast_rk3 = init_fastdiv_values(q->ne[3] / k->ne[3]); + kparams->broadcast_rv2 = init_fastdiv_values(q->ne[2] / v->ne[2]); + kparams->broadcast_rv3 = init_fastdiv_values(q->ne[3] / v->ne[3]); + if (mask) { + kparams->src3_div2 = init_fastdiv_values(mask->ne[2]); + kparams->src3_div3 = init_fastdiv_values(mask->ne[3]); + } + + return true; +} + +static bool htp_kparams_solve_gated_delta_net( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_gdn_kernel_params * kparams +) { + if (opt_gdn_select < 1) { + return false; + } + + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * q = op->src[0]; + const struct ggml_tensor * k = op->src[1]; + const struct ggml_tensor * v = op->src[2]; + const struct ggml_tensor * g = op->src[3]; + const struct ggml_tensor * beta = op->src[4]; + const struct ggml_tensor * state = op->src[5]; + const struct ggml_tensor * dst = op; + + if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 || + g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 || + dst->type != GGML_TYPE_F32) { + return false; + } + + if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) || + !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) || + !ggml_is_contiguous(dst)) { + return false; + } + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + const int64_t n_tokens = v->ne[2]; + const int64_t n_seqs = v->ne[3]; + const int64_t K = ggml_get_op_params_i32(op, 0); + + if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) { + return false; + } + if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 || + q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 || + (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) { + return false; + } + if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) { + return false; + } + if (state->ne[0] != S_v || state->ne[1] != S_v || state->ne[2] != H || state->ne[3] != n_seqs) { + return false; + } + if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K || + dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + + const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]); + const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]); + const uint32_t total_rows = (uint32_t) (H * n_seqs); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), total_rows); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const bool can_use_hmx = (opt_gdn_select >= 2) && + (htp_session_n_hmx(sess) > 0) && + (S_v % 64 == 0) && + (n_tokens >= HTP_GDN_MIN_TOKENS) && + (g->ne[0] == 1) && + (K == 1); + + struct htp_gdn_hmx_vtcm_layout hmx_layout; + struct htp_gdn_vtcm_layout hvx_layout; + uint32_t n_heads_batch = 1; + + if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_hvx, true, &n_heads_batch)) { + kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED; + kparams->pipeline = hmx_layout.pipeline ? 1 : 0; + kparams->chunk_size = HTP_GDN_CHUNK_SIZE; + kparams->n_chunks = ((uint32_t) n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE; + kparams->n_heads_batch = (uint16_t) n_heads_batch; + kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes; + } else { + htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_hvx); + if (hvx_layout.total_bytes > vtcm_size) { + return false; + } + kparams->kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT; + kparams->pipeline = 0; + kparams->n_heads_batch = 1; + kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes; + } + + kparams->n_hvx = n_hvx; + kparams->S_v = (uint32_t) S_v; + kparams->H = (uint32_t) H; + kparams->n_tokens = (uint32_t) n_tokens; + kparams->n_seqs = (uint32_t) n_seqs; + kparams->K = (uint32_t) K; + kparams->total_rows = total_rows; + kparams->kda = (g->ne[0] == S_v) ? 1 : 0; + kparams->state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float)); + kparams->state_size_per_snap = (uint32_t) (S_v * S_v * H * n_seqs); + kparams->scale = 1.0f / sqrtf((float) S_v); + + if (H > 0) kparams->div_H = init_fastdiv_values((uint32_t) H); + if (q->ne[1] > 0) kparams->div_q1 = init_fastdiv_values((uint32_t) q->ne[1]); + if (k->ne[1] > 0) kparams->div_k1 = init_fastdiv_values((uint32_t) k->ne[1]); + if (rq3 > 0) kparams->div_rq3 = init_fastdiv_values(rq3); + if (rk3 > 0) kparams->div_rk3 = init_fastdiv_values(rk3); + return true; +} + +static bool htp_matmul_is_hmx_eligible( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + bool is_matmul_id, + bool is_batched +) { + GGML_UNUSED(dst); + + if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) { + return false; + } + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int wtype = src0->type; + + if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) { + return false; + } + + if (!htp_is_tiled_weight_type((ggml_type) wtype)) { + return false; + } + + if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) { + return false; + } + + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); + if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) { + return false; + } + if (!is_matmul_id && is_batched && is_tiled) { + return false; + } + + if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) { + return false; + } + + const int m = is_matmul_id ? ne12 : ne11; + if (m <= HTP_MM_HMX_MIN_NROWS) { + return false; + } + + return true; +} + +static bool htp_kparams_solve_hmx_mm( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + const struct ggml_tensor * dst, + int wtype, + int ne00_padded, + int ne01_padded, + int ne02, + int ne11, + int ne12, + int ne11_padded, + bool is_matmul_id, + bool is_batched, + size_t src2_size, + size_t vtcm_budget, + struct htp_mm_kernel_params * kparams +) { + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); + const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0; + const int n_hvx = (int) htp_session_n_hvx(sess); + const uint32_t mdev_count = htp_session_mdev_count(sess); + const int ne10 = act->ne[0]; + + int m_for_solver = ne11; + int m_for_solver_padded = ne11_padded; + if (!is_matmul_id && mdev_count > 1 && ((uint32_t) ne11 >= mdev_count)) { + const bool dst_row_split = dst ? htp_can_row_partition(dst) : true; + const bool act_row_split = htp_can_row_partition(act); + if (dst_row_split && act_row_split) { + m_for_solver = (ne11 + (int) mdev_count - 1) / (int) mdev_count; + m_for_solver_padded = hex_round_up(std::max(m_for_solver, 32), 32); + } + } + const bool pipeline = is_matmul_id ? false : htp_mm_hmx_pipeline(m_for_solver); + const bool is_batched_val = is_matmul_id ? false : is_batched; + const int group_size = (ne02 > 0 ? ne12 / ne02 : 1); + + size_t m_chunk = 0; + size_t n_chunk = 0; + size_t vtcm_size = 0; + bool use_grouped = false; + int act_hvx_selected = 0; + + if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) { + if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_hvx, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_hvx_selected, &vtcm_size)) { + use_grouped = true; + } + } + + if (!use_grouped) { + int m_id_rows = 0; + if (dst && is_matmul_id) { + const int n_experts = ne02 > 0 ? ne02 : 1; + const size_t total_expert_rows = (size_t) dst->ne[1] * dst->ne[2]; + int m_per_expert = (int) ((total_expert_rows + n_experts - 1) / n_experts); + if (mdev_count > 1 && htp_can_row_partition(dst)) { + m_per_expert = (m_per_expert + (int) mdev_count - 1) / (int) mdev_count; + } + m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32); + } + const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver; + if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_hvx, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_hvx_selected, &vtcm_size)) { + return false; + } + } + + kparams->n_hmx = 1; + kparams->pipeline = pipeline ? 1 : 0; + kparams->m_chunk = m_chunk; + kparams->n_chunk = n_chunk; + kparams->n_hvx = n_hvx; + kparams->n_act_hvx = act_hvx_selected; + kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0; + kparams->aligned_tile_size = aligned_tile_size; + kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + kparams->vtcm_size = vtcm_size; + kparams->div_n_act_hvx = init_fastdiv_values(act_hvx_selected); + kparams->div_ne00_padded = init_fastdiv_values(ne00_padded); + kparams->vtcm_bias_size = (int32_t) src2_size; + kparams->kernel_type = (is_batched && !is_matmul_id) ? HTP_MM_KERNEL_HMX_BATCHED : HTP_MM_KERNEL_HMX_2D; + return true; +} + +static void htp_kparams_solve_hvx_mm( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + int wtype, + bool is_tiled, + int ne02, + int ne03, + int ne10, + int ne11, + int ne12, + int ne13, + bool is_matmul_id, + const size_t src2_row_size, + size_t vtcm_budget, + struct htp_mm_kernel_params * kparams +) { + if (opt_mm_select < 1) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } + + const uint32_t n_hvx = htp_session_n_hvx(sess); + kparams->n_hmx = 0; + kparams->n_hvx = n_hvx; + + GGML_UNUSED(ne02); + GGML_UNUSED(ne03); + + const int src1_nrows = ne11 * ne12 * ne13; + + if (is_tiled) { + kparams->tile_size = htp_tensor_get_tile_size(wtype); + kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + + const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0); + + if (is_matmul_id) { + kparams->kernel_type = (src1_nrows < (int) n_hvx) + ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; + kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + + struct htp_mm_hvx_vtcm_layout L; + uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, + 0, src0->nb[1], kparams->act_row_size, 0, d, true, false + ); + if (L.total_bytes <= vtcm_budget) { + best_n_prefetch = d; + break; + } + } + if (L.total_bytes > vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } + kparams->n_prefetch = best_n_prefetch; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_bias_size = 0; + return; + } else { + if (k_align && opt_mm_select >= 1) { + kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + kparams->kernel_type = (src1_nrows < (int) n_hvx) + ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; + + struct htp_mm_hvx_vtcm_layout L; + uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false + ); + if (L.total_bytes <= vtcm_budget) { + best_n_prefetch = d; + break; + } + } + + uint32_t m_chunk = 0; + if (htp_mm_hvx_solve_vtcm_params( + kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget, + &L, &m_chunk)) { + kparams->n_prefetch = best_n_prefetch; + kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_bias_size = L.bias_bytes; + return; + } + } + + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } + } else if (is_matmul_id) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } else if (wtype == GGML_TYPE_F16 || wtype == GGML_TYPE_F32 || wtype == GGML_TYPE_BF16) { + int32_t ktype = HTP_MM_KERNEL_HVX_F16_F16_VTCM; + int32_t act_elem = 2; + if (wtype == GGML_TYPE_F32) { + ktype = HTP_MM_KERNEL_HVX_F32_F32_VTCM; + act_elem = 4; + } else if (wtype == GGML_TYPE_BF16) { + ktype = HTP_MM_KERNEL_HVX_BF16_F32_VTCM; + act_elem = 4; + } + struct htp_mm_hvx_vtcm_layout L; + uint32_t m_chunk = 0; + if (htp_mm_hvx_solve_vtcm_params( + ktype, wtype, ne10, src1_nrows, n_hvx, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget, + &L, &m_chunk)) { + kparams->kernel_type = ktype; + kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; + kparams->act_row_size = hex_round_up(ne10 * act_elem, 128); + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_bias_size = L.bias_bytes; + kparams->n_prefetch = 16; + return; + } + } + + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; +} + +static bool htp_kparams_solve_matmul_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + const size_t src2_row_size, + const size_t src2_size, + struct htp_mm_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; + + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); + const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; + const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00; + const int ne11_padded = hex_round_up(ne11, 32); + const int ne01_tiled = hex_round_up(ne01, 32); + + const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID); + const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); + const size_t vtcm_budget = htp_session_vtcm_size(sess); + + const bool hmx_enabled = (htp_session_n_hmx(sess) > 0) && (opt_mm_select >= 2); + if (hmx_enabled && htp_matmul_is_hmx_eligible(src0, src1, dst, is_matmul_id, is_batched)) { + if (htp_kparams_solve_hmx_mm(sess, src0, src1, dst, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, src2_size, vtcm_budget, kparams)) { + goto finalize; + } + } + + htp_kparams_solve_hvx_mm(sess, src0, src1, dst, wtype, is_tiled, ne02, ne03, ne10, ne11, ne12, ne13, is_matmul_id, src2_row_size, vtcm_budget, kparams); + +finalize: + kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11); + kparams->div_ne1 = init_fastdiv_values(ne11); + kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); + kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); + kparams->div_ne12 = init_fastdiv_values(ne12); + + if (kparams->kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams->vtcm_size > vtcm_budget) { + HEX_VERBOSE("ggml-hex: %s supported %s VTCM size needed (%d) > budget (%zu)\n", + htp_session_name(sess), is_matmul_id ? "MUL_MAT_ID" : "MUL_MAT", kparams->vtcm_size, vtcm_budget); + return false; + } + return true; +} + +static bool htp_matmul_check_types( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst +) { + const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID); + + if (dst->type != GGML_TYPE_F32) { + return false; + } + + if (ggml_prec_get_src(dst, 1) == GGML_PREC_BF16 || ggml_prec_get_src(dst, 1) == GGML_PREC_F32) { + return false; + } + + if (is_matmul_id) { + const struct ggml_tensor * src2 = dst->src[2]; + if (src1->type != GGML_TYPE_F32 || !src2 || src2->type != GGML_TYPE_I32) { + return false; + } + switch (src0->type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q2_K: + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) { + return false; + } + if (src0->ne[0] % ggml_blck_size(src0->type) != 0) { + return false; + } + return true; + + case GGML_TYPE_F16: + case GGML_TYPE_F32: + case GGML_TYPE_BF16: + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) { + return false; + } + return src0->ne[0] % 32 == 0; + + default: + return false; + } + } + + if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { + return false; + } + if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3] || + src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) { + return false; + } + + switch (src0->type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q2_K: + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) { + return false; + } + return src0->ne[0] % ggml_blck_size(src0->type) == 0; + + case GGML_TYPE_F32: + if (src1->type != GGML_TYPE_F32) { + return false; + } + return src0->nb[1] >= src0->nb[0]; + + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + return src0->nb[1] >= src0->nb[0]; + + default: + return false; + } +} + +static bool htp_kparams_solve_matmul( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_mm_kernel_params * kparams +) { + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (!htp_matmul_check_types(src0, src1, dst)) { + return false; + } + if (htp_matmul_can_collapse(src0, src1, dst)) { + const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1); + const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst); + return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams); + } + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, kparams); +} + +bool htp_matmul_add_can_collapse( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * src2, + const struct ggml_tensor * dst +) { + if (!htp_matmul_can_collapse(src0, src1, dst)) { + return false; + } + if (!src2) { + return true; + } + const int64_t src2_nrows = src2->ne[1] * src2->ne[2] * src2->ne[3]; + if (src2_nrows == 1) { + return true; + } + size_t s2; + return src2->nb[0] == ggml_type_size(src2->type) && + ggml_are_same_shape(src2, dst) && + htp_rows_stride(src2->ne, src2->nb, &s2); +} + +bool htp_kparams_solve_matmul_add( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * src2, + const struct ggml_tensor * dst, + struct htp_mm_kernel_params * kparams +) { + if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) { + const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1); + const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst); + const ggml_tensor src2_collapsed = (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) + ? htp_tensor_collapse_rows(src2) + : (src2 ? *src2 : ggml_tensor{}); + const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr; + const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0; + return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams); + } + const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0; + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams); +} + +bool htp_kparams_solve_mmnx( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + int32_t n_weights, + struct htp_mm_kernel_params * kparams +) { + const uint32_t n_hvx = htp_session_n_hvx(sess); + const uint32_t n_hmx = htp_session_n_hmx(sess); + const size_t vtcm_budget = htp_session_vtcm_size(sess); + + memset(kparams, 0, sizeof(*kparams)); + kparams->n_hvx = n_hvx; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const int ne10 = act->ne[0]; + const int ne11 = act->ne[1]; + const int ne12 = act->ne[2]; + const int ne13 = act->ne[3]; + + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); + const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; + const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00; + const int ne11_padded = hex_round_up(ne11, 32); + const int ne01_tiled = hex_round_up(ne01, 32); + const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); + + const bool hmx_enabled = (n_hmx > 0) && (opt_mm_select >= 2); + if (hmx_enabled && htp_matmul_is_hmx_eligible(src0, act, nullptr, false, is_batched)) { + if (htp_kparams_solve_hmx_mm(sess, src0, act, nullptr, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, false, is_batched, 0, vtcm_budget, kparams)) { + kparams->n_weights = n_weights; + goto finalize; + } + } + + if (!is_tiled || act->type != GGML_TYPE_F32 || ne10 % 32 != 0) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return false; + } + + { + const int act_nrows = ne11 * ne12 * ne13; + const size_t act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + const size_t src0_row_size = src0->nb[1]; + + const uint32_t max_prefetch = (act_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_hvx, + 0, src0_row_size, act_row_size, 0, d, false, true + ); + if (L.total_bytes <= vtcm_budget) { + best_n_prefetch = d; + break; + } + } + + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_hvx, + 0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true + ); + + if (opt_mm_select >= 1 && L.total_bytes <= vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; + kparams->tile_size = htp_tensor_get_tile_size(wtype); + kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + kparams->act_row_size = act_row_size; + kparams->vtcm_bias_size = 0; + kparams->vtcm_size = L.total_bytes; + kparams->n_prefetch = best_n_prefetch; + kparams->n_weights = n_weights; + } else { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return false; + } + } + +finalize: + kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11); + kparams->div_ne1 = init_fastdiv_values(ne11); + kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); + kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); + kparams->div_ne12 = init_fastdiv_values(ne12); + return (size_t) kparams->vtcm_size <= vtcm_budget; +} + +bool htp_kparams_solve_mmidnx( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + const struct ggml_tensor * dst, + int32_t n_weights, + struct htp_mm_kernel_params * kparams +) { + const bool ok = htp_kparams_solve_matmul_impl(sess, src0, act, dst, 0, 0, kparams); + kparams->n_weights = n_weights; + return ok; +} + +static bool htp_kparams_solve_binary( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op_tensor, + struct htp_binary_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const uint32_t op = htp_op_remap(op_tensor); + const struct ggml_tensor * src0 = op_tensor->src[0]; + const struct ggml_tensor * src1 = op_tensor->src[1]; + const struct ggml_tensor * src2 = (op == HTP_OP_ADD_ID) ? op_tensor->src[2] : nullptr; + const struct ggml_tensor * dst = op_tensor; + + const bool is_add_id = (op == HTP_OP_ADD_ID); + if (is_add_id) { + if (!src2) { + return false; + } + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) { + return false; + } + if (!ggml_are_same_shape(src0, dst)) { + return false; + } + if (src1->ne[0] != src0->ne[0] || src1->ne[2] != 1 || src1->ne[3] != 1) { + return false; + } + if (src2->ne[0] != src0->ne[1] || src2->ne[1] != src0->ne[2]) { + return false; + } + if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t)) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { + return false; + } + if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) { + return false; + } + } else { + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || + src1->type != src0->type || dst->type != src0->type) { + return false; + } + if (ggml_is_permuted(src0) || ggml_is_permuted(dst) || !ggml_are_same_shape(src0, dst)) { + return false; + } + if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) { + return false; + } + } + + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const size_t elem_size = ggml_type_size(src0->type); + const size_t src0_row_size = src0->ne[0] * elem_size; + const size_t src1_row_size = src1->ne[0] * elem_size; + const size_t dst_row_size = dst->ne[0] * elem_size; + + const size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128); + const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128); + const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128); + + if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) { + return false; + } + + const bool is_scalar = !is_add_id && src1->ne[0] == 1; + const bool is_row_bcast = !is_add_id && !is_scalar && + src1->ne[0] == src0->ne[0] && + (src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) && + src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1; + const bool is_same_shape = !is_add_id && !is_scalar && + src1->ne[0] == src0->ne[0] && + src1->ne[1] == src0->ne[1] && + (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) && + (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1); + const bool is_complex = !is_add_id && !is_scalar && !is_same_shape && !is_row_bcast && (src1->ne[0] == src0->ne[0]); + const bool is_contig = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst); + const bool is_scalar_broadcast = !is_add_id && (ggml_nelements(src1) == 1); + + if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) { + const uint32_t total_elems = (uint32_t) ggml_nelements(src0); + const uint32_t max_chunk_elems = 32768 / elem_size; + const uint32_t min_chunk_elems = 256; + const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_hvx_sess) - 1) / (2 * max_hvx_sess), 32); + const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems)); + const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); + const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size; + const uint32_t n_hvx = (total_chunks >= 2) ? (std::min)(max_hvx_sess, total_chunks) : 1; + + kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; + kparams->n_hvx = n_hvx; + kparams->n_srcs = 2; + kparams->rows_per_buffer = 1; + kparams->src0_row_size_aligned = src0_row_size_aligned; + kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned; + kparams->dst_row_size_aligned = dst_row_size_aligned; + kparams->src1_size = 0; + kparams->chunk_size = chunk_size; + kparams->chunk_bytes = chunk_bytes; + kparams->is_scalar = is_scalar_broadcast ? 1 : 0; + + struct htp_binary_vtcm_layout L; + htp_binary_vtcm_layout_build(&L, kparams, vtcm_size); + if (L.total_bytes == 0 || L.total_bytes > vtcm_size) { + return false; + } + + kparams->vtcm_size = L.total_bytes; + return true; + } + + enum htp_binary_kernel_type kernel_type; + size_t src1_size = 0; + size_t src2_size = 0; + + if (is_add_id) { + kernel_type = HTP_BINARY_KERNEL_ADD_ID; + src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128); + src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128); + if (src1_size + src2_size > vtcm_size) { + return false; + } + } else if (is_row_bcast) { + kernel_type = HTP_BINARY_KERNEL_ROW_BCAST; + src1_size = src1_row_size_aligned; + } else if (is_scalar) { + src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128); + if (src1_size > vtcm_size) { + return false; + } + kernel_type = HTP_BINARY_KERNEL_SCALAR; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; + } else if (is_same_shape) { + kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE; + } else if (is_complex) { + src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128); + if (src1_size > vtcm_size) { + return false; + } + kernel_type = HTP_BINARY_KERNEL_COMPLEX; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; + } else { + src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128); + if (src1_size > vtcm_size) { + return false; + } + kernel_type = HTP_BINARY_KERNEL_REPEAT; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + + kparams->kernel_type = kernel_type; + kparams->n_hvx = (std::min)(max_hvx_sess, max_hvx); + kparams->n_srcs = 2; + kparams->src0_row_size_aligned = src0_row_size_aligned; + kparams->src1_row_size_aligned = src1_row_size_aligned; + kparams->dst_row_size_aligned = dst_row_size_aligned; + kparams->src1_size = src1_size; + kparams->src2_size = src2_size; + + const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]); + const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]); + const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]); + + const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]); + const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]); + const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]); + + bool split_at_ne01 = false; + if (src0->ne[2] > 1) { + if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) { + split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1; + } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1; + } else { + split_at_ne01 = true; + } + } + kparams->split_at_ne01 = split_at_ne01 ? 1 : 0; + + bool split_at_ne02 = false; + if (src0->ne[3] > 1) { + if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) { + split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2; + } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2; + } else { + split_at_ne02 = true; + } + } + kparams->split_at_ne02 = split_at_ne02 ? 1 : 0; + + kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + kparams->div_ne012 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[2])); + kparams->div_ne11 = init_fastdiv_values((uint32_t) src1->ne[1]); + kparams->div_ne12 = init_fastdiv_values((uint32_t) src1->ne[2]); + kparams->div_ne13 = init_fastdiv_values((uint32_t) src1->ne[3]); + + struct htp_binary_vtcm_layout L; + htp_binary_vtcm_layout_build(&L, kparams, vtcm_size); + if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) { + if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) { + const uint32_t total_elems = (uint32_t) ggml_nelements(src0); + return htp_binary_solve_chunked(&L, kparams, total_elems, (uint32_t) elem_size, + max_hvx_sess, 2, is_scalar_broadcast, vtcm_size); + } + return false; + } + + kparams->rows_per_buffer = L.rows_per_buffer; + kparams->vtcm_size = L.total_bytes; + + return true; +} + +// collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector +static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_hvx) { + if (n_tensors == 0 || !tensors[0]) { + return; + } + for (size_t t = 0; t < n_tensors; t++) { + if (!tensors[t] || tensors[t]->nb[0] != elem_size) { + return; + } + if (t > 0 && !ggml_are_same_shape(tensors[0], tensors[t])) { + return; + } + } + + struct ggml_tensor * t0 = tensors[0]; + + while (t0->ne[1] > 1 || t0->ne[2] > 1 || t0->ne[3] > 1) { + const size_t row_size = (size_t) t0->ne[0] * elem_size; + bool rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->nb[1] != (size_t) tensors[t]->ne[0] * elem_size) { + rows_contig = false; + break; + } + } + bool all_ne1_one = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->ne[1] != 1) { + all_ne1_one = false; + break; + } + } + const bool can_merge = all_ne1_one || + (rows_contig && (t0->ne[0] == 1 || (row_size < 128 && row_size * t0->ne[1] <= 2048))); + if (!can_merge) { + break; + } + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] *= tensors[t]->ne[1]; + tensors[t]->ne[1] = tensors[t]->ne[2]; + tensors[t]->ne[2] = tensors[t]->ne[3]; + tensors[t]->ne[3] = 1; + tensors[t]->nb[1] = tensors[t]->nb[2]; + tensors[t]->nb[2] = tensors[t]->nb[3]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } + } + + // ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1) + const size_t row_size = (size_t) t0->ne[0] * elem_size; + bool rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->nb[1] != row_size) { + rows_contig = false; + break; + } + } + if (t0->ne[1] > 1 && row_size < 128 && rows_contig) { + int64_t k = 2048 / row_size; + while (k > 1 && t0->ne[1] % k != 0) { + k--; + } + if (k > 1) { + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] *= k; + tensors[t]->ne[1] /= k; + tensors[t]->nb[1] *= k; + tensors[t]->nb[2] = tensors[t]->nb[1] * tensors[t]->ne[1]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } + } + } + + // nrows too small to saturate threads: split ne0 into k rows (k divides ne0) + const size_t row_size_curr = (size_t) t0->ne[0] * elem_size; + const uint32_t nrows = (uint32_t) (t0->ne[1] * t0->ne[2] * t0->ne[3]); + rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->ne[1] != 1 && tensors[t]->nb[1] != row_size_curr) { + rows_contig = false; + break; + } + } + if (nrows < n_hvx && rows_contig && row_size_curr >= 4096) { + const size_t vec_elems = 128 / elem_size; + int64_t k = std::max<int64_t>(row_size_curr / 16384, (n_hvx * 4 + nrows - 1) / nrows); + k = std::min<int64_t>(k, row_size_curr / 2048); + while (k > 1 && t0->ne[0] % (k * vec_elems) != 0) { + k--; + } + if (k > 1) { + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] /= k; + tensors[t]->ne[1] *= k; + tensors[t]->nb[1] = tensors[t]->ne[0] * elem_size; + tensors[t]->nb[2] = tensors[t]->nb[1] * tensors[t]->ne[1]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } + } + } +} + +static bool htp_kparams_solve_unary_impl( + const struct ggml_hexagon_session * sess, + uint32_t op, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (!htp_op_is_unary((htp_op_code) op)) { + return false; + } + + if (op == HTP_OP_HADAMARD) { + const int64_t n = src0->ne[0]; + if (n < 32 || (n & (n - 1)) != 0) { + return false; + } + if (!dst->src[0] || dst->src[0]->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || + (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16)) { + return false; + } + } else { + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || dst->type != src0->type) { + return false; + } + if (op == HTP_OP_TRI && (src0->type != GGML_TYPE_F32 || !ggml_is_contiguous(src0))) { + return false; + } + if (src0->type == GGML_TYPE_F16) { + switch (op) { + case HTP_OP_NORM: + case HTP_OP_RMS_NORM: + case HTP_OP_L2_NORM: + case HTP_OP_SCALE: + case HTP_OP_CLAMP: + case HTP_OP_SQR: + case HTP_OP_SQRT: + case HTP_OP_UNARY_LOG: + case HTP_OP_UNARY_ABS: + case HTP_OP_UNARY_STEP: + case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: + break; + default: + return false; + } + } + } + + if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous(dst) || !ggml_are_same_shape(src0, dst)) { + return false; + } + + const size_t elem_size = ggml_type_size(src0->type); + + const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + kparams->n_hvx = n_hvx; + + const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); + const bool broadcast_weight = has_weight && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); + if (op == HTP_OP_RMS_NORM_MUL_ADD && !broadcast_weight) { + return false; + } + kparams->broadcast_weight = broadcast_weight; + + const bool src0_contig = ggml_is_contiguous_1(src0); + const bool src1_contig = (has_weight && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true; + const uint32_t src0_nrows_per_hvx = (src0_nrows + n_hvx - 1) / n_hvx; + const uint32_t min_block_rows = (uint32_t) (std::max<size_t>(1, 2048 / dst_data_row_size)); + uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1]; + if (src0_nrows_per_hvx >= 2 * min_block_rows) { + max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_hvx / 4)); + } + + kparams->src0_contig = src0_contig ? 1 : 0; + kparams->src1_contig = src1_contig ? 1 : 0; + + struct htp_unary_vtcm_layout L; + uint32_t col_tile = 0; + uint32_t vtcm_row_per_hvx = 0; + + htp_unary_solve_layout(&L, op, src0->ne[0], dst->ne[0], + has_weight ? src1->ne[0] : 0, + max_block, broadcast_weight, n_hvx, vtcm_size, elem_size, + &col_tile, &vtcm_row_per_hvx); + + kparams->col_tile = col_tile; + kparams->vtcm_size = L.total_bytes; + kparams->block = col_tile ? 0 : vtcm_row_per_hvx; + + const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1; + kparams->tiles_per_row = tiles_per_row; + kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); + kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]); + kparams->div_tpr = init_fastdiv_values(tiles_per_row); + + return (col_tile > 0 || vtcm_row_per_hvx > 0) && L.total_bytes <= vtcm_size; +} + +static bool htp_kparams_solve_unary( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_unary_kernel_params * kparams +) { + const htp_op_code opcode = htp_op_remap(op); + const struct ggml_tensor * src0 = (opcode == HTP_OP_HADAMARD) ? op->src[1] : op->src[0]; + const struct ggml_tensor * src1 = (opcode == HTP_OP_HADAMARD) ? nullptr : op->src[1]; + return htp_kparams_solve_unary_impl(sess, opcode, src0, src1, op, kparams); +} + +bool htp_kparams_solve_rms_norm_mul( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +) { + return htp_kparams_solve_unary_impl(sess, HTP_OP_RMS_NORM_MUL, src0, src1, dst, kparams); +} + +bool htp_kparams_solve_rms_norm_mul_add( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +) { + return htp_kparams_solve_unary_impl(sess, HTP_OP_RMS_NORM_MUL_ADD, src0, src1, dst, kparams); +} + +bool htp_kparams_solve_repeat_get_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_get_rows_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 && + src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && + src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K) { + return false; + } + + if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(src1) || !ggml_is_contiguous_rows(dst)) { + return false; + } + + if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) { + return false; + } + + if (src0->type == dst->type) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) { + return false; + } + } else if (src0->type == GGML_TYPE_I32) { + return false; + } else if (dst->type != GGML_TYPE_F32) { + return false; + } + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + + const uint32_t ne10 = src1->ne[0]; + const uint32_t ne11 = src1->ne[1]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t nr = ne10 * ne11 * ne12; + + const bool tiled = ggml_hexagon_tensor_is_tiled(src0); + const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; + + if (tiled) { + kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED; + } else if (src0->type == dst->type) { + kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE; + } else { + kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT; + } + + const uint32_t total_tasks = nr; + const uint32_t min_tasks_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst->nb[1] - 1) / dst->nb[1])); + const uint32_t max_hvx = (std::max)(1u, total_tasks / min_tasks_per_hvx); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + + struct htp_get_rows_vtcm_layout vtcm_layout = {}; + while (kparams->n_hvx > 0) { + htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_hvx); + if (vtcm_layout.total_bytes <= vtcm_size) { + break; + } + --kparams->n_hvx; + } + + if (kparams->n_hvx == 0 && total_tasks > 0) { + htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1); + } + + kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes; + kparams->total_tasks = total_tasks; + + kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0}; + kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0}; + kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0}; + kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0}; + + if (total_tasks == 0) { + return true; + } + return kparams->n_hvx > 0 && (size_t) kparams->vtcm_size <= vtcm_size; +} + +static bool htp_kparams_solve_get_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_get_rows_kernel_params * kparams +) { + return htp_kparams_solve_repeat_get_rows(sess, op->src[0], op->src[1], op, kparams); +} + +static bool htp_kparams_solve_set_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_set_rows_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op; + + if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) { + return false; + } + if (src0->type != GGML_TYPE_F32) { + return false; + } + if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) { + return false; + } + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) { + return false; + } + + const uint32_t nr = src0->ne[1]; + + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), nr); + kparams->total_tasks = nr; + + struct htp_set_rows_vtcm_layout vtcm_layout; + htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_hvx); + kparams->vtcm_size = vtcm_layout.total_bytes; + + return vtcm_layout.total_bytes <= htp_session_vtcm_size(sess); +} + +static bool htp_kparams_solve_softmax( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_softmax_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * src2 = op->src[2]; + const struct ggml_tensor * dst = op; + + if (src2) { + return false; + } + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src0->ne[2] > 512) { + return false; + } + if (src1) { + if ((src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) || + src0->ne[0] != src1->ne[0] || src1->ne[1] < src0->ne[1] || + src0->ne[2] % src1->ne[2] != 0 || src0->ne[3] % src1->ne[3] != 0 || + !ggml_is_contiguous(src1)) { + return false; + } + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + return false; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = op->ne[0] * ggml_type_size(op->type); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + + float scale = 1.0f; + float max_bias = 0.0f; + memcpy(&scale, &op->op_params[0], sizeof(float)); + memcpy(&max_bias, &op->op_params[1], sizeof(float)); + + kparams->scale = scale; + kparams->max_bias = max_bias; + + const uint32_t n_head = src0->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); + kparams->n_head = n_head; + kparams->n_head_log2 = n_head_log2; + + if (max_bias > 0.0f && n_head_log2 > 0) { + kparams->m0 = powf(2.0f, -(max_bias) / n_head_log2); + kparams->m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + } else { + kparams->m0 = 1.0f; + kparams->m1 = 1.0f; + } + + kparams->use_src1 = (src1 != nullptr) ? 1 : 0; + kparams->use_f16 = (src1 != nullptr && src1->type == GGML_TYPE_F16) ? 1 : 0; + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne10 = src1 ? src1->ne[0] : 1; + + struct htp_softmax_vtcm_layout layout; + htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_hvx); + if (layout.total_bytes > htp_session_vtcm_size(sess)) { + return false; + } + + kparams->n_hvx = n_hvx; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + + if (!kparams->use_src1) { + kparams->kernel_type = HTP_SOFTMAX_KERNEL_NOMASK; + } else if (kparams->use_f16) { + kparams->kernel_type = HTP_SOFTMAX_KERNEL_MASK_F16; + } else { + kparams->kernel_type = HTP_SOFTMAX_KERNEL_MASK_F32; + } + + if (src0->ne[1] > 0) kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); + if (src0->ne[2] > 0) kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); + const uint32_t ne12 = src1 ? src1->ne[2] : 1; + const uint32_t ne13 = src1 ? src1->ne[3] : 1; + if (ne12 > 0) kparams->div_ne12 = init_fastdiv_values(ne12); + if (ne13 > 0) kparams->div_ne13 = init_fastdiv_values(ne13); + + return true; +} + +static bool htp_kparams_solve_rope( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_rope_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * src2 = op->src[2]; + const struct ggml_tensor * dst = op; + + if (!ggml_rope_get_truncate(op)) { + return false; + } + + if (!ggml_are_same_shape(src0, dst)) { + return false; + } + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) { + return false; + } + if (src0->ne[0] <= 0) { + return false; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + if (src0_nrows == 0) { + return false; + } + + const int32_t * op_params = &op->op_params[0]; + const int n_dims = op_params[1]; + const int mode = op_params[2]; + const int n_offs = op_params[15]; + + if (n_dims < 0 || n_dims % 2 != 0) { + return false; + } + if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) { + return false; + } + + float freq_base; + memcpy(&freq_base, op_params + 5, sizeof(float)); + if (freq_base < 0.0f) { + return false; + } + + if (mode != GGML_ROPE_TYPE_NORMAL && + mode != GGML_ROPE_TYPE_NEOX && + mode != GGML_ROPE_TYPE_MROPE && + mode != GGML_ROPE_TYPE_VISION && + mode != GGML_ROPE_TYPE_IMROPE) { + return false; + } + + const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0; + + if (mode == GGML_ROPE_TYPE_VISION && (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0)) { + return false; + } + if (is_mrope) { + const int32_t * sections = op_params + 11; + if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) { + return false; + } + } + + const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2]; + if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) { + return false; + } + + if (src2) { + if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2) || src2->ne[0] < (n_dims / 2)) { + return false; + } + } + + if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || + src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) { + return false; + } + + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), src0_nrows); + const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0; + + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_hvx, n_freq_factors); + if (layout.total_bytes > htp_session_vtcm_size(sess)) { + return false; + } + + kparams->n_hvx = n_hvx; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); + kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); + + return true; +} + +static bool htp_kparams_solve_sum_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_sum_rows_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32) { + return false; + } + if (op->op == GGML_OP_ARGMAX) { + if (dst->type != GGML_TYPE_I32) { + return false; + } + } else if (dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + return false; + } + + const uint32_t ne00 = src0->ne[0]; + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); + + if (op->op == GGML_OP_SUM) { + const uint32_t total_elems = (uint32_t) ggml_nelements(src0); + const uint32_t n_hvx = (total_elems >= 1024) ? max_hvx_sess : 1; + const uint32_t raw_chunk = (total_elems + n_hvx - 1) / n_hvx; + + kparams->n_hvx = n_hvx; + kparams->elems_per_hvx = hex_round_up(raw_chunk, 32); + } else if (op->op == GGML_OP_ARGMAX && src0_nrows <= 1) { + const uint32_t n_hvx = (ne00 >= 1024) ? max_hvx_sess : 1; + const uint32_t raw_chunk = (ne00 + n_hvx - 1) / n_hvx; + + kparams->n_hvx = n_hvx; + kparams->elems_per_hvx = hex_round_up(raw_chunk, 32); + } else { + const uint32_t min_rpt = (src0->nb[1] < 1024) ? 32 : 1; + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rpt); + + kparams->n_hvx = (std::min)(max_hvx_sess, max_hvx); + } + + return true; +} + +bool htp_kparams_solve_unary_mul_glu( + const struct ggml_hexagon_session * sess, + uint32_t op, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_glu_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (!htp_op_is_glu((htp_op_code) op)) { + return false; + } + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || dst->type != src0->type) { + return false; + } + if ((op == HTP_OP_GLU_GEGLU_QUICK || op == HTP_OP_GLU_GEGLU_ERF) && src0->type != GGML_TYPE_F32) { + return false; + } + + const size_t elem_size = ggml_type_size(src0->type); + const auto is_valid_src = [elem_size](const struct ggml_tensor * t) { + return ggml_is_contiguous_1(t) && t->nb[0] == elem_size && + (t->ne[1] <= 1 || t->nb[1] >= (size_t) t->ne[0] * elem_size); + }; + + if (!is_valid_src(src0) || !ggml_is_contiguous(dst)) { + return false; + } + if (src1 && (src1->type != src0->type || !ggml_are_same_shape(src0, src1) || !is_valid_src(src1))) { + return false; + } + + const size_t dst_row_size = dst->ne[0] * elem_size; + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const size_t row_size_aligned = hex_round_up(dst_row_size, 128); + uint32_t vtcm_row_per_hvx = (uint32_t) (vtcm_size / (n_hvx * 3 * row_size_aligned)); + if (vtcm_row_per_hvx < 2) { + return false; + } + + const uint32_t src0_nrows_per_hvx = (src0_nrows + n_hvx - 1) / n_hvx; + if (src0_nrows_per_hvx >= 2 && vtcm_row_per_hvx > src0_nrows_per_hvx) { + uint32_t target_rpt = (src0_nrows_per_hvx >= 4) ? (src0_nrows_per_hvx / 2) & ~1u : 2; + vtcm_row_per_hvx = (std::max)(2u, (std::min)(vtcm_row_per_hvx, target_rpt)); + } + + struct htp_glu_vtcm_layout L; + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, vtcm_row_per_hvx); + + kparams->n_hvx = n_hvx; + kparams->vtcm_row_per_hvx = L.vtcm_row_per_hvx; + kparams->vtcm_size = (uint32_t) L.total_bytes; + + return true; +} + +static bool htp_kparams_solve_glu( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_glu_kernel_params * kparams +) { + return htp_kparams_solve_unary_mul_glu(sess, htp_op_remap(op), op->src[0], op->src[1], op, kparams); +} + +static bool htp_kparams_solve_repeat( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_repeat_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || src0->type != dst->type) { + return false; + } + if (!ggml_can_repeat(src0, dst) || ggml_is_transposed(src0) || ggml_is_transposed(dst)) { + return false; + } + + const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const size_t dst_row_size = dst->ne[0] * ggml_type_size(dst->type); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, total_dst_rows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + + kparams->n_hvx = n_hvx; + kparams->nr0 = dst->ne[0] / src0->ne[0]; + kparams->nr1 = dst->ne[1] / src0->ne[1]; + kparams->nr2 = dst->ne[2] / src0->ne[2]; + kparams->nr3 = dst->ne[3] / src0->ne[3]; + + kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); + kparams->div_ne12 = init_fastdiv_values(dst->ne[1] * dst->ne[2]); + kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); + kparams->div_ne03 = init_fastdiv_values(src0->ne[3]); + + return true; +} + +static bool htp_kparams_solve_ssm_conv( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_ssm_conv_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) { + return false; + } + + const int64_t d_conv_64 = src1->ne[0]; + const int64_t d_inner_64 = src0->ne[1]; + const int64_t n_t_64 = dst->ne[1]; + const int64_t n_s_64 = dst->ne[2]; + + if (d_conv_64 <= 0 || d_conv_64 > 32 || d_inner_64 <= 0) { + return false; + } + if (src0->ne[0] != d_conv_64 - 1 + n_t_64 || src0->ne[1] != d_inner_64 || src0->ne[2] != n_s_64) { + return false; + } + if (src1->ne[0] != d_conv_64 || src1->ne[1] != d_inner_64) { + return false; + } + if (dst->ne[0] != d_inner_64 || dst->ne[1] != n_t_64 || dst->ne[2] != n_s_64) { + return false; + } + if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (src0->nb[1] != src0->ne[0] * sizeof(float) || src1->nb[1] != src1->ne[0] * sizeof(float)) { + return false; + } + + const uint32_t d_conv = (uint32_t) d_conv_64; + const uint32_t d_inner = (uint32_t) d_inner_64; + const uint32_t n_t = (uint32_t) n_t_64; + const uint32_t n_s = (uint32_t) n_s_64; + const uint32_t ncs = (uint32_t) src0->ne[0]; + + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), (d_inner + 31) / 32); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + kparams->n_hvx = n_hvx; + kparams->d_conv = d_conv; + kparams->d_inner = d_inner; + kparams->n_t = n_t; + kparams->n_s = n_s; + + const uint32_t raw_rpt = (d_inner + n_hvx - 1) / n_hvx; + const uint32_t d_inner_per_hvx = hex_round_up(raw_rpt, 32); + + if (n_t == 1) { + kparams->d_inner_tile = d_inner_per_hvx; + } else { + const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_hvx * d_conv * sizeof(float), 128) + 128; + const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_hvx * sizeof(float), 128); + const uint32_t vtcm_src1_per_hvx = src1_raw_bytes + src1_T_bytes; + + const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_hvx : (1024 * 1024)); + const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128); + const size_t fixed_bytes = vtcm_src1_per_hvx + src0_block_T; + const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024); + + uint32_t target_max_tile = hex_round_up((d_inner_per_hvx + 3) / 4, 32); + target_max_tile = (std::max)(target_max_tile, 32u); + target_max_tile = (std::min)(target_max_tile, 128u); + + const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float); + uint32_t d_inner_tile = (uint32_t) (avail_for_tiles / (2 * (raw_stride_bytes + n_t * sizeof(float)))); + d_inner_tile = (d_inner_tile / 32) * 32; + if (d_inner_tile == 0) { + d_inner_tile = 32; + } + if (d_inner_tile > target_max_tile) { + d_inner_tile = target_max_tile; + } + if (d_inner_tile > d_inner_per_hvx) { + d_inner_tile = d_inner_per_hvx; + } + kparams->d_inner_tile = d_inner_tile; + } + + struct htp_ssm_conv_vtcm_layout L; + htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_hvx, d_conv, n_t, kparams->d_inner_tile, n_hvx); + + kparams->vtcm_size = (uint32_t) L.total_bytes; + kparams->div_d_inner_tile = init_fastdiv_values(kparams->d_inner_tile); + + return L.total_bytes <= vtcm_size; +} + +static bool htp_kparams_solve_sort( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_sort_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + const bool is_top_k = (op->op == GGML_OP_TOP_K); + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) { + return false; + } + + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t ne00 = src0->ne[0]; + const uint32_t k = dst->ne[0]; + const uint32_t n_hvx_max = htp_session_n_hvx(sess) > 0 ? htp_session_n_hvx(sess) : 4; + const size_t vtcm_budget = htp_session_vtcm_size(sess) > 0 ? htp_session_vtcm_size(sess) : (8 * 1024 * 1024); + + int32_t order = GGML_SORT_ORDER_DESC; + if (!is_top_k) { + order = ((const int32_t *) op->op_params)[0]; + } + + struct htp_sort_vtcm_layout layout; + if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_hvx_max, vtcm_budget, is_top_k)) { + return false; + } + + kparams->n_hvx = layout.n_hvx; + kparams->order = (uint32_t) order; + kparams->is_top_k = is_top_k ? 1 : 0; + kparams->chunk_elems = layout.chunk_elems; + kparams->n_chunks = layout.n_chunks; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->merge_elems = layout.merge_elems; + kparams->n_slots = layout.n_slots; + kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->div_ne01_ne02 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[2])); + + return true; +} + +static void htp_pool_interior_range( + uint32_t src_x, uint32_t dst_x, uint32_t kernel_x, uint32_t stride_x, int32_t pad_x, + uint32_t * ox_lo, uint32_t * ox_hi +) { + const uint32_t lo_raw = ((uint32_t) pad_x + stride_x - 1) / stride_x; + *ox_lo = (lo_raw < dst_x) ? lo_raw : dst_x; + + const int32_t numer_hi = (int32_t) src_x - (int32_t) kernel_x + pad_x; + if (numer_hi < 0) { + *ox_hi = 0; + } else { + const uint32_t hi_raw = (uint32_t) numer_hi / stride_x + 1; + *ox_hi = (hi_raw < dst_x) ? hi_raw : dst_x; + } + if (*ox_hi < *ox_lo) { + *ox_hi = *ox_lo; + } +} + +static bool htp_kparams_solve_pool_2d( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_pool_2d_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + const bool is_pool_1d = (op->op == GGML_OP_POOL_1D); + + const int32_t * params = dst->op_params; + if (params[0] != GGML_OP_POOL_AVG && params[0] != GGML_OP_POOL_MAX) { + return false; + } + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + return false; + } + if (ggml_nbytes(src0) > UINT32_MAX - 256 || ggml_nbytes(dst) > UINT32_MAX - 256) { + return false; + } + + const int32_t kernel_x = params[1]; + const int32_t kernel_y = is_pool_1d ? 1 : params[2]; + const int32_t stride_x = params[is_pool_1d ? 2 : 3]; + const int32_t stride_y = is_pool_1d ? 1 : params[4]; + const int32_t pad_x = params[is_pool_1d ? 3 : 5]; + const int32_t pad_y = is_pool_1d ? 0 : params[6]; + + if (kernel_x <= 0 || kernel_y <= 0 || stride_x <= 0 || stride_y <= 0 || + pad_x < 0 || pad_y < 0) { + return false; + } + + const uint32_t src_x = (uint32_t) src0->ne[0]; + const uint32_t src_y = is_pool_1d ? 1 : (uint32_t) src0->ne[1]; + const uint32_t dst_x = (uint32_t) dst->ne[0]; + const uint32_t dst_y = is_pool_1d ? 1 : (uint32_t) dst->ne[1]; + const uint32_t dst_plane_bytes = dst_x * dst_y * sizeof(float); + const uint32_t planes = (uint32_t) (is_pool_1d + ? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3] + : (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), planes > 0 ? planes : 1); + const uint32_t planes_per_chunk = (dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1; + + struct htp_pool_vtcm_layout layout; + if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_hvx, htp_session_vtcm_size(sess))) { + return false; + } + + kparams->src_x = src_x; + kparams->src_y = src_y; + kparams->dst_x = dst_x; + kparams->dst_y = dst_y; + kparams->kernel_x = (uint32_t) kernel_x; + kparams->kernel_y = (uint32_t) kernel_y; + kparams->stride_x = (uint32_t) stride_x; + kparams->stride_y = (uint32_t) stride_y; + kparams->pad_x = pad_x; + kparams->pad_y = pad_y; + kparams->n_hvx = n_hvx; + kparams->planes = planes; + kparams->planes_per_chunk = planes_per_chunk; + kparams->pool_op = (uint32_t) params[0]; + + const bool exact_tiling = (kparams->pad_x == 0 && kparams->pad_y == 0 && + kparams->stride_x == kparams->kernel_x && kparams->stride_y == kparams->kernel_y); + const bool fast_ok = exact_tiling && (kparams->kernel_x == 1 || kparams->kernel_x == 2); + const bool narrow_ok = fast_ok && ((uint64_t) kparams->dst_x * kparams->kernel_x <= 32); + const bool global_ok = exact_tiling && + kparams->kernel_x == kparams->src_x && kparams->kernel_y == kparams->src_y && + kparams->dst_x == 1 && kparams->dst_y == 1; + const bool block_ok = exact_tiling && + kparams->kernel_y == 1 && kparams->stride_y == 1 && + kparams->kernel_x >= 32; + + if (global_ok) { + kparams->kernel_type = HTP_POOL_KERNEL_GLOBAL; + } else if (block_ok) { + kparams->kernel_type = HTP_POOL_KERNEL_BLOCK; + } else if (narrow_ok) { + kparams->kernel_type = HTP_POOL_KERNEL_HVX_NARROW; + } else if (fast_ok) { + kparams->kernel_type = HTP_POOL_KERNEL_HVX; + } else { + kparams->kernel_type = HTP_POOL_KERNEL_GENERAL; + } + + kparams->avg_divide_count = (is_pool_1d && kparams->pool_op == GGML_OP_POOL_AVG) ? 1 : 0; + htp_pool_interior_range(kparams->src_x, kparams->dst_x, kparams->kernel_x, + kparams->stride_x, kparams->pad_x, + &kparams->ox_lo, &kparams->ox_hi); + kparams->inv_kernel_area = 1.0f / (float) (kparams->kernel_x * kparams->kernel_y); + + return true; +} + +bool htp_kparams_solve_upscale( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * dst, + struct htp_upscale_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + const uint32_t src_x = (uint32_t) src0->ne[0]; + const uint32_t src_y = (uint32_t) src0->ne[1]; + const uint32_t src_z = (uint32_t) src0->ne[2]; + const uint32_t src_w = (uint32_t) src0->ne[3]; + + const uint32_t dst_x = (uint32_t) dst->ne[0]; + const uint32_t dst_y = (uint32_t) dst->ne[1]; + const uint32_t dst_z = (uint32_t) dst->ne[2]; + const uint32_t dst_w = (uint32_t) dst->ne[3]; + + if (src_x == 0 || src_y == 0 || src_z == 0 || src_w == 0 || + dst_x == 0 || dst_y == 0 || dst_z == 0 || dst_w == 0) { + return false; + } + + const bool is_src_contig_row = (src0->nb[0] == sizeof(float)); + const bool is_src_strided_2d = (src0->nb[0] > sizeof(float)) && + (src0->nb[0] % sizeof(float) == 0) && + (src0->nb[1] == (size_t) src_x * src0->nb[0]) && + ((size_t) src_x * src_y <= 0xFFFFu); + + if ((!is_src_contig_row && !is_src_strided_2d) || dst->nb[0] != sizeof(float)) { + return false; + } + + const int32_t mode_flags = dst->op_params[0]; + const uint32_t mode_raw = (uint32_t) (mode_flags & 0xFF); + if (mode_raw >= GGML_SCALE_MODE_COUNT) { + return false; + } + + const bool align_corners = (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) != 0; + const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS) != 0; + + float sf0 = (float) dst_x / (float) src_x; + float sf1 = (float) dst_y / (float) src_y; + float sf2 = (float) dst_z / (float) src_z; + float sf3 = (float) dst_w / (float) src_w; + float pixel_offset = 0.5f; + + if (align_corners) { + pixel_offset = 0.0f; + sf0 = (dst_x > 1 && src_x > 1) ? (float) (dst_x - 1) / (float) (src_x - 1) : sf0; + sf1 = (dst_y > 1 && src_y > 1) ? (float) (dst_y - 1) / (float) (src_y - 1) : sf1; + } + + const float inv_sf0 = 1.0f / sf0; + const float inv_sf1 = 1.0f / sf1; + const float inv_sf2 = 1.0f / sf2; + const float inv_sf3 = 1.0f / sf3; + + const float filter_radius0 = antialias ? (std::max)(1.0f, inv_sf0) : 0.0f; + const float filter_radius1 = antialias ? (std::max)(1.0f, inv_sf1) : 0.0f; + const float invscale0 = antialias ? (1.0f / filter_radius0) : 0.0f; + const float invscale1 = antialias ? (1.0f / filter_radius1) : 0.0f; + + const uint32_t total_slices = dst_z * dst_w; + uint32_t n_hvx = htp_session_n_hvx(sess); + + struct htp_upscale_vtcm_layout layout; + uint32_t dst_nrows_per_chunk = 0; + while (n_hvx > 0) { + if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_hvx, + htp_session_vtcm_size(sess), &dst_nrows_per_chunk)) { + break; + } + n_hvx--; + } + if (n_hvx == 0) { + return false; + } + + size_t lut_x_size = 0; + size_t elem_y_sz = sizeof(uint32_t); + if (mode_raw == HTP_UPSCALE_MODE_BICUBIC) { + lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_bicubic_x), 128); + elem_y_sz = sizeof(struct upscale_bicubic_y); + } else if (antialias) { + lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_antialias_x), 128); + elem_y_sz = sizeof(struct upscale_antialias_y); + } else if (mode_raw == HTP_UPSCALE_MODE_BILINEAR) { + const uint32_t n_chunks = (dst_x + 31) / 32; + lut_x_size = hex_round_up(n_chunks * sizeof(struct upscale_bilinear_x_chunk), 128); + elem_y_sz = sizeof(struct upscale_bilinear_y); + } else { + lut_x_size = hex_round_up(dst_x * sizeof(uint32_t), 128); + } + const size_t total_lut_size = lut_x_size + hex_round_up(dst_y * elem_y_sz, 128); + if (total_lut_size > HTP_DDR_SPAD_SIZE) { + return false; + } + + const uint32_t chunks_per_slice = (dst_y + dst_nrows_per_chunk - 1) / dst_nrows_per_chunk; + const uint32_t total_chunks = total_slices * chunks_per_slice; + + n_hvx = (std::min)(n_hvx, total_chunks > 0 ? total_chunks : 1); + htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_hvx); + + const uint32_t slice_dst_bytes = dst_x * dst_y * sizeof(float); + const uint32_t chunk_dst_bytes = dst_x * dst_nrows_per_chunk * sizeof(float); + const bool dst_contig = (dst->nb[1] == (size_t) dst_x * sizeof(float)) && + (dst_z <= 1 || dst->nb[2] == (size_t) dst_y * dst->nb[1]) && + (dst_w <= 1 || dst->nb[3] == (size_t) dst_z * dst->nb[2]); + + uint32_t chunks_per_group = 0; + if (dst_contig) { + const uint32_t slices_per_group = (slice_dst_bytes > 0) + ? (HEX_L2_LINE_SIZE / hex_gcd_u32(slice_dst_bytes, HEX_L2_LINE_SIZE)) + : 1; + if (slices_per_group == 1 && (chunk_dst_bytes % HEX_L2_LINE_SIZE == 0)) { + chunks_per_group = 1; + } else { + chunks_per_group = slices_per_group * chunks_per_slice; + } + } + + uint8_t kernel_type = HTP_UPSCALE_KERNEL_BILINEAR; + if (mode_raw == HTP_UPSCALE_MODE_NEAREST) { + if (dst_x == 2 * src_x && dst_y == 2 * src_y && !align_corners) { + kernel_type = HTP_UPSCALE_KERNEL_NEAREST_2X; + } else if (dst_x == 4 * src_x && dst_y == 4 * src_y && !align_corners) { + kernel_type = HTP_UPSCALE_KERNEL_NEAREST_4X; + } else { + kernel_type = HTP_UPSCALE_KERNEL_NEAREST; + } + } else if (mode_raw == HTP_UPSCALE_MODE_BICUBIC) { + kernel_type = HTP_UPSCALE_KERNEL_BICUBIC; + } else if (antialias) { + kernel_type = HTP_UPSCALE_KERNEL_BILINEAR_AA; + } else if (dst_x == 2 * src_x && dst_y == 2 * src_y && !align_corners && (src_x % 32 == 0)) { + kernel_type = HTP_UPSCALE_KERNEL_BILINEAR_2X; + } else { + kernel_type = HTP_UPSCALE_KERNEL_BILINEAR; + } + + kparams->kernel_type = kernel_type; + kparams->mode = (uint8_t) mode_raw; + kparams->align_corners = align_corners ? 1 : 0; + kparams->antialias = antialias ? 1 : 0; + + kparams->inv_sf0 = inv_sf0; + kparams->inv_sf1 = inv_sf1; + kparams->inv_sf2 = inv_sf2; + kparams->inv_sf3 = inv_sf3; + kparams->pixel_offset = pixel_offset; + + kparams->filter_radius0 = filter_radius0; + kparams->filter_radius1 = filter_radius1; + kparams->invscale0 = invscale0; + kparams->invscale1 = invscale1; + + kparams->n_hvx = n_hvx; + kparams->chunks_per_group = chunks_per_group; + + kparams->dst_nrows_per_chunk = dst_nrows_per_chunk; + kparams->chunks_per_slice = chunks_per_slice; + kparams->total_chunks = total_chunks; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + + kparams->div_chunks_per_slice = init_fastdiv_values(chunks_per_slice > 0 ? chunks_per_slice : 1); + kparams->div_dst_z = init_fastdiv_values(dst_z > 0 ? dst_z : 1); + + return true; +} + +static bool htp_kparams_solve_concat( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_concat_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->kernel_type = HTP_CONCAT_KERNEL_UNSUPPORTED; + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (!src0 || !src1 || !dst) { + return false; + } + + int dim = ((const int32_t *) op->op_params)[0]; + if (dim < 0 || dim >= GGML_MAX_DIMS) { + return false; + } + kparams->dim = dim; + + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) { + return false; + } + if (src0->type != dst->type || src1->type != dst->type) { + return false; + } + + const uint32_t type_size = ggml_type_size(dst->type); + const uint32_t n_hvx = htp_session_n_hvx(sess); + const uint32_t mdev_count = htp_session_mdev_count(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + for (int d = 0; d < GGML_MAX_DIMS; d++) { + const int64_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d]; + if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) { + return false; + } + } + + const uint32_t max_gather_ne0 = 128 / type_size; + const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + const bool b_dense = (src1->nb[0] == type_size); + const bool gather_strides_ok = + src0->nb[1] >= (size_t) src0->ne[0] * type_size && + (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size + : src1->nb[0] >= (size_t) src1->ne[1] * type_size); + + if (dim == 0 && + (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) && + dst->ne[0] <= max_gather_ne0 && + dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size && + src0->nb[0] == type_size && + (src1->nb[0] == type_size || src1->nb[1] == type_size) && + gather_strides_ok && + htp_gather_rows_tensor_outer_contiguous(dst) && + htp_gather_rows_tensor_outer_contiguous(src0) && + htp_gather_rows_tensor_outer_contiguous(src1)) { + + if (htp_gather_rows_solve_layout(&kparams->u.gather, + (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], + (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense, + type_size, total_rows, + n_hvx, mdev_count, vtcm_size)) { + kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = kparams->u.gather.slice * n_hvx; + return true; + } + } + + const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size); + if (dma_strides_ok) { + kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR; + kparams->n_hvx = 1; + return true; + } + + const bool is_src1_transposed = (src1->nb[0] > src1->nb[1]); + const bool is_src0_transposed = (src0->nb[0] > src0->nb[1]); + const bool transposed_rows_ok = (src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size); + + if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok && + (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) { + + const uint32_t nth = n_hvx > 0 ? n_hvx : 8; + struct htp_concat_transposed_vtcm_layout layout; + htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, nth); + + if (vtcm_size > 0 && layout.total_bytes > vtcm_size) { + return false; + } + + kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED; + kparams->n_hvx = nth; + kparams->vtcm_size = layout.total_bytes; + kparams->u.transposed.div_ne2 = init_fastdiv_values((uint32_t) dst->ne[2]); + return true; + } + + return false; +} + +// Permuted copy with strided dim 0 (e.g. channel-first <-> channel-last) +static bool htp_kparams_solve_cpy_transpose( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * dst, + uint32_t type_size, + struct htp_copy_kernel_params * kparams) { + + // View contiguous dst with src shape to support CPY with reshape + const bool dst_cont = ggml_is_contiguous(dst); + if (!dst_cont && !ggml_are_same_shape(src0, dst)) { + return false; + } + + // Drop size-1 dims and merge dims adjacent in both tensors + size_t ne[GGML_MAX_DIMS]; + size_t nbs[GGML_MAX_DIMS]; + size_t nbd[GGML_MAX_DIMS]; + size_t dst_stride = type_size; + int n = 0; + for (int d = 0; d < GGML_MAX_DIMS; d++) { + const size_t sd = dst_cont ? dst_stride : dst->nb[d]; + dst_stride *= (size_t) src0->ne[d]; + if (src0->ne[d] == 1) { + continue; + } + if (n > 0 && src0->nb[d] == nbs[n - 1] * ne[n - 1] && sd == nbd[n - 1] * ne[n - 1]) { + ne[n - 1] *= (size_t) src0->ne[d]; + continue; + } + ne[n] = (size_t) src0->ne[d]; + nbs[n] = src0->nb[d]; + nbd[n] = sd; + n++; + } + + // Fold leading dense dim into element block + size_t es = type_size; + for (int i = 0; i < n; i++) { + if (nbs[i] == type_size && nbd[i] == type_size) { + es *= ne[i]; + for (int j = i; j + 1 < n; j++) { + ne[j] = ne[j + 1]; + nbs[j] = nbs[j + 1]; + nbd[j] = nbd[j + 1]; + } + n--; + break; + } + } + if (es > 128 || (es & (es - 1)) != 0) { + return false; + } + + // Transpose axes: dim a dense in dst, dim k dense in src + int a = -1; + int k = -1; + for (int i = 0; i < n; i++) { + if (a < 0 && nbd[i] == es) { + a = i; + } + if (k < 0 && nbs[i] == es) { + k = i; + } + } + if (a < 0 || k < 0 || a == k) { + return false; + } + + struct htp_copy_transpose_params tp = {}; + tp.elem_size = (uint8_t) es; + tp.ne[0] = (uint32_t) ne[a]; + tp.ne[1] = (uint32_t) ne[k]; + tp.ne[2] = 1; + tp.src_nb[0] = (uint32_t) nbs[a]; + tp.dst_nb[0] = (uint32_t) nbd[k]; + tp.packed = nbd[k] == ne[a] * es && ne[a] * es < 128 && 128 % (ne[a] * es) == 0; + + size_t ne3 = 1; + int m = 0; + for (int i = 0; i < n; i++) { + if (i == a || i == k) { + continue; + } + if (m == 0) { + tp.ne[2] = (uint32_t) ne[i]; + } else { + ne3 = ne[i]; + } + tp.src_nb[1 + m] = (uint32_t) nbs[i]; + tp.dst_nb[1 + m] = (uint32_t) nbd[i]; + m++; + } + + const uint32_t n_hvx = htp_session_n_hvx(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + uint64_t vtcm = 0; + for (uint32_t blk_bytes = HTP_COPY_TRANSPOSE_BLK_BYTES; ; blk_bytes /= 2) { + if (!htp_copy_transpose_layout_build(&tp, blk_bytes)) { + return false; + } + vtcm = (uint64_t) n_hvx * 2 * (tp.in_buf_size + tp.out_buf_size); + if (vtcm <= vtcm_size) { + break; + } + } + + const uint64_t total_tiles = (uint64_t) tp.n0b * tp.nkb * tp.ne[2] * ne3; + if (total_tiles > UINT32_MAX) { + return false; + } + tp.total_tiles = (uint32_t) total_tiles; + tp.div_nkb = init_fastdiv_values(tp.nkb); + tp.div_n0b = init_fastdiv_values(tp.n0b); + tp.div_ne2 = init_fastdiv_values(tp.ne[2]); + + kparams->kernel_type = HTP_COPY_KERNEL_TRANSPOSE; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = (uint32_t) vtcm; + kparams->u.transpose = tp; + return true; +} + +static bool htp_kparams_solve_cpy( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_copy_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->kernel_type = HTP_COPY_KERNEL_UNSUPPORTED; + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (!src0 || !dst) { + return false; + } + + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_I32) { + return false; + } + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) { + return false; + } + + const int64_t nelem_src = ggml_nelements(src0); + const int64_t nelem_dst = ggml_nelements(dst); + if (nelem_src != nelem_dst || nelem_src < 0) { + return false; + } + + const uint32_t src_type_size = ggml_type_size(src0->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + kparams->src0_type_size = (uint8_t) src_type_size; + kparams->dst_type_size = (uint8_t) dst_type_size; + kparams->total_elems = (uint32_t) nelem_src; + + if (nelem_src == 0) { + kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG; + return true; + } + + if (nelem_src == 1) { + if (src0->type == dst->type) { + kparams->kernel_type = HTP_COPY_KERNEL_SCALAR; + return true; + } + if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) || + (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) || + (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) || + (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32)) { + kparams->kernel_type = HTP_COPY_KERNEL_SCALAR; + return true; + } + return false; + } + + const bool sametype = (src0->type == dst->type); + const bool same_extents = ggml_are_same_shape(src0, dst); + + const bool transposed = (src0->nb[0] > src0->nb[1]) || (dst->nb[0] > dst->nb[1]) || + (src0->nb[0] != src_type_size) || (dst->nb[0] != dst_type_size) || + (src0->nb[1] < (size_t) src0->ne[0] * src_type_size) || (dst->nb[1] < (size_t) dst->ne[0] * dst_type_size); + const bool sameshape = same_extents && !transposed; + + const bool src_is_contiguous = ggml_is_contiguous(src0); + const bool dst_is_contiguous = ggml_is_contiguous(dst); + + const uint32_t n_hvx = htp_session_n_hvx(sess); + const uint32_t mdev_count = htp_session_mdev_count(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + if (sametype) { + if (src_is_contiguous && dst_is_contiguous) { + kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG; + return true; + } + + const uint32_t max_gather_ne0 = 128 / src_type_size; + const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && + dst_is_contiguous && !src_is_contiguous && + src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 && + src0->nb[1] >= (size_t) src0->ne[0] * src_type_size && + total_rows > 1 && + htp_gather_rows_tensor_outer_contiguous(src0)) { + + if (htp_gather_rows_solve_layout(&kparams->u.gather, + (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], + 0, 0, false, + src_type_size, total_rows, + n_hvx, mdev_count, vtcm_size)) { + kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = kparams->u.gather.slice * n_hvx; + return true; + } + } + + if (htp_kparams_solve_cpy_transpose(sess, src0, dst, src_type_size, kparams)) { + return true; + } + + if (sameshape) { + kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE; + kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + return true; + } + + kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE; + kparams->n_hvx = n_hvx > 0 ? (uint8_t) n_hvx : 4; + kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]); + kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0])); + kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0])); + kparams->u.reshape.div_ne00 = init_fastdiv_values((uint32_t) src0->ne[0]); + kparams->u.reshape.div_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[0])); + kparams->u.reshape.div_ne02_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1] * src0->ne[0])); + return true; + } + + if (!sameshape) { + return false; + } + + const bool valid_conversion = (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) || + (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) || + (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) || + (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32); + if (!valid_conversion) { + return false; + } + + const uint32_t nth = n_hvx > 0 ? n_hvx : 4; + struct htp_copy_convert_vtcm_layout layout; + htp_copy_convert_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, nth); + + if (vtcm_size > 0 && layout.total_bytes > vtcm_size) { + return false; + } + + kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT; + kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + kparams->n_hvx = (uint8_t) nth; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->u.convert.blk_rows = layout.blk_rows; + kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1])); + + return true; +} + +static bool im2col_3d_calc_output_size( + uint32_t input, uint32_t kernel, int32_t stride, int32_t padding, int32_t dilation, uint32_t * output +) { + if (stride <= 0 || padding < 0 || dilation <= 0 || kernel == 0) { + return false; + } + + const int64_t numerator = (int64_t) input + 2LL * padding - (int64_t) dilation * (kernel - 1) - 1; + if (numerator < 0) { + return false; + } + + const uint64_t size = (uint64_t) (numerator / stride) + 1; + if (size == 0 || size > UINT32_MAX) { + return false; + } + + *output = (uint32_t) size; + return true; +} + +static bool htp_kparams_solve_im2col( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_im2col_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED; + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (!src0 || !src1 || !dst) { + return false; + } + if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) { + return false; + } + if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) { + return false; + } + + const int32_t * op_params = (const int32_t *) op->op_params; + const int32_t s0 = op_params[0]; + const int32_t s1 = op_params[1]; + const int32_t p0 = op_params[2]; + const int32_t p1 = op_params[3]; + const int32_t d0 = op_params[4]; + const int32_t d1 = op_params[5]; + const bool is_2D = (op_params[6] == 1); + + if (s0 <= 0 || d0 <= 0 || p0 < 0) { + return false; + } + if (is_2D && (s1 <= 0 || d1 <= 0 || p1 < 0)) { + return false; + } + + const uint32_t N = is_2D ? (uint32_t) src1->ne[3] : (uint32_t) src1->ne[2]; + const uint32_t IC = is_2D ? (uint32_t) src1->ne[2] : (uint32_t) src1->ne[1]; + const uint32_t IH = is_2D ? (uint32_t) src1->ne[1] : 1; + const uint32_t IW = (uint32_t) src1->ne[0]; + const uint32_t KH = is_2D ? (uint32_t) src0->ne[1] : 1; + const uint32_t KW = (uint32_t) src0->ne[0]; + const uint32_t OH = is_2D ? (uint32_t) dst->ne[2] : 1; + const uint32_t OW = (uint32_t) dst->ne[1]; + + if (KW == 0 || KH == 0 || IW == 0 || IH == 0 || IC == 0 || OW == 0 || OH == 0 || N == 0) { + return false; + } + + const uint64_t patch_stride_64 = (uint64_t) IC * KH * KW; + const uint64_t total_rows_64 = (uint64_t) N * OH; + if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX) { + return false; + } + const uint32_t patch_stride = (uint32_t) patch_stride_64; + const uint32_t total_rows = (uint32_t) total_rows_64; + + if (dst->ne[0] != patch_stride || dst->ne[1] != OW || (is_2D && dst->ne[2] != OH)) { + return false; + } + + const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); + + bool is_patchembed = ggml_is_contiguous(src1) && (s0 == (int32_t) KW && p0 == 0 && d0 == 1); + if (is_2D) { + is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1); + } + + const uint32_t max_hvx = (sess && htp_session_n_hvx(sess) > 0) ? htp_session_n_hvx(sess) : 8; + const uint32_t n_hvx = (total_rows < max_hvx) ? total_rows : max_hvx; + const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0; + const uint32_t row_size = (uint32_t) dst->nb[2]; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + + if (is_patchembed) { + const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * IW * sizeof(float); + const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem; + if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) { + const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256); + const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, + htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_PATCHEMBED_DMA, KH, patch_stride)); + + if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { + kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->src_row_bytes = src_row_bytes; + kparams->dst_row_bytes = dst_row_bytes; + kparams->owb = OW; + kparams->wb = IW; + kparams->nbpr = 1; + kparams->N = N; + kparams->IC = IC; + kparams->ID = 1; + kparams->IH = IH; + kparams->IW = IW; + kparams->KD = 1; + kparams->KH = KH; + kparams->KW = KW; + kparams->OD = 1; + kparams->OH = OH; + kparams->OW = OW; + kparams->patch_stride = patch_stride; + kparams->total_rows = total_rows; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_OH = init_fastdiv_values(OH); + kparams->div_OD = init_fastdiv_values(1); + return true; + } + } + } + + for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) { + const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1; + if (Wb > UINT32_MAX) { + if (owb == 1) break; + continue; + } + + const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * Wb * sizeof(float); + const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem; + if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) { + if (owb == 1) break; + continue; + } + + const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256); + const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); + + // threads work on output-col blocks, so a single output row (1D) can use them all + const uint32_t nbpr = (OW + owb - 1) / owb; + const uint64_t nblocks = (uint64_t) total_rows * nbpr; + const uint32_t blk_threads = (uint32_t) (nblocks < max_hvx ? nblocks : max_hvx); + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, blk_threads, + htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_BLOCKED_DMA, KH, patch_stride)); + + if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { + kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA; + kparams->n_hvx = (uint8_t) blk_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->src_row_bytes = src_row_bytes; + kparams->dst_row_bytes = dst_row_bytes; + kparams->owb = owb; + kparams->wb = (uint32_t) Wb; + kparams->nbpr = nbpr; + kparams->N = N; + kparams->IC = IC; + kparams->ID = 1; + kparams->IH = IH; + kparams->IW = IW; + kparams->KD = 1; + kparams->KH = KH; + kparams->KW = KW; + kparams->OD = 1; + kparams->OH = OH; + kparams->OW = OW; + kparams->patch_stride = patch_stride; + kparams->total_rows = total_rows; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_OH = init_fastdiv_values(OH); + kparams->div_OD = init_fastdiv_values(1); + kparams->div_nbpr = init_fastdiv_values(nbpr); + return true; + } + if (owb == 1) break; + } + + return false; +} + +static bool htp_kparams_solve_im2col_3d( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_im2col_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED; + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (!src0 || !src1 || !dst) { + return false; + } + if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) { + return false; + } + if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) { + return false; + } + + const int32_t * params = (const int32_t *) op->op_params; + const int32_t s0 = params[0], s1 = params[1], s2 = params[2]; + const int32_t p0 = params[3], p1 = params[4], p2 = params[5]; + const int32_t d0 = params[6], d1 = params[7], d2 = params[8]; + const int32_t ic = params[9]; + + if (s0 <= 0 || s1 <= 0 || s2 <= 0 || + p0 < 0 || p1 < 0 || p2 < 0 || + d0 <= 0 || d1 <= 0 || d2 <= 0 || ic <= 0) { + return false; + } + if (src1->ne[3] % ic != 0) { + return false; + } + + const uint32_t IC = (uint32_t) ic; + const uint32_t N = (uint32_t) (src1->ne[3] / IC); + if (N == 0) { + return false; + } + + uint32_t OW, OH, OD; + if (!im2col_3d_calc_output_size(src1->ne[0], src0->ne[0], s0, p0, d0, &OW) || + !im2col_3d_calc_output_size(src1->ne[1], src0->ne[1], s1, p1, d1, &OH) || + !im2col_3d_calc_output_size(src1->ne[2], src0->ne[2], s2, p2, d2, &OD)) { + return false; + } + + const uint32_t KD = (uint32_t) src0->ne[2]; + const uint32_t KH = (uint32_t) src0->ne[1]; + const uint32_t KW = (uint32_t) src0->ne[0]; + const uint32_t ID = (uint32_t) src1->ne[2]; + const uint32_t IH = (uint32_t) src1->ne[1]; + const uint32_t IW = (uint32_t) src1->ne[0]; + + const uint64_t patch_stride_64 = (uint64_t) IC * KD * KH * KW; + const uint64_t total_rows_64 = (uint64_t) N * OD * OH; + const uint64_t dst_ne3_64 = (uint64_t) OD * N; + + if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX || dst_ne3_64 > UINT32_MAX) { + return false; + } + const uint32_t patch_stride = (uint32_t) patch_stride_64; + const uint32_t total_rows = (uint32_t) total_rows_64; + const uint32_t dst_ne3 = (uint32_t) dst_ne3_64; + + if (dst->ne[0] != patch_stride || dst->ne[1] != OW || dst->ne[2] != OH || dst->ne[3] != dst_ne3) { + return false; + } + + const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); + + const bool is_patchembed = ggml_is_contiguous(src1) && + (s0 == (int32_t) KW && s1 == (int32_t) KH && s2 == (int32_t) KD && + p0 == 0 && p1 == 0 && p2 == 0 && + d0 == 1 && d1 == 1 && d2 == 1); + + const uint32_t max_hvx = (sess && htp_session_n_hvx(sess) > 0) ? htp_session_n_hvx(sess) : 8; + const uint32_t n_hvx = (total_rows < max_hvx) ? total_rows : max_hvx; + const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0; + const uint32_t row_size = (uint32_t) dst->nb[2]; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + + if (is_patchembed) { + const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * IW * sizeof(float); + const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem; + if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) { + const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256); + const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, 256); + + if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { + kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->src_row_bytes = src_row_bytes; + kparams->dst_row_bytes = dst_row_bytes; + kparams->owb = OW; + kparams->wb = IW; + kparams->nbpr = 1; + kparams->N = N; + kparams->IC = IC; + kparams->ID = ID; + kparams->IH = IH; + kparams->IW = IW; + kparams->KD = KD; + kparams->KH = KH; + kparams->KW = KW; + kparams->OD = OD; + kparams->OH = OH; + kparams->OW = OW; + kparams->patch_stride = patch_stride; + kparams->total_rows = total_rows; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_OH = init_fastdiv_values(OH); + kparams->div_OD = init_fastdiv_values(OD); + return true; + } + } + } + + for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) { + const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1; + if (Wb > UINT32_MAX) { + if (owb == 1) break; + continue; + } + + const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * Wb * sizeof(float); + const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem; + if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) { + if (owb == 1) break; + continue; + } + + const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256); + const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, 256); + + if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { + kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->src_row_bytes = src_row_bytes; + kparams->dst_row_bytes = dst_row_bytes; + kparams->owb = owb; + kparams->wb = (uint32_t) Wb; + kparams->nbpr = (OW + owb - 1) / owb; + kparams->N = N; + kparams->IC = IC; + kparams->ID = ID; + kparams->IH = IH; + kparams->IW = IW; + kparams->KD = KD; + kparams->KH = KH; + kparams->KW = KW; + kparams->OD = OD; + kparams->OH = OH; + kparams->OW = OW; + kparams->patch_stride = patch_stride; + kparams->total_rows = total_rows; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_OH = init_fastdiv_values(OH); + kparams->div_OD = init_fastdiv_values(OD); + return true; + } + if (owb == 1) break; + } + + return false; +} + +static bool htp_kparams_solve_pad( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_pad_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + const int32_t lp0 = ((const int32_t *) op->op_params)[0]; + const int32_t rp0 = ((const int32_t *) op->op_params)[1]; + const bool circular = (((const int32_t *) op->op_params)[8] != 0); + + if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) { + return false; + } + + const uint32_t type_size = (uint32_t) ggml_type_size(src0->type); + const uint32_t total_dst_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + const uint32_t max_hvx = (std::max)(1u, total_dst_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + + struct htp_pad_vtcm_layout layout; + htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_hvx); + + const bool use_dma = (src0->nb[0] == type_size) && (src0->ne[0] >= 512) && + (htp_session_vtcm_size(sess) >= layout.total_bytes); + + kparams->n_hvx = n_hvx; + kparams->total_rows = total_dst_rows; + kparams->use_dma = use_dma ? 1 : 0; + kparams->elem_size = type_size; + kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0; + + if (circular && use_dma) { + kparams->kernel_type = HTP_PAD_KERNEL_HVX_CIRCULAR_DMA; + } else if (circular) { + kparams->kernel_type = HTP_PAD_KERNEL_HVX_CIRCULAR; + } else if (use_dma) { + kparams->kernel_type = HTP_PAD_KERNEL_HVX_DMA; + } else { + kparams->kernel_type = HTP_PAD_KERNEL_HVX; + } + + kparams->div_ne1 = init_fastdiv_values((uint32_t) dst->ne[1]); + kparams->div_ne1_ne2 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[2])); + kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + kparams->div_ne03 = init_fastdiv_values((uint32_t) src0->ne[3]); + + return true; +} + +static bool htp_kparams_solve_cumsum( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_cumsum_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + return false; + } + + const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + + struct htp_cumsum_vtcm_layout layout; + htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_hvx); + + const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes); + + kparams->n_hvx = n_hvx; + kparams->total_rows = total_rows; + kparams->use_dma = use_dma ? 1 : 0; + kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0; + + return true; +} + +// Shrink until the layout fits: halve tile_cols down to 1024 columns, then drop threads. False below the VTCM floor. +static bool htp_top_k_mask_search(struct htp_top_k_mask_vtcm_layout * L, const struct htp_top_k_mask_shape * S, uint32_t kernel_type, + size_t vtcm_size, uint32_t * n_hvx, uint32_t * tile_cols) { + const uint32_t min_cols = 1024; + while (true) { + htp_top_k_mask_vtcm_layout_build(L, S, kernel_type, *n_hvx, *tile_cols); + if (L->total_bytes <= vtcm_size) { + return true; + } + if (*tile_cols > min_cols) { + *tile_cols = (std::max)(min_cols, (uint32_t) hex_round_up(*tile_cols / 2, 64)); + } else if (*n_hvx > 1) { + (*n_hvx)--; + } else { + return false; + } + } +} + +// Row r of t (same ne[1..3] as dst) is at r * nb[1]: the NPU needs one stride per input. +// A mask broadcasts by i mod ne: its rows are dense above nb[1], the divisors and dst ne[1..2] fit 16 bits. +static bool htp_top_k_mask_mask_ok(const struct ggml_tensor * m, const struct ggml_tensor * dst) { + return m->ne[1] <= UINT16_MAX && m->ne[2] <= UINT16_MAX && m->ne[3] <= UINT16_MAX && dst->ne[1] <= UINT16_MAX && dst->ne[2] <= UINT16_MAX && + (m->ne[2] == 1 || m->nb[2] == m->nb[1] * m->ne[1]) && + (m->ne[3] == 1 || m->nb[3] == m->nb[1] * m->ne[1] * m->ne[2]); +} + +static bool htp_top_k_mask_row_flat(const struct ggml_tensor * t, const struct ggml_tensor * dst) { + return t->ne[1] == dst->ne[1] && t->ne[2] == dst->ne[2] && t->ne[3] == dst->ne[3] && + (t->ne[2] == 1 || t->nb[2] == t->nb[1] * t->ne[1]) && + (t->ne[3] == 1 || t->nb[3] == t->nb[1] * t->ne[1] * t->ne[2]); +} + +static bool htp_kparams_solve_top_k_mask( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_top_k_mask_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * mask = op->src[0]; + const struct ggml_tensor * top_k = op->src[1]; + const struct ggml_tensor * score = op->src[2]; + const struct ggml_tensor * blocks = op->src[3]; + const struct ggml_tensor * extra = op->src[4]; + const struct ggml_tensor * dst = op; + + if ((mask->type != GGML_TYPE_F16 && mask->type != GGML_TYPE_F32) || dst->type != mask->type) { + return false; + } + const size_t elem_size = ggml_type_size(dst->type); + if (mask->nb[0] != elem_size || dst->nb[0] != elem_size || !ggml_is_contiguous(dst)) { + return false; + } + if (top_k->type != GGML_TYPE_I32 || top_k->nb[0] != 4) { + return false; + } + if (score && (score->type != GGML_TYPE_F32 || score->nb[0] != 4)) { + return false; + } + if (extra && (extra->type != GGML_TYPE_I32 || extra->nb[0] != 4)) { + return false; + } + // flat row addressing: inputs follow the dst rows, a mask with more than one row too + if (!htp_top_k_mask_row_flat(top_k, dst) || (score && !htp_top_k_mask_row_flat(score, dst)) || + (extra && !htp_top_k_mask_row_flat(extra, dst)) || (ggml_nrows(mask) > 1 && !htp_top_k_mask_mask_ok(mask, dst))) { + return false; + } + // the kernel scales block ids by blocks->nb[1] with a 16-bit vector multiply + if (blocks && (blocks->type != GGML_TYPE_I32 || blocks->nb[0] != 4 || blocks->nb[1] != (size_t) blocks->ne[0] * 4 || blocks->nb[1] > INT16_MAX)) { + return false; + } + + struct htp_top_k_mask_shape S = {}; + S.n_kv = (uint32_t) dst->ne[0]; + S.elem_size = (uint32_t) elem_size; + S.n_top = (uint32_t) top_k->ne[0]; + S.n_blk = blocks ? (uint32_t) blocks->ne[0] : 0; + S.n_pick = (uint32_t) (blocks ? blocks->ne[1] : score ? score->ne[0] : dst->ne[0]); + S.n_extra = extra ? (uint32_t) extra->ne[0] : 0; + const uint32_t total_rows = (uint32_t) ggml_nrows(dst); + S.has_score = score ? 1 : 0; + S.has_blocks = blocks ? 1 : 0; + S.mask_resident = ggml_nrows(mask) == 1 ? 1 : 0; + + // empty inputs would give out-of-range gathers and zero-length DMA rows: leave them to the CPU + if (S.n_kv == 0 || total_rows == 0 || S.n_pick == 0 || S.n_top == 0 || (blocks && S.n_blk == 0) || (extra && S.n_extra == 0)) { + return false; + } + + // PICKS for one row with a resident mask when its single row tile fits, ROWS otherwise + const uint32_t sess_threads = (std::max)(1u, htp_session_n_hvx(sess)); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + uint32_t kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS; + uint32_t n_hvx = sess_threads; + uint32_t tile_cols = 0; + + struct htp_top_k_mask_vtcm_layout L; + bool found = false; + if (total_rows == 1 && S.mask_resident && sess_threads > 1) { + kernel_type = HTP_TOP_K_MASK_KERNEL_PICKS; + tile_cols = hex_round_up(S.n_kv, 64); + htp_top_k_mask_vtcm_layout_build(&L, &S, kernel_type, n_hvx, tile_cols); + found = L.total_bytes <= vtcm_size; + } + if (!found) { + kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS; + n_hvx = (std::max)(1u, (std::min)(sess_threads, total_rows)); + tile_cols = hex_round_up(S.n_kv, 64); + found = htp_top_k_mask_search(&L, &S, kernel_type, vtcm_size, &n_hvx, &tile_cols); + } + if (!found) { + return false; + } + + // compact kernel_params fields: sizes and offsets in HTP_TOP_K_MASK_UNIT bytes as u16, all multiples of the unit + const uint64_t off_extra = htp_top_k_mask_rowin_off_extra(S.n_top); + const uint64_t off_score = htp_top_k_mask_rowin_off_score(S.n_top, S.n_extra); + const uint64_t u16_max = (uint64_t) UINT16_MAX * HTP_TOP_K_MASK_UNIT; + if (L.rowin_bytes > u16_max || L.tile_bytes > u16_max || off_extra > u16_max || off_score > u16_max || L.off_scratch > u16_max) { + return false; + } + const auto units = [](uint64_t bytes) { return (uint16_t) (bytes / HTP_TOP_K_MASK_UNIT); }; + + kparams->n_hvx = (uint8_t) n_hvx; + kparams->kernel_type = (uint8_t) kernel_type; + kparams->flags = (dst->type == GGML_TYPE_F16 ? HTP_TOP_K_MASK_F16 : 0) | + (score ? HTP_TOP_K_MASK_HAS_SCORE : 0) | + (blocks ? HTP_TOP_K_MASK_HAS_BLOCKS : 0) | + (extra ? HTP_TOP_K_MASK_HAS_EXTRA : 0) | + (S.mask_resident ? HTP_TOP_K_MASK_MASK_RESIDENT : 0); + kparams->elem_size = (uint8_t) elem_size; + kparams->vtcm_size = (uint32_t) L.total_bytes; + kparams->n_kv = S.n_kv; + kparams->n_top = S.n_top; + kparams->n_pick = S.n_pick; + kparams->n_extra = S.n_extra; + kparams->ninf = dst->type == GGML_TYPE_F16 ? 0xfc00fc00u : 0xff800000u; + kparams->blocks_bytes = (uint32_t) L.blocks_bytes; + kparams->score_bytes = score ? S.n_pick * 4 : 0; + kparams->tile_cols = tile_cols; + kparams->n_col_tiles = (S.n_kv + tile_cols - 1) / tile_cols; + kparams->n_blk = (uint16_t) S.n_blk; + kparams->blocks_nb1 = (uint16_t) (blocks ? blocks->nb[1] : 0); + kparams->off_blocks_u = units(L.off_blocks); + kparams->off_mask_u = units(L.off_mask); + kparams->off_rowin_u = units(L.off_rowin); + kparams->off_tiles_u = units(L.off_tiles); + kparams->off_mtiles_u = units(L.off_mtiles); + kparams->off_scratch_u = units(L.off_scratch); + kparams->rowin_units = units(L.rowin_bytes); + kparams->tile_units = units(L.tile_bytes); + kparams->off_extra_units = units(off_extra); + kparams->off_score_units = units(off_score); + + if (kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS) { + // per thread slices of the static copies and of the flag clear, a multiple of the unit + const auto slice = [&](uint64_t n) { return units(hex_round_up((n + n_hvx - 1) / n_hvx, HTP_TOP_K_MASK_UNIT)); }; + kparams->u.picks.picks_per_hvx = (uint32_t) hex_round_up((S.n_top + n_hvx - 1) / n_hvx, 32); + kparams->u.picks.blocks_slice_u = slice(L.blocks_bytes); + kparams->u.picks.top_slice_u = slice((uint64_t) S.n_top * 4); + kparams->u.picks.extra_slice_u = slice((uint64_t) S.n_extra * 4); + kparams->u.picks.score_slice_u = slice(kparams->score_bytes); + kparams->u.picks.flags_slice_u = slice(L.tile_bytes); + } else { + kparams->u.rows.total_rows = total_rows; + kparams->u.rows.rs_top = (uint32_t) top_k->nb[1]; + kparams->u.rows.rs_extra = extra ? (uint32_t) extra->nb[1] : 0; + kparams->u.rows.rs_score = score ? (uint32_t) score->nb[1] : 0; + kparams->u.rows.rs_dst = (uint32_t) dst->nb[1]; + kparams->u.rows.mask_nb1 = (uint32_t) mask->nb[1]; + if (S.mask_resident) { + kparams->u.rows.m.mask_bytes = (uint32_t) L.mask_bytes; + } else { + const int64_t d[5] = { dst->ne[1], dst->ne[2], mask->ne[1], mask->ne[2], mask->ne[3] }; + for (int k = 0; k < 5; k++) { + const struct fastdiv_values fd = init_fastdiv_values((uint32_t) d[k]); + kparams->u.rows.m.dec.mp[k] = fd.mp; + kparams->u.rows.m.dec.l[k] = (uint8_t) fd.l; + kparams->u.rows.m.dec.ne[k] = (uint16_t) d[k]; + } + } + } + + return true; +} + +static bool htp_kparams_solve_fill( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_fill_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * dst = op; + + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { + return false; + } + if (!ggml_is_contiguous_rows(dst) || ggml_is_permuted(dst)) { + return false; + } + + const uint32_t ne1 = (uint32_t) dst->ne[1]; + const uint32_t ne2 = (uint32_t) dst->ne[2]; + const uint32_t ne3 = (uint32_t) dst->ne[3]; + const uint32_t nb1 = (uint32_t) dst->nb[1]; + + const uint32_t total_rows = ne1 * ne2 * ne3; + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + const uint32_t elem_size = (uint32_t) ggml_type_size(dst->type); + const uint32_t rows_per_chunk = (nb1 > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(nb1, HEX_L2_LINE_SIZE)) : 1; + + kparams->n_hvx = n_hvx; + kparams->total_rows = total_rows; + kparams->elem_size = elem_size; + kparams->kernel_type = ggml_is_contiguous(dst) ? HTP_FILL_KERNEL_CONTIGUOUS : HTP_FILL_KERNEL_STRIDED; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_ne1 = init_fastdiv_values(ne1); + kparams->div_ne1_ne2 = init_fastdiv_values(ne1 * ne2); + + return true; +} + +static bool htp_kparams_solve_roll( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_roll_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_are_same_shape(src0, dst)) { + return false; + } + if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous(dst)) { + return false; + } + + const uint32_t ne0 = (uint32_t) dst->ne[0]; + const uint32_t ne1 = (uint32_t) dst->ne[1]; + const uint32_t ne2 = (uint32_t) dst->ne[2]; + const uint32_t ne3 = (uint32_t) dst->ne[3]; + + const int32_t s0 = ((const int32_t *) op->op_params)[0]; + uint32_t i0_src0 = 0; + if (-s0 < 0) { + i0_src0 = (uint32_t) (-s0 + (int32_t) ne0); + } else if ((uint32_t) (-s0) >= ne0) { + i0_src0 = (uint32_t) (-s0) - ne0; + } else { + i0_src0 = (uint32_t) (-s0); + } + + const uint32_t total_rows = ne1 * ne2 * ne3; + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + + kparams->n_hvx = n_hvx; + kparams->total_rows = total_rows; + kparams->i0_src0 = i0_src0; + kparams->n0 = ne0 - i0_src0; + + if (htp_session_mdev_count(sess) <= 1) { + if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) { + kparams->kernel_type = HTP_ROLL_KERNEL_DMA_CONTIGUOUS; + } else { + kparams->kernel_type = HTP_ROLL_KERNEL_DMA_STRIDED; + } + } else { + kparams->kernel_type = HTP_ROLL_KERNEL_HVX; + } + + kparams->div_ne1 = init_fastdiv_values(ne1); + kparams->div_ne2_ne1 = init_fastdiv_values(ne2 * ne1); + + return true; +} + +static bool htp_kparams_solve_diag( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_diag_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (src0->ne[1] != 1 || dst->ne[0] != dst->ne[1] || dst->ne[0] != src0->ne[0]) { + return false; + } + if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(dst) || + ggml_is_permuted(src0) || ggml_is_permuted(dst)) { + return false; + } + + const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]); + const uint32_t max_hvx = (std::max)(1u, total_batches); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); + const uint32_t dst_batch_size = (uint32_t) (dst->ne[1] * dst->nb[1]); + + bool can_split = (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !ggml_is_permuted(dst); + uint32_t batches_per_chunk = 1; + if (can_split) { + if (dst->ne[2] > 1 && (dst->nb[2] & (HEX_L2_LINE_SIZE - 1)) == 0 && + (dst->ne[3] <= 1 || (dst->nb[3] & (HEX_L2_LINE_SIZE - 1)) == 0)) { + batches_per_chunk = 1; + } else if (dst->nb[2] == dst_batch_size && + (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) { + batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1; + } else { + can_split = false; + } + } + + const uint32_t src_batch_size = (uint32_t) (src0->ne[0] * sizeof(float)); + const uint32_t dst_row_size = (uint32_t) (dst->ne[0] * sizeof(float)); + + struct htp_diag_vtcm_layout layout; + htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_hvx); + + const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes); + + kparams->n_hvx = n_hvx; + kparams->total_batches = total_batches; + kparams->use_dma = use_dma ? 1 : 0; + kparams->can_split = can_split ? 1 : 0; + kparams->batches_per_chunk = batches_per_chunk; + kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0; + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + + return true; +} + +static bool htp_kparams_solve_solve_tri( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_solve_tri_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (src0->ne[0] != src0->ne[1] || src0->ne[1] != src1->ne[1] || + src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3] || + !ggml_are_same_shape(src1, dst)) { + return false; + } + + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); + const uint32_t k = (uint32_t) src1->ne[0]; + const uint32_t col_block = 32; + const uint32_t k_chunks = (k + col_block - 1) / col_block; + const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]); + const bool batched = total_batches >= max_hvx_sess; + const uint32_t total_jobs = batched ? total_batches : (total_batches * k_chunks); + const uint32_t max_hvx = (std::max)(1u, total_jobs); + const uint32_t n_hvx = (std::min)(max_hvx_sess, max_hvx); + const uint32_t batch_size = (uint32_t) dst->nb[2]; + const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1; + + kparams->n_hvx = n_hvx; + kparams->total_batches = total_batches; + kparams->total_jobs = total_jobs; + kparams->k_chunks = k_chunks; + kparams->col_block = col_block; + kparams->batched = batched ? 1 : 0; + kparams->batches_per_chunk = batches_per_chunk; + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + kparams->div_k_chunks = init_fastdiv_values(k_chunks > 0 ? k_chunks : 1); + + return true; +} + +// Implicit GEMM over sliding windows of fp16 input croutons (HMX activation:single), no im2col +static bool htp_kparams_conv2d_win( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * dst, + struct htp_conv2d_kernel_params * kp +) { + if (htp_session_n_hmx(sess) == 0) { + return false; + } + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + const int32_t s0 = ggml_get_op_params_i32(dst, 0); + const int32_t s1 = ggml_get_op_params_i32(dst, 1); + const int32_t p0 = ggml_get_op_params_i32(dst, 2); + const int32_t p1 = ggml_get_op_params_i32(dst, 3); + const int32_t d0 = ggml_get_op_params_i32(dst, 4); + const int32_t d1 = ggml_get_op_params_i32(dst, 5); + + const uint32_t KW = src0->ne[0]; + const uint32_t KH = src0->ne[1]; + const uint32_t Cin = src0->ne[2]; + const uint32_t Cout = src0->ne[3]; + const uint32_t IW = src1->ne[0]; + const uint32_t IH = src1->ne[1]; + const uint32_t OW = dst->ne[0]; + const uint32_t OH = dst->ne[1]; + const uint32_t N = dst->ne[3]; + const size_t es = ggml_type_size(src0->type); + + if (s0 != 1 || s1 != 1 || p0 < 0 || p1 < 0 || dst->nb[0] != sizeof(float)) { + return false; + } + if (dst->nb[1] < OW * sizeof(float) || (Cout > 1 && dst->nb[2] < OW * sizeof(float))) { + return false; + } + + const bool whcn = src1->nb[0] == sizeof(float) && src1->nb[1] >= IW * sizeof(float) && (Cin == 1 || src1->nb[2] >= IH * src1->nb[1]); + const bool cwhn = !whcn && src1->nb[2] == sizeof(float) && src1->nb[0] == Cin * sizeof(float) && src1->nb[1] >= IW * src1->nb[0]; + if (!whcn && !cwhn) { + return false; + } + + const bool w_rows = src0->nb[0] == es && (KH == 1 || src0->nb[1] == KW * es) && (Cin == 1 || src0->nb[2] == KW * KH * es); + const bool co_c = (Cout == 1 || src0->nb[3] == es) && (Cin == 1 || src0->nb[2] >= (std::min)(Cout, 32u) * es); + if (!w_rows && !co_c) { + return false; + } + + // short output rows: fold the kw taps into channels so the virtual row is OW wide and output rows are contiguous + const bool fold = (whcn || (cwhn && d0 == 1)) && KW > 1 && KW * Cin <= 32 && OW < 64 && KW * KH <= 16 && Cout >= 16; + + const uint32_t T = KW * KH; + const uint32_t n_ch = fold ? KW * Cin : Cin; + const uint32_t n_cc = (n_ch + 31) / 32; + const uint32_t rem = n_ch - (n_cc - 1) * 32; + const uint32_t stop = (rem + 7) / 8 * 8 - 1; + const uint32_t SR = fold ? OW : IW + 2 * p0; + const uint32_t IHS = IH + 2 * p1; + const uint32_t n_wt = fold ? KH : T; + const uint32_t moff = (KH - 1) * d1 * SR + (fold ? 0 : (KW - 1) * d0); + const uint32_t halo = moff / 32 + 1; + const uint32_t n_rt = (Cout + 31) / 32; + const uint64_t n_q = (uint64_t) (N * IHS - (IHS - OH) - 1) * SR + OW; + const uint32_t nth = kp->n_hvx; + const size_t tile = 2048; + + if ((uint64_t) N * IHS * SR + moff + 64 >= (1ull << 31)) { + return false; + } + const uint32_t n_tiles = (uint32_t) ((n_q + 31) / 32); + + // weight packing scratch + uint32_t w_cs = 0; + size_t wraw = 0; + size_t wscr = 0; + if (w_rows) { + const uint32_t kpw = es == 2 ? 64 : 32; + for (uint32_t cs = 32; cs >= 2; cs /= 2) { + w_cs = cs; + wraw = (size_t) 32 * ((cs * T + kpw - 1) / kpw) * 128; + if (3 * wraw + 64 * 128 <= 256 * 1024 || (fold && cs == 32)) { + break; + } + } + // enough packing units for all threads + const uint32_t kpw2 = es == 2 ? 64 : 32; + while (!fold && w_cs > 2 && n_rt * ((Cin + 31) / 32) * (32 / w_cs) < kp->n_hvx) { + w_cs /= 2; + } + wraw = (std::min)(wraw, (size_t) 32 * ((w_cs * T + kpw2 - 1) / kpw2) * 128); + wscr = 3 * wraw + 64 * 128 + 128; + } else { + w_cs = fold ? KW : (std::min)(T, 16u); + while (!fold && w_cs > 1 && n_rt * ((Cin + 31) / 32) * ((T + w_cs - 1) / w_cs) < kp->n_hvx) { + w_cs = (w_cs + 1) / 2; + } + wscr = 2 * (size_t) w_cs * 32 * 128; + } + + uint32_t rt_block = n_rt; + uint32_t ct = (std::max)(4 * halo, (n_tiles + 3) / 4); + ct = (std::min)((std::max)(ct, 8u), 512u); + ct = (std::min)(ct, n_tiles); + uint32_t seg_b = 8; + // longer output rows make the output DMA more efficient + uint32_t seg_t = (std::min)(32u, (std::max)(8u, (OW + 31) / 32)); + + while (rt_block > 1 && (size_t) rt_block * n_wt * n_cc * tile > htp_session_vtcm_size(sess) / 2) { + rt_block = (rt_block + 1) / 2; + } + + size_t stage = 0, thr = 0, act = 0, out = 0, w_size = 0, total = 0; + uint32_t n_bc = 0; + while (true) { + n_bc = ct + halo; + stage = cwhn ? (size_t) seg_b * 32 * Cin * sizeof(float) + 256 : (size_t) n_ch * seg_b * 32 * sizeof(float); + if (fold) { + // raw input rows of the staged pixels with 128 bytes of read slack on both sides + stage = (size_t) Cin * ((seg_b * 32 + OW - 1) / OW + 1) * IW * sizeof(float) + 256; + } + stage = hex_round_up(stage, 128); + thr = hex_round_up((std::max)(2 * stage + (size_t) HTP_CONV2D_WIN_OBUF * 32 * seg_t * 32 * sizeof(float), wscr), tile); + w_size = (size_t) rt_block * n_wt * n_cc * tile; + act = (size_t) n_cc * n_bc * tile; + out = (size_t) rt_block * ct * tile; + struct htp_conv2d_vtcm_layout L; + htp_conv2d_vtcm_layout_build(&L, rt_block, w_size, act, out, thr, nth); + total = L.total_bytes; + if (total <= htp_session_vtcm_size(sess)) { + break; + } + if (ct > 64) { + ct /= 2; + } else if (rt_block > 1) { + rt_block = (rt_block + 1) / 2; + } else if (seg_b > 1 || seg_t > 1) { + seg_b = (std::max)(seg_b / 2, 1u); + seg_t = (std::max)(seg_t / 2, 1u); + } else if (ct > 1) { + ct /= 2; + } else { + return false; + } + } + + uint32_t n_hvx = nth; + // Keep the fold schedule, where output traffic can dominate input packing. + if (!fold && rt_block < n_rt && ct > 0 && nth > 0) { + const uint32_t old_rt = rt_block; + const uint32_t ct_max = (std::min)((std::min)((std::max)(4 * halo, (n_tiles + 3) / 4), 512u), n_tiles); + const size_t budget = htp_session_vtcm_size(sess); + // Approximate chunk overhead in input crouton work units. + const size_t c_chunk = 8; + const uint32_t tail = n_tiles % ct; + const size_t work = + (size_t) (n_tiles / ct) * ((ct + halo + nth - 1) / nth) + (tail ? (tail + halo + nth - 1) / nth : 0); + size_t best_cost = + (size_t) ((n_rt + rt_block - 1) / rt_block) * (n_cc * work + c_chunk * (n_tiles / ct + (tail != 0))); + bool found = false; + + // Keep output DMA segments. Try larger kernel blocks to avoid rebuilding input croutons. + for (uint32_t rb = n_rt; rb > old_rt && !found; rb = (rb + 1) / 2) { + for (uint32_t nt = nth; nt > 0; nt /= 2) { + for (uint32_t sb = 8; sb > 0; sb /= 2) { + const size_t input_stage = hex_round_up(cwhn ? (size_t) sb * 32 * Cin * sizeof(float) + 256 : (size_t) n_ch * sb * 32 * sizeof(float), 128); + const size_t scratch = hex_round_up( + (std::max)(2 * input_stage + (size_t) HTP_CONV2D_WIN_OBUF * 32 * seg_t * 32 * sizeof(float), + wscr), + tile); + if (n_cc == 0 || n_wt > budget / tile / n_cc / rb || halo > budget / tile / n_cc || + scratch > budget / nt) { + continue; + } + const size_t weights = (size_t) rb * n_wt * n_cc * tile; + htp_conv2d_vtcm_layout fixed; + htp_conv2d_vtcm_layout_build(&fixed, rb, weights, (size_t) n_cc * halo * tile, 0, scratch, nt); + if (fixed.total_bytes >= budget) { + continue; + } + const size_t per_ct = 2 * ((size_t) n_cc + rb) * tile; + const uint32_t cap = (uint32_t) (std::min)((size_t) ct_max, (budget - fixed.total_bytes) / per_ct); + const uint32_t aligned = (cap + halo) / nt * nt; + const uint32_t candidates[2] = { cap, aligned > halo ? aligned - halo : 0 }; + for (uint32_t i = 0; i < 2; i++) { + const uint32_t candidate = candidates[i]; + if (candidate == 0 || (i > 0 && candidate == cap)) { + continue; + } + if (hex_round_up((size_t) T * sizeof(uint32_t), 128) + 2 * hex_round_up((size_t) candidate, 128) > HTP_DDR_SPAD_SIZE) { + continue; + } + const uint32_t rem = n_tiles % candidate; + const size_t input_work = (size_t) (n_tiles / candidate) * ((candidate + halo + nt - 1) / nt) + (rem ? (rem + halo + nt - 1) / nt : 0); + const size_t chunks = n_tiles / candidate + (rem != 0); + const size_t cost = (size_t) ((n_rt + rb - 1) / rb) * (n_cc * input_work + c_chunk * chunks); + if (cost < best_cost) { + const size_t input = (size_t) n_cc * (candidate + halo) * tile; + const size_t output = (size_t) rb * candidate * tile; + htp_conv2d_vtcm_layout L; + htp_conv2d_vtcm_layout_build(&L, rb, weights, input, output, scratch, nt); + if (L.total_bytes > budget) { + continue; + } + found = true; + best_cost = cost; + rt_block = rb; + ct = candidate; + seg_b = sb; + n_hvx = nt; + stage = input_stage; + thr = scratch; + act = input; + out = output; + w_size = weights; + total = L.total_bytes; + n_bc = candidate + halo; + } + } + } + } + } + } + + if (hex_round_up((size_t) T * sizeof(uint32_t), 128) + 2 * hex_round_up((size_t) ct, 128) > HTP_DDR_SPAD_SIZE) { + return false; + } + + const uint32_t TG = w_cs; + const uint32_t CS = w_cs; + const uint32_t n_tg = (T + TG - 1) / TG; + const uint32_t n_cs = (32 + CS - 1) / CS; + + kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_WINDOW; + kp->n_hvx = (uint8_t) n_hvx; + kp->cwhn = cwhn ? 1 : 0; + kp->w_kmajor = !w_rows ? 1 : 0; + kp->n_row_tiles = n_rt; + kp->ct_chunk = ct; + kp->rt_block = rt_block; + kp->stage_size = stage; + kp->thr_size = thr; + kp->vtcm_size = total; + kp->window.fold = fold; + kp->window.n_cc = n_cc; + kp->window.act_rt_last = 0x780 | 0x2 | (stop << 2); + kp->window.sr = SR; + kp->window.ihs = IHS; + kp->window.max_off = moff; + kp->window.n_bc = n_bc; + kp->window.seg_b = seg_b; + kp->window.seg_t = seg_t; + kp->window.w_cs = w_cs; + kp->window.div_sr = init_fastdiv_values(SR); + kp->window.div_ihs = init_fastdiv_values(IHS); + kp->window.div_kw = init_fastdiv_values(KW); + kp->window.div_tg = init_fastdiv_values(n_tg); + kp->window.div_cc = init_fastdiv_values(n_cc); + kp->window.div_cs = init_fastdiv_values(n_cs); + return true; +} + +struct htp_conv_norm_geom { + uint32_t KW, KH, Cin, Cout; + uint32_t OW, OH, N; + int32_t s0, d0; + size_t es; + bool is3d; +}; + +static bool htp_conv_normalize_geom(const struct ggml_tensor * dst, struct htp_conv_norm_geom * g) { + const struct ggml_tensor * src0 = dst->src[0]; + const bool is3d = dst->op == GGML_OP_CONV_3D; + const int32_t s0 = ggml_get_op_params_i32(dst, 0); + const int32_t d0 = ggml_get_op_params_i32(dst, is3d ? 6 : 4); + + const int64_t kh_w = is3d ? src0->ne[2] * src0->ne[1] : src0->ne[1]; + const int64_t cin_w = is3d ? ggml_get_op_params_i32(dst, 9) : src0->ne[2]; + const int64_t oh_w = is3d ? dst->ne[2] * dst->ne[1] : dst->ne[1]; + const int64_t n_w = is3d ? ggml_get_op_params_i32(dst, 10) : dst->ne[3]; + const int64_t k_w = src0->ne[0] * kh_w * cin_w; + const int64_t ct_w = n_w * oh_w * ((dst->ne[0] + 31) / 32); + + if (kh_w < 1 || kh_w > 255 || k_w < 1 || k_w > (1 << 24) || ct_w < 1 || ct_w > UINT32_MAX) { + return false; + } + if (s0 <= 0 || d0 <= 0 || dst->nb[0] != sizeof(float)) { + return false; + } + + *g = { + /* .KW = */ (uint32_t) src0->ne[0], + /* .KH = */ (uint32_t) kh_w, + /* .Cin = */ (uint32_t) cin_w, + /* .Cout = */ (uint32_t) (is3d ? ggml_get_op_params_i32(dst, 11) : src0->ne[3]), + /* .OW = */ (uint32_t) dst->ne[0], + /* .OH = */ (uint32_t) oh_w, + /* .N = */ (uint32_t) n_w, + /* .s0 = */ s0, + /* .d0 = */ d0, + /* .es = */ ggml_type_size(src0->type), + /* .is3d = */ is3d, + }; + return true; +} + +// Tiled implicit GEMM, MMA on HMX if available, else HVX +static bool htp_kparams_conv2d_tiled( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * dst, + struct htp_conv2d_kernel_params * kp +) { + if (htp_session_n_hmx(sess) == 0) { + return false; + } + + struct htp_conv_norm_geom g; + if (!htp_conv_normalize_geom(dst, &g)) { + return false; + } + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + // kernel k order must match the staged input: (cin, kh, kw) for WHCN input, (kh, kw, cin) for CWHN input. + // Kernel is either k-contiguous rows per cout (WHCN only) or cout-contiguous with a uniform k stride. + const bool whcn = g.is3d || src1->nb[0] == sizeof(float); + const bool cwhn = !whcn && src1->nb[2] == sizeof(float) && src1->nb[0] == g.Cin * sizeof(float); + const bool co_c = g.Cout == 1 || src0->nb[3] == g.es; + const bool w_rows = g.is3d || (whcn && src0->nb[0] == g.es && (g.KH == 1 || src0->nb[1] == g.KW * g.es) && (g.Cin == 1 || src0->nb[2] == g.KW * g.KH * g.es)); + const size_t ks = whcn ? src0->nb[0] : src0->nb[2]; + bool kmajor = false; + if (whcn) { + kmajor = co_c && ks >= g.Cout * g.es && (g.KH == 1 || src0->nb[1] == g.KW * ks) && (g.Cin == 1 || src0->nb[2] == g.KW * g.KH * ks); + } else if (cwhn) { + kmajor = co_c && ks >= g.Cout * g.es && src0->nb[0] == g.Cin * ks && (g.KH == 1 || src0->nb[1] == g.KW * g.Cin * ks); + } + if (!w_rows && !kmajor) { + return false; + } + kmajor = kmajor && !w_rows; + + const uint32_t K = g.KW * g.KH * g.Cin; + const uint32_t kg = !cwhn ? K : g.d0 == 1 ? g.KW * g.Cin : g.Cin; + const uint32_t n_grp = K / kg; + const uint32_t kg_p = hex_round_up(kg, 32); + const uint32_t k_pad = hex_round_up(n_grp * kg_p, 64); + const uint32_t n_kt = k_pad / 32; + const uint32_t n_rt = (g.Cout + 31) / 32; + const uint32_t n_owt = (g.OW + 31) / 32; + const uint32_t n_ct = g.N * g.OH * n_owt; + const uint32_t wk = (std::min)(k_pad, 512u); + const uint32_t max_hvx = htp_session_n_hvx(sess); + const size_t tile = 2048; + + uint32_t rt_block = n_rt; + uint32_t ct = 0; + uint32_t seg = 0; + uint32_t nth = 0; + uint32_t n_stage = 1; + uint32_t sr = 0; + size_t stage = 0, thr_size = 0; + size_t best_cost = SIZE_MAX; + const size_t wpack = kmajor ? 2 * (size_t) wk * 64 * sizeof(float) + 96 * 128 : 2 * (size_t) 32 * wk * (sizeof(float) + sizeof(uint16_t)); + const size_t budget = htp_session_vtcm_size(sess); + // Approximate chunk overhead in patch K-tile work units. + const size_t c_chunk = 8; + + // Keep the kernel resident to avoid regenerating the input for each rt block. + // Balance patch work per thread and chunk overhead before reducing rt. + for (uint32_t rb = n_rt; rb > 0 && ct == 0; rb = rb > 1 ? (rb + 1) / 2 : 0) { + for (uint32_t nt = max_hvx; nt > 0; nt /= 2) { + for (uint32_t sg = (std::min)(8u, n_owt); sg > 0; --sg) { + // Keep the unused final stride lane for full-vector loads during packing. + const uint64_t wb = (uint64_t) 32 * sg * g.s0 + (uint64_t) (g.KW - 1) * g.d0; + if (wb > budget / sizeof(float) || wb > INT32_MAX) { + continue; + } + const uint64_t width = cwhn ? wb * g.Cin : wb; + if (width > budget / sizeof(float) || width > UINT32_MAX - 63) { + continue; + } + const uint32_t row = hex_round_up((uint32_t) width, cwhn ? 64 : 32); + const uint64_t rows = cwhn ? g.KH : (uint64_t) g.Cin * g.KH; + if (rows > budget / ((uint64_t) row * sizeof(float))) { + continue; + } + const size_t input_stage = cwhn ? (size_t) g.KH * row * sizeof(float) : (size_t) g.Cin * g.KH * row * sizeof(float); + const size_t output_stage = 2 * 32 * (size_t) sg * 32 * sizeof(float); + const size_t transpose = cwhn ? input_stage / 2 + 256 + 64 * 128 : 0; + const size_t scratch = hex_round_up((std::max)(input_stage + output_stage + transpose, wpack), tile); + htp_conv2d_vtcm_layout fixed; + htp_conv2d_vtcm_layout_build(&fixed, rb, (size_t) rb * n_kt * tile, 0, 0, scratch, nt); + if (fixed.total_bytes >= budget) { + continue; + } + + // Each spatial tile adds two activation tiles and two output tiles per rt. + const size_t per_ct = 2 * ((size_t) n_kt + rb) * tile; + const uint32_t nc = (uint32_t) (std::min)((size_t) (std::min)(n_ct, 64u), (budget - fixed.total_bytes) / per_ct); + const uint32_t candidates[2] = { nc, nc / nt * nt }; + for (uint32_t i = 0; i < 2; i++) { + const uint32_t candidate = candidates[i]; + if (candidate == 0 || (i > 0 && candidate == candidates[0])) { + continue; + } + const uint32_t per_hvx = (candidate + nt - 1) / nt; + if (sg > per_hvx) { + continue; + } + const size_t work = (size_t) (n_ct / candidate) * per_hvx + (n_ct % candidate + nt - 1) / nt; + const size_t chunks = n_ct / candidate + (n_ct % candidate != 0); + const size_t cost = work * n_kt + c_chunk * chunks; + if (cost < best_cost) { + best_cost = cost; + rt_block = rb; + ct = candidate; + seg = sg; + nth = nt; + sr = row; + stage = input_stage; + thr_size = scratch; + } + } + } + } + } + if (ct == 0) { + return false; + } + + const size_t w_size = (size_t) rt_block * n_kt * tile; + const size_t act_size = (size_t) ct * n_kt * tile; + const size_t out_size = (size_t) rt_block * ct * tile; + const size_t ostage = 2 * 32 * (size_t) seg * 32 * sizeof(float); + const size_t prow = cwhn ? stage / 2 + 256 + 64 * 128 : 0; + struct htp_conv2d_vtcm_layout L; + htp_conv2d_vtcm_layout_build(&L, rt_block, w_size, act_size, out_size, thr_size, nth); + size_t total = L.total_bytes; + + // Second stage buffer: DMA of the next segment overlaps HVX pack of this one. + // Skip large stages. They already fill VTCM, and the extra DMA slows HMX. + { + const size_t thr2 = hex_round_up((std::max)(2 * stage + ostage + prow, wpack), tile); + struct htp_conv2d_vtcm_layout L2; + htp_conv2d_vtcm_layout_build(&L2, rt_block, w_size, act_size, out_size, thr2, nth); + if (stage <= 128 * 1024 && 2 * g.KH <= 255 && L2.total_bytes <= htp_session_vtcm_size(sess)) { + n_stage = 2; + thr_size = thr2; + total = L2.total_bytes; + } + } + + kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_TILED; + kp->n_hvx = (uint8_t) nth; + kp->cwhn = cwhn ? 1 : 0; + kp->w_kmajor = kmajor ? 1 : 0; + kp->n_row_tiles = n_rt; + kp->ct_chunk = ct; + kp->rt_block = rt_block; + kp->stage_size = stage; + kp->thr_size = thr_size; + kp->vtcm_size = total; + kp->tiled.w_kstride = ks; + kp->tiled.k_size = K; + kp->tiled.k_group = kg; + kp->tiled.k_group_pad = kg_p; + kp->tiled.n_groups = n_grp; + kp->tiled.n_k_tiles = n_kt; + kp->tiled.n_ow_tiles = n_owt; + kp->tiled.seg_tiles = seg; + kp->tiled.stage_row = sr; + kp->tiled.n_stage = n_stage; + kp->tiled.wk_chunk = wk; + kp->tiled.div_ow_tiles = init_fastdiv_values(n_owt); + kp->tiled.div_oh = init_fastdiv_values(g.OH); + kp->tiled.div_khr = init_fastdiv_values(src0->ne[1]); + kp->tiled.div_ohr = init_fastdiv_values(dst->ne[1]); + return true; +} + +static bool htp_kparams_solve_conv2d_unchecked( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * dst, + struct htp_conv2d_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + kparams->total_rows = total_rows; + kparams->n_hvx = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_hvx(sess), total_rows); + + if (htp_kparams_conv2d_win(sess, dst, kparams)) { + return true; + } + + return htp_kparams_conv2d_tiled(sess, dst, kparams); +} + +static bool htp_kparams_solve_conv2d( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_conv2d_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || + src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + if (src0->ne[2] != src1->ne[2]) { + return false; + } + + const int32_t s0 = ggml_get_op_params_i32(op, 0); + const int32_t s1 = ggml_get_op_params_i32(op, 1); + const int32_t p0 = ggml_get_op_params_i32(op, 2); + const int32_t p1 = ggml_get_op_params_i32(op, 3); + const int32_t d0 = ggml_get_op_params_i32(op, 4); + const int32_t d1 = ggml_get_op_params_i32(op, 5); + + if (s0 <= 0 || s1 <= 0 || d0 <= 0 || d1 <= 0) { + return false; + } + + const int64_t ow = (src1->ne[0] + 2 * p0 - d0 * (src0->ne[0] - 1) - 1) / s0 + 1; + const int64_t oh = (src1->ne[1] + 2 * p1 - d1 * (src0->ne[1] - 1) - 1) / s1 + 1; + if (ow != dst->ne[0] || oh != dst->ne[1] || src0->ne[3] != dst->ne[2] || src1->ne[3] != dst->ne[3]) { + return false; + } + + return htp_kparams_solve_conv2d_unchecked(sess, op, kparams); +} + +static bool htp_kparams_solve_conv2d_dw( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_conv2d_dw_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || + src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + if (src0->ne[2] != 1 || src0->ne[3] != src1->ne[2]) { + return false; + } + + if (src0->ne[0] * src0->ne[1] > 64) { + return false; + } + + const int32_t s0 = ggml_get_op_params_i32(op, 0); + const int32_t s1 = ggml_get_op_params_i32(op, 1); + const int32_t p0 = ggml_get_op_params_i32(op, 2); + const int32_t p1 = ggml_get_op_params_i32(op, 3); + const int32_t d0 = ggml_get_op_params_i32(op, 4); + const int32_t d1 = ggml_get_op_params_i32(op, 5); + + if (s0 <= 0 || s1 <= 0 || d0 <= 0 || d1 <= 0) { + return false; + } + + const int64_t ow = (src1->ne[0] + 2 * p0 - d0 * (src0->ne[0] - 1) - 1) / s0 + 1; + const int64_t oh = (src1->ne[1] + 2 * p1 - d1 * (src0->ne[1] - 1) - 1) / s1 + 1; + if (ow != dst->ne[0] || oh != dst->ne[1] || dst->ne[2] != src1->ne[2] || dst->ne[3] != src1->ne[3]) { + return false; + } + + const bool cwhn = ggml_is_contiguous_channels(src1); + if (!cwhn && !ggml_is_contiguous(src1)) { + return false; + } + + if (!cwhn && (d0 != 1 || (s0 != 1 && s0 != 2))) { + return false; + } + + const uint32_t c = (uint32_t) src1->ne[2]; + const uint32_t n = (uint32_t) src1->ne[3]; + const uint32_t w = (uint32_t) src1->ne[0]; + const uint32_t h = (uint32_t) src1->ne[1]; + const uint32_t kw = (uint32_t) src0->ne[0]; + const uint32_t kh = (uint32_t) src0->ne[1]; + const uint32_t ow_u = (uint32_t) ow; + const uint32_t oh_u = (uint32_t) oh; + + const uint32_t row_src_bytes = cwhn + ? (w * c * (uint32_t) sizeof(float)) + : (w * (uint32_t) sizeof(float)); + const uint32_t row_dst_bytes = cwhn + ? (ow_u * c * (uint32_t) sizeof(float)) + : (ow_u * (uint32_t) sizeof(float)); + + uint8_t kernel_type = HTP_CONV2D_DW_KERNEL_UNSUPPORTED; + if (cwhn) { + kernel_type = HTP_CONV2D_DW_KERNEL_CWHN; + } else if ((w & 31) == 0 && kw == 3 && kh == 3 && p0 == 1 && p1 == 1 && d0 == 1 && d1 == 1 && s0 == 1 && s1 == 1) { + kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S1_K3X3; + } else if ((w & 31) == 0 && kw == 3 && kh == 3 && p0 == 1 && p1 == 1 && d0 == 1 && d1 == 1 && s0 == 2 && s1 == 2) { + if (w == 32 && ow_u == 16) { + kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W32; + } else if (w == 64 && ow_u == 32) { + kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W64; + } else { + kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3; + } + } else if (s0 == 1) { + kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S1; + } else { + kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S2; + } + + const bool fast_k3x3 = kernel_type >= HTP_CONV2D_DW_KERNEL_WHCN_S1_K3X3; + const uint32_t pad_rows = fast_k3x3 ? 4 : 0; + const uint32_t pad_top_stride = fast_k3x3 ? w : 0; + + const size_t vtcm_budget = (sess && htp_session_vtcm_size(sess) > 0) ? htp_session_vtcm_size(sess) : 0; + if (vtcm_budget == 0) { + return false; + } + + const uint32_t w_size = (uint32_t) (kw * kh * c * ggml_type_size(src0->type)); + + uint32_t max_hvx_sess = (uint32_t) htp_session_n_hvx(sess); + uint32_t oh_chunk = oh_u; + + uint32_t n_hvx = max_hvx_sess; + struct htp_conv2d_dw_vtcm_layout layout; + + while (true) { + const uint32_t max_ih_chunk = (std::min)(h, (oh_chunk - 1) * (uint32_t) s1 + (kh - 1) * (uint32_t) d1 + 1); + const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128; + const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128; + + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_hvx); + + if (layout.total_bytes <= vtcm_budget && layout.total_bytes > 0) { + break; + } + + if (oh_chunk > 1) { + oh_chunk = (oh_chunk + 1) / 2; + } else if (n_hvx > 1) { + n_hvx--; + } else { + return false; + } + } + + const uint32_t n_chunks = (oh_u + oh_chunk - 1) / oh_chunk; + const uint32_t total_tasks = cwhn ? (n * n_chunks) : (c * n * n_chunks); + n_hvx = (std::min)(n_hvx, total_tasks); + if (n_hvx == 0) { + n_hvx = 1; + } + + const uint32_t max_ih_chunk = (std::min)(h, (oh_chunk - 1) * (uint32_t) s1 + (kh - 1) * (uint32_t) d1 + 1); + const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128; + const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128; + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_hvx); + + kparams->kernel_type = kernel_type; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->cwhn = cwhn ? 1 : 0; + kparams->is_f16 = (src0->type == GGML_TYPE_F16) ? 1 : 0; + kparams->total_tasks = total_tasks; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->w = w; + kparams->h = h; + kparams->c = c; + kparams->n = n; + kparams->kw = kw; + kparams->kh = kh; + kparams->ow = ow_u; + kparams->oh = oh_u; + kparams->s0 = s0; + kparams->s1 = s1; + kparams->p0 = p0; + kparams->p1 = p1; + kparams->d0 = d0; + kparams->d1 = d1; + kparams->oh_chunk = oh_chunk; + kparams->n_chunks = n_chunks; + kparams->row_src_bytes = row_src_bytes; + kparams->row_dst_bytes = row_dst_bytes; + kparams->pad_top_stride = pad_top_stride; + kparams->div_c = init_fastdiv_values(c); + kparams->div_n_chunks = init_fastdiv_values(n_chunks > 0 ? n_chunks : 1); + + return true; +} + +static bool htp_kparams_solve_conv3d( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_conv2d_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || + src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + const int32_t IC = ggml_get_op_params_i32(op, 9); + const int32_t N = ggml_get_op_params_i32(op, 10); + const int32_t OC = ggml_get_op_params_i32(op, 11); + if (IC <= 0 || N <= 0 || OC <= 0 || src0->ne[3] != (int64_t) IC * OC || src1->ne[3] != (int64_t) IC * N || + dst->ne[3] != (int64_t) OC * N) { + return false; + } + + for (int i = 0; i < 3; i++) { + const int32_t s = ggml_get_op_params_i32(op, i); + const int32_t p = ggml_get_op_params_i32(op, 3 + i); + const int32_t d = ggml_get_op_params_i32(op, 6 + i); + if (s <= 0 || d <= 0 || p < 0 || src0->ne[i] <= 0) { + return false; + } + const int64_t o = (src1->ne[i] + 2 * p - d * (src0->ne[i] - 1) - 1) / s + 1; + if (o != dst->ne[i]) { + return false; + } + } + + const size_t es = ggml_type_size(src0->type); + const bool w_rows = src0->nb[0] == es && src0->nb[1] == src0->ne[0] * es && src0->nb[2] == src0->ne[1] * src0->nb[1] && + ((int64_t) IC * OC == 1 || src0->nb[3] == src0->ne[2] * src0->nb[2]); + if (!w_rows || src1->nb[0] != sizeof(float) || dst->nb[2] != dst->ne[1] * dst->nb[1]) { + return false; + } + + const int64_t total_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + if (total_rows > UINT32_MAX) { + return false; + } + kparams->total_rows = (uint32_t) total_rows; + kparams->n_hvx = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_hvx(sess), (uint32_t) total_rows); + + return htp_kparams_conv2d_tiled(sess, op, kparams); +} + +static bool htp_dsv4_hc_pre_can_collapse( + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst +) { + if (!x || !weights || !dst) { + return false; + } + if (x->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (x->nb[0] != sizeof(float) || weights->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (x->ne[1] != 4 || dst->ne[0] != x->ne[0]) { + return false; + } + + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + const int64_t x_tokens = x->ne[2] * x->ne[3]; + const int64_t d_tokens = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const int64_t w_tokens = gated ? (weights->ne[2] * weights->ne[3]) : (weights->ne[1] * weights->ne[2] * weights->ne[3]); + + if (x_tokens != d_tokens || x_tokens != w_tokens) { + return false; + } + + if (x->ne[3] == 1 && dst->ne[2] == 1 && dst->ne[3] == 1 && + (gated ? (weights->ne[3] == 1) : (weights->ne[2] == 1 && weights->ne[3] == 1))) { + return false; + } + + if (x->ne[3] > 1 && x->nb[3] != x->ne[2] * x->nb[2]) return false; + + if (dst->ne[2] > 1 && dst->nb[2] != dst->ne[1] * dst->nb[1]) return false; + if (dst->ne[3] > 1 && dst->nb[3] != dst->ne[2] * dst->nb[2]) return false; + + if (gated) { + if (weights->ne[3] > 1 && weights->nb[3] != weights->ne[2] * weights->nb[2]) return false; + } else { + if (weights->ne[2] > 1 && weights->nb[2] != weights->ne[1] * weights->nb[1]) return false; + if (weights->ne[3] > 1 && weights->nb[3] != weights->ne[2] * weights->nb[2]) return false; + } + + return true; +} + +static void htp_dsv4_hc_pre_collapse_tensors( + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst, + struct ggml_tensor * x_c, + struct ggml_tensor * w_c, + struct ggml_tensor * d_c +) { + const int64_t n_tokens = x->ne[2] * x->ne[3]; + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + + *x_c = *x; + x_c->ne[2] = n_tokens; + x_c->ne[3] = 1; + x_c->nb[3] = x_c->ne[2] * x_c->nb[2]; + + *d_c = *dst; + d_c->ne[1] = n_tokens; + d_c->ne[2] = 1; + d_c->ne[3] = 1; + d_c->nb[2] = d_c->ne[1] * d_c->nb[1]; + d_c->nb[3] = d_c->nb[2]; + + *w_c = *weights; + if (gated) { + w_c->ne[2] = n_tokens; + w_c->ne[3] = 1; + w_c->nb[3] = w_c->ne[2] * w_c->nb[2]; + } else { + w_c->ne[1] = n_tokens; + w_c->ne[2] = 1; + w_c->ne[3] = 1; + w_c->nb[2] = w_c->ne[1] * w_c->nb[1]; + w_c->nb[3] = w_c->nb[2]; + } +} + +static bool htp_kparams_solve_dsv4_hc_pre_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst, + struct htp_dsv4_hc_pre_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (x->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (x->ne[1] != 4 || x->ne[3] != 1) { + return false; + } + if (x->ne[0] % HTP_DSV4_HC_CHUNK_ELEMS != 0) { + return false; + } + if (dst->ne[0] != x->ne[0] || dst->ne[1] != x->ne[2] || dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + if (x->nb[0] != sizeof(float) || weights->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (ggml_is_permuted(x) || ggml_is_permuted(weights) || ggml_is_permuted(dst)) { + return false; + } + + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + if (gated) { + if (weights->ne[0] != x->ne[0] || weights->ne[1] != x->ne[1] || weights->ne[2] != x->ne[2] || weights->ne[3] != 1) { + return false; + } + } else { + if (weights->ne[0] != x->ne[1] || weights->ne[1] != x->ne[2] || weights->ne[2] != 1 || weights->ne[3] != 1) { + return false; + } + } + + const uint32_t n_tokens = (uint32_t) x->ne[2]; + kparams->kernel_type = gated ? HTP_DSV4_HC_PRE_KERNEL_GATED : HTP_DSV4_HC_PRE_KERNEL_PLAIN; + + kparams->n_chunks = ((uint32_t) x->ne[0] + HTP_DSV4_HC_CHUNK_ELEMS - 1) / HTP_DSV4_HC_CHUNK_ELEMS; + kparams->total_jobs = n_tokens * kparams->n_chunks; + if (kparams->total_jobs == 0) { + return true; + } + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), kparams->total_jobs); + kparams->jobs_per_hvx = (kparams->total_jobs + kparams->n_hvx - 1) / kparams->n_hvx; + + const size_t vtcm_budget = htp_session_vtcm_size(sess); + const uint32_t n_rows = gated ? HTP_DSV4_HC_PRE_GATED_ROWS : HTP_DSV4_HC_PRE_PLAIN_ROWS; + uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks); + while (block_chunks > 16) { + const uint32_t stride = block_chunks * 128; + const uint32_t buf_sz = n_rows * stride; + if (2 * buf_sz * kparams->n_hvx <= vtcm_budget) { + break; + } + block_chunks /= 2; + } + + kparams->dma_block_chunks = block_chunks; + kparams->dma_row_stride = block_chunks * 128; + kparams->dma_buffer_size = n_rows * kparams->dma_row_stride; + kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_hvx; + if (kparams->vtcm_size > vtcm_budget) { + return false; + } + + kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks); + kparams->div_n_hvx = init_fastdiv_values(kparams->n_hvx); + + return true; +} + +static inline bool htp_kparams_solve_dsv4_hc_pre( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_dsv4_hc_pre_kernel_params * kparams +) { + if (htp_dsv4_hc_pre_can_collapse(op->src[0], op->src[1], op)) { + struct ggml_tensor x_c, w_c, d_c; + htp_dsv4_hc_pre_collapse_tensors(op->src[0], op->src[1], op, &x_c, &w_c, &d_c); + return htp_kparams_solve_dsv4_hc_pre_impl(sess, &x_c, &w_c, &d_c, kparams); + } + return htp_kparams_solve_dsv4_hc_pre_impl(sess, op->src[0], op->src[1], op, kparams); +} + +static bool htp_dsv4_hc_post_can_collapse( + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst +) { + if (!x || !residual || !post || !dst) { + return false; + } + if (x->type != GGML_TYPE_F32 || residual->type != GGML_TYPE_F32 || post->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (comb && comb->type != GGML_TYPE_F32) { + return false; + } + if (x->nb[0] != sizeof(float) || residual->nb[0] != sizeof(float) || post->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (comb && comb->nb[0] != sizeof(float)) { + return false; + } + if (residual->ne[0] != x->ne[0] || residual->ne[1] != 4 || dst->ne[0] != x->ne[0] || dst->ne[1] != 4 || post->ne[0] != 4) { + return false; + } + if (comb && (comb->ne[0] != 4 || comb->ne[1] != 4)) { + return false; + } + + const int64_t x_tokens = x->ne[1] * x->ne[2] * x->ne[3]; + const int64_t r_tokens = residual->ne[2] * residual->ne[3]; + const int64_t p_tokens = post->ne[1] * post->ne[2] * post->ne[3]; + const int64_t d_tokens = dst->ne[2] * dst->ne[3]; + if (x_tokens != r_tokens || x_tokens != p_tokens || x_tokens != d_tokens) { + return false; + } + if (comb && (comb->ne[2] * comb->ne[3] != x_tokens)) { + return false; + } + + if (x->ne[2] == 1 && x->ne[3] == 1 && + residual->ne[3] == 1 && + post->ne[2] == 1 && post->ne[3] == 1 && + dst->ne[3] == 1 && + (!comb || comb->ne[3] == 1)) { + return false; + } + + if (x->ne[2] > 1 && x->nb[2] != x->ne[1] * x->nb[1]) return false; + if (x->ne[3] > 1 && x->nb[3] != x->ne[2] * x->nb[2]) return false; + + if (residual->ne[3] > 1 && residual->nb[3] != residual->ne[2] * residual->nb[2]) return false; + if (dst->ne[3] > 1 && dst->nb[3] != dst->ne[2] * dst->nb[2]) return false; + + if (post->ne[2] > 1 && post->nb[2] != post->ne[1] * post->nb[1]) return false; + if (post->ne[3] > 1 && post->nb[3] != post->ne[2] * post->nb[2]) return false; + + if (comb && comb->ne[3] > 1 && comb->nb[3] != comb->ne[2] * comb->nb[2]) return false; + + return true; +} + +static void htp_dsv4_hc_post_collapse_tensors( + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst, + struct ggml_tensor * x_c, + struct ggml_tensor * r_c, + struct ggml_tensor * p_c, + struct ggml_tensor * c_c, + struct ggml_tensor * d_c +) { + const int64_t n_tokens = x->ne[1] * x->ne[2] * x->ne[3]; + + *x_c = *x; + x_c->ne[1] = n_tokens; + x_c->ne[2] = 1; + x_c->ne[3] = 1; + x_c->nb[2] = x_c->ne[1] * x_c->nb[1]; + x_c->nb[3] = x_c->nb[2]; + + *r_c = *residual; + r_c->ne[2] = n_tokens; + r_c->ne[3] = 1; + r_c->nb[3] = r_c->ne[2] * r_c->nb[2]; + + *d_c = *dst; + d_c->ne[2] = n_tokens; + d_c->ne[3] = 1; + d_c->nb[3] = d_c->ne[2] * d_c->nb[2]; + + *p_c = *post; + p_c->ne[1] = n_tokens; + p_c->ne[2] = 1; + p_c->ne[3] = 1; + p_c->nb[2] = p_c->ne[1] * p_c->nb[1]; + p_c->nb[3] = p_c->nb[2]; + + if (comb && c_c) { + *c_c = *comb; + c_c->ne[2] = n_tokens; + c_c->ne[3] = 1; + c_c->nb[3] = c_c->ne[2] * c_c->nb[2]; + } +} + +static bool htp_kparams_solve_dsv4_hc_post_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst, + struct htp_dsv4_hc_post_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (x->type != GGML_TYPE_F32 || residual->type != GGML_TYPE_F32 || post->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (comb && comb->type != GGML_TYPE_F32) { + return false; + } + if (x->ne[2] != 1 || x->ne[3] != 1) { + return false; + } + if (x->ne[0] % HTP_DSV4_HC_CHUNK_ELEMS != 0) { + return false; + } + if (residual->ne[0] != x->ne[0] || residual->ne[1] != 4 || residual->ne[2] != x->ne[1] || residual->ne[3] != 1) { + return false; + } + if (post->ne[0] != 4 || post->ne[1] != x->ne[1] || post->ne[2] != 1 || post->ne[3] != 1) { + return false; + } + if (comb && (comb->ne[0] != 4 || comb->ne[1] != 4 || comb->ne[2] != x->ne[1] || comb->ne[3] != 1)) { + return false; + } + if (dst->ne[0] != x->ne[0] || dst->ne[1] != 4 || dst->ne[2] != x->ne[1] || dst->ne[3] != 1) { + return false; + } + if (x->nb[0] != sizeof(float) || residual->nb[0] != sizeof(float) || post->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (comb && comb->nb[0] != sizeof(float)) { + return false; + } + if (ggml_is_permuted(x) || ggml_is_permuted(residual) || ggml_is_permuted(post) || ggml_is_permuted(dst) || (comb && ggml_is_permuted(comb))) { + return false; + } + + const uint32_t n_tokens = (uint32_t) x->ne[1]; + + kparams->has_comb = comb != nullptr; + kparams->kernel_type = kparams->has_comb ? HTP_DSV4_HC_POST_KERNEL_COMB : HTP_DSV4_HC_POST_KERNEL_IDENTITY; + kparams->n_chunks = ((uint32_t) x->ne[0] + HTP_DSV4_HC_CHUNK_ELEMS - 1) / HTP_DSV4_HC_CHUNK_ELEMS; + kparams->total_jobs = n_tokens * kparams->n_chunks; + if (kparams->total_jobs == 0) { + return true; + } + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), kparams->total_jobs); + kparams->jobs_per_hvx = (kparams->total_jobs + kparams->n_hvx - 1) / kparams->n_hvx; + + const size_t vtcm_budget = htp_session_vtcm_size(sess); + uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks); + while (block_chunks > 16) { + const uint32_t stride = block_chunks * 128; + const uint32_t buf_sz = HTP_DSV4_HC_POST_ROWS * stride; + if (2 * buf_sz * kparams->n_hvx <= vtcm_budget) { + break; + } + block_chunks /= 2; + } + + kparams->dma_block_chunks = block_chunks; + kparams->dma_row_stride = block_chunks * 128; + kparams->dma_buffer_size = HTP_DSV4_HC_POST_ROWS * kparams->dma_row_stride; + kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_hvx; + if (kparams->vtcm_size > vtcm_budget) { + return false; + } + + kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks); + kparams->div_n_hvx = init_fastdiv_values(kparams->n_hvx); + + return true; +} + +static inline bool htp_kparams_solve_dsv4_hc_post( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_dsv4_hc_post_kernel_params * kparams +) { + if (htp_dsv4_hc_post_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) { + struct ggml_tensor x_c, r_c, p_c, c_c, d_c; + htp_dsv4_hc_post_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op, &x_c, &r_c, &p_c, op->src[3] ? &c_c : nullptr, &d_c); + return htp_kparams_solve_dsv4_hc_post_impl(sess, &x_c, &r_c, &p_c, op->src[3] ? &c_c : nullptr, &d_c, kparams); + } + return htp_kparams_solve_dsv4_hc_post_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams); +} + +static bool htp_lightning_indexer_can_collapse( + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * dst +) { + if (!q || !k || !w || !m || !dst) { + return false; + } + + // 2D K normalization: ne[1] is n_kv, ne[2] is 1 + const bool can_collapse_k = (k->ne[1] > 1 && k->ne[2] == 1); + + // Stream-to-token collapse across q, w, dst (and m) + bool can_collapse_stream = false; + if (q->ne[3] > 1 && k->ne[3] == 1) { + const bool q_ok = (q->nb[3] == (size_t) q->ne[2] * q->nb[2]); + const bool dst_ok = (dst->ne[3] == q->ne[3] && dst->ne[2] == 1 && dst->ne[1] == q->ne[2] && + dst->nb[3] == (size_t) dst->ne[1] * dst->nb[1]); + const bool w_ok = (w->ne[3] == q->ne[3] && w->ne[2] == 1 && w->ne[1] == q->ne[2] && + w->nb[3] == (size_t) w->ne[1] * w->nb[1]) || + (w->ne[3] == 1 && w->ne[1] == q->ne[2] * q->ne[3]); + const bool m_ok = (m->ne[3] == q->ne[3] && m->ne[2] == 1 && m->ne[1] == q->ne[2] && + m->nb[3] == (size_t) m->ne[1] * m->nb[1]) || + (m->ne[3] == 1 && m->ne[1] == q->ne[2] * q->ne[3]); + can_collapse_stream = q_ok && dst_ok && w_ok && m_ok; + } + + return can_collapse_k || can_collapse_stream; +} + +static void htp_lightning_indexer_collapse_tensors( + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * dst, + struct ggml_tensor * q_c, + struct ggml_tensor * k_c, + struct ggml_tensor * w_c, + struct ggml_tensor * m_c, + struct ggml_tensor * dst_c +) { + *q_c = *q; + *k_c = *k; + *w_c = *w; + *m_c = *m; + *dst_c = *dst; + + if (k->ne[1] > 1 && k->ne[2] == 1) { + k_c->ne[2] = k->ne[1]; + k_c->ne[1] = 1; + k_c->nb[1] = k->nb[1]; + k_c->nb[2] = k->nb[1]; + k_c->nb[3] = k->nb[2]; + } + + if (q->ne[3] > 1 && k->ne[3] == 1) { + const int64_t n_tokens_collapsed = (int64_t) q->ne[2] * q->ne[3]; + + q_c->ne[2] = n_tokens_collapsed; + q_c->ne[3] = 1; + q_c->nb[3] = (size_t) q_c->ne[2] * q_c->nb[2]; + + if (w->ne[3] == q->ne[3]) { + w_c->ne[1] = n_tokens_collapsed; + w_c->ne[3] = 1; + w_c->nb[2] = (size_t) w_c->ne[1] * w_c->nb[1]; + w_c->nb[3] = w_c->nb[2]; + } + + dst_c->ne[1] = n_tokens_collapsed; + dst_c->ne[3] = 1; + dst_c->nb[2] = (size_t) dst_c->ne[1] * dst_c->nb[1]; + dst_c->nb[3] = dst_c->nb[2]; + + if (m->ne[3] == q->ne[3] && m->ne[1] == q->ne[2]) { + m_c->ne[1] = n_tokens_collapsed; + m_c->ne[3] = 1; + m_c->nb[2] = (size_t) m_c->ne[1] * m_c->nb[1]; + m_c->nb[3] = m_c->nb[2]; + } + } +} + +static bool htp_kparams_solve_lightning_indexer_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * op, + struct htp_lid_kernel_params * kparams +) { + if (opt_li_select < 1) { + return false; + } + + if (k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0 && k->type != GGML_TYPE_Q4_0) { + return false; + } + + if (q->ne[0] % 32 != 0 || + !ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || + !ggml_is_contiguous_rows(w) || !ggml_is_contiguous_rows(m) || + !ggml_is_contiguous_rows(op)) { + return false; + } + + const uint32_t n_kv_blocks = (k->ne[2] + HTP_LID_KV_BLOCK - 1) / HTP_LID_KV_BLOCK; + if (n_kv_blocks == 0 || q->ne[2] == 0) { + return false; + } + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_tokens = q->ne[2]; + const size_t k_row_size = ggml_row_size(k->type, D); + + kparams->k_row_size = (uint32_t) k_row_size; + kparams->div_n_head = init_fastdiv_values(n_head); + kparams->div_m_ne3 = init_fastdiv_values((uint32_t) m->ne[3]); + + if (opt_li_select >= 2 && htp_session_n_hmx(sess) > 0 && D % HMX_FP16_TILE_N_COLS == 0) { + const uint32_t n_hvx = htp_session_n_hvx(sess); + const uint32_t n_stream = q->ne[3]; + const uint32_t n_dot = D / HMX_FP16_TILE_N_COLS; + + auto vtcm_usage = [&](uint32_t n_tok, uint32_t kv_block, uint32_t n_qsets) { + return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_hvx, k_row_size); + }; + + uint32_t n_tok = (std::min)(n_tokens, (uint32_t) HTP_LID_HMX_TOKENS_MAX); + while (n_tok > 32 && vtcm_usage(n_tok, 64, 1) > htp_session_vtcm_size(sess)) { + n_tok /= 2; + } + + uint32_t n_qsets = n_tok == n_tokens ? (std::min)(n_stream, (uint32_t) HTP_LID_HMX_MAX_QSETS) : 1; + while (n_qsets > 1 && vtcm_usage(n_tok, 64, n_qsets) > htp_session_vtcm_size(sess)) { + n_qsets /= 2; + } + + uint32_t n_qbuf = 1; + if (n_qsets == 1 && n_head >= 16 && n_stream * ((n_tokens + n_tok - 1) / n_tok) > 1) { + uint32_t nt2 = n_tok; + while (nt2 > 32 && vtcm_usage(nt2, 64, 2) > htp_session_vtcm_size(sess)) { + nt2 /= 2; + } + const size_t q_bytes = (size_t) nt2 * n_head * D * sizeof(float); + const size_t k_bytes = (size_t) k->ne[2] * ggml_row_size(k->type, D); + if (vtcm_usage(nt2, 64, 2) <= htp_session_vtcm_size(sess) && q_bytes >= k_bytes) { + n_tok = nt2; + n_qbuf = 2; + } + } + + const uint32_t n_ch = (n_tok * n_head + 31) / 32; + const uint32_t kv_seg = hex_round_up(k->ne[2], 64); + const uint32_t n_rows = n_qsets * kv_seg; + const uint32_t bc_max = n_rows <= 512 ? n_rows : (std::max)(256u, n_rows / 4); + uint32_t kv_block = 64; + while (kv_block < HTP_LID_HMX_KV_BLOCK_MAX && kv_block < bc_max && (kv_block / 32) * n_ch * (n_dot + 1) < 256 && + vtcm_usage(n_tok, kv_block * 2, n_qsets * n_qbuf) <= htp_session_vtcm_size(sess)) { + kv_block *= 2; + } + + const size_t vtcm_size = vtcm_usage(n_tok, kv_block, n_qsets * n_qbuf); + if (vtcm_size <= htp_session_vtcm_size(sess)) { + const uint32_t n_tt = (n_tokens + n_tok - 1) / n_tok; + kparams->kernel_type = HTP_LID_KERNEL_HMX; + kparams->n_hvx = n_hvx; + kparams->n_tokens_tile = n_tok; + kparams->n_kv_blocks = (n_rows + kv_block - 1) / kv_block; + kparams->blocks_per_hvx = 0; + kparams->kv_block = kv_block; + kparams->n_qsets = n_qsets; + kparams->n_qbuf = n_qbuf; + kparams->kv_seg = kv_seg; + kparams->n_tt = n_tt; + kparams->vtcm_size = (uint32_t) vtcm_size; + kparams->div_kv_seg = init_fastdiv_values(kv_seg); + kparams->div_n_tt = init_fastdiv_values(n_tt); + kparams->div_n_ch = init_fastdiv_values(n_ch); + return true; + } + } + + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), n_kv_blocks); + uint32_t n_tok = (std::min)((uint32_t) q->ne[2], (uint32_t) HTP_LID_TOKENS_MAX); + while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_hvx, k_row_size) > htp_session_vtcm_size(sess)) { + n_tok /= 2; + } + + const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_hvx, k_row_size); + if (vtcm_size > htp_session_vtcm_size(sess)) { + return false; + } + + kparams->kernel_type = HTP_LID_KERNEL_HVX; + kparams->n_hvx = n_hvx; + kparams->n_tokens_tile = n_tok; + kparams->n_kv_blocks = n_kv_blocks; + kparams->blocks_per_hvx = (n_kv_blocks + n_hvx - 1) / n_hvx; + kparams->kv_block = HTP_LID_KV_BLOCK; + kparams->n_qsets = 1; + kparams->n_qbuf = 1; + kparams->kv_seg = 0; + kparams->n_tt = 0; + kparams->vtcm_size = (uint32_t) vtcm_size; + + return true; +} + +static bool htp_kparams_solve_lightning_indexer( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_lid_kernel_params * kparams +) { + if (htp_lightning_indexer_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) { + struct ggml_tensor q_c, k_c, w_c, m_c, d_c; + htp_lightning_indexer_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op, + &q_c, &k_c, &w_c, &m_c, &d_c); + return htp_kparams_solve_lightning_indexer_impl(sess, &q_c, &k_c, &w_c, &m_c, &d_c, kparams); + } + return htp_kparams_solve_lightning_indexer_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams); +} +template <auto SolveFn> +static inline bool htp_kparams_invoke( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + void * kparams +) { + using KParams = decltype(htp_kparams_type(SolveFn)); + KParams tmp; + return SolveFn(sess, op, kparams ? (KParams *) kparams : &tmp); +} + +void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & node) { + const uint32_t n_hvx = htp_session_n_hvx(sess); + + if (htp_op_is_unary(node.opcode) && htp_unary_is_elementwise(node.opcode)) { + if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) { + return; + } + if (!ggml_are_same_shape(node.inputs[0], node.outputs[0])) { + return; + } + const size_t elem_size = ggml_type_size(node.inputs[0]->type); + struct ggml_tensor src0_c = *node.inputs[0]; + struct ggml_tensor dst_c = *node.outputs[0]; + struct ggml_tensor * tensors[2] = { &src0_c, &dst_c }; + htp_collapse_dims(tensors, 2, elem_size, n_hvx); + if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { + node.inputs[0] = node.add_dummy(src0_c); + node.outputs[0] = node.add_dummy(dst_c); + } + return; + } + + if (htp_op_is_glu(node.opcode)) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + if (!ggml_are_same_shape(node.inputs[0], node.outputs[0]) || + !ggml_are_same_shape(node.inputs[1], node.outputs[0])) { + return; + } + const size_t elem_size = ggml_type_size(node.inputs[0]->type); + struct ggml_tensor src0_c = *node.inputs[0]; + struct ggml_tensor src1_c = *node.inputs[1]; + struct ggml_tensor dst_c = *node.outputs[0]; + struct ggml_tensor * tensors[3] = { &src0_c, &src1_c, &dst_c }; + htp_collapse_dims(tensors, 3, elem_size, n_hvx); + if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { + node.inputs[0] = node.add_dummy(src0_c); + node.inputs[1] = node.add_dummy(src1_c); + node.outputs[0] = node.add_dummy(dst_c); + } + return; + } + + if (node.opcode == HTP_OP_MUL_MAT) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * src0 = node.inputs[0]; + const struct ggml_tensor * src1 = node.inputs[1]; + const struct ggml_tensor * dst = node.outputs[0]; + if (htp_matmul_can_collapse(src0, src1, dst)) { + node.inputs[1] = node.add_dummy(htp_tensor_collapse_rows(src1)); + node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst)); + } + return; + } + + if (node.opcode == HTP_OP_MUL_MAT_ADD) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * src0 = node.inputs[0]; + const struct ggml_tensor * src1 = node.inputs[1]; + const struct ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const struct ggml_tensor * dst = node.outputs[0]; + if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) { + node.inputs[1] = node.add_dummy(htp_tensor_collapse_rows(src1)); + node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst)); + if (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) { + node.inputs[2] = node.add_dummy(htp_tensor_collapse_rows(src2)); + } + } + return; + } + + if (node.opcode == HTP_OP_UPSCALE) { + if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * src0 = node.inputs[0]; + const struct ggml_tensor * dst = node.outputs[0]; + + if ((src0->ne[3] > 1 || dst->ne[3] > 1) && + src0->ne[2] == dst->ne[2] && src0->ne[3] == dst->ne[3] && + src0->nb[3] == (size_t) src0->ne[2] * src0->nb[2] && + dst->nb[3] == (size_t) dst->ne[2] * dst->nb[2]) { + struct ggml_tensor src0_c = *src0; + struct ggml_tensor dst_c = *dst; + src0_c.ne[2] *= src0_c.ne[3]; + src0_c.ne[3] = 1; + src0_c.nb[3] = (size_t) src0_c.ne[2] * src0_c.nb[2]; + dst_c.ne[2] *= dst_c.ne[3]; + dst_c.ne[3] = 1; + dst_c.nb[3] = (size_t) dst_c.ne[2] * dst_c.nb[2]; + node.inputs[0] = node.add_dummy(src0_c); + node.outputs[0] = node.add_dummy(dst_c); + } + return; + } + + if (node.opcode == HTP_OP_DSV4_HC_PRE) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * x = node.inputs[0]; + const struct ggml_tensor * weights = node.inputs[1]; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_dsv4_hc_pre_can_collapse(x, weights, dst)) { + struct ggml_tensor x_c, w_c, d_c; + htp_dsv4_hc_pre_collapse_tensors(x, weights, dst, &x_c, &w_c, &d_c); + node.inputs[0] = node.add_dummy(x_c); + node.inputs[1] = node.add_dummy(w_c); + node.outputs[0] = node.add_dummy(d_c); + } + return; + } + + if (node.opcode == HTP_OP_DSV4_HC_POST) { + if (node.inputs.size() < 3 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * x = node.inputs[0]; + const struct ggml_tensor * residual = node.inputs[1]; + const struct ggml_tensor * post = node.inputs[2]; + const struct ggml_tensor * comb = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_dsv4_hc_post_can_collapse(x, residual, post, comb, dst)) { + struct ggml_tensor x_c, r_c, p_c, c_c, d_c; + htp_dsv4_hc_post_collapse_tensors(x, residual, post, comb, dst, &x_c, &r_c, &p_c, comb ? &c_c : nullptr, &d_c); + node.inputs[0] = node.add_dummy(x_c); + node.inputs[1] = node.add_dummy(r_c); + node.inputs[2] = node.add_dummy(p_c); + if (comb) { + node.inputs[3] = node.add_dummy(c_c); + } + node.outputs[0] = node.add_dummy(d_c); + } + return; + } + + if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + if (node.inputs.size() < 4 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || !node.inputs[3] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * q = node.inputs[0]; + const struct ggml_tensor * k = node.inputs[1]; + const struct ggml_tensor * w = node.inputs[2]; + const struct ggml_tensor * m = node.inputs[3]; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_lightning_indexer_can_collapse(q, k, w, m, dst)) { + struct ggml_tensor q_c, k_c, w_c, m_c, d_c; + htp_lightning_indexer_collapse_tensors(q, k, w, m, dst, &q_c, &k_c, &w_c, &m_c, &d_c); + node.inputs[0] = node.add_dummy(q_c); + node.inputs[1] = node.add_dummy(k_c); + node.inputs[2] = node.add_dummy(w_c); + node.inputs[3] = node.add_dummy(m_c); + node.outputs[0] = node.add_dummy(d_c); + } + return; + } +} + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + htp_opnode & node +) { + htp_opnode_collapse(sess, node); + + const ggml_tensor * src0 = node.inputs.empty() ? nullptr : node.inputs[0]; + const ggml_tensor * src1 = node.inputs.size() > 1 ? node.inputs[1] : nullptr; + const ggml_tensor * dst = node.dst(); + + if (htp_op_is_unary(node.opcode)) { + return htp_kparams_solve_unary_impl(sess, node.opcode, src0, src1, dst, node.kparams<htp_unary_kernel_params>()); + } + if (htp_op_is_glu(node.opcode)) { + return htp_kparams_solve_unary_mul_glu(sess, node.opcode, src0, src1, dst, node.kparams<htp_glu_kernel_params>()); + } + if (node.opcode == HTP_OP_MUL_MAT) { + if (!htp_matmul_check_types(src0, src1, dst)) { + return false; + } + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, node.kparams<htp_mm_kernel_params>()); + } + if (node.opcode == HTP_OP_MUL_MAT_ADD) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0; + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, node.kparams<htp_mm_kernel_params>()); + } + if (node.opcode == HTP_OP_UPSCALE) { + return htp_kparams_solve_upscale(sess, src0, dst, node.kparams<htp_upscale_kernel_params>()); + } + if (node.opcode == HTP_OP_DSV4_HC_PRE) { + return htp_kparams_solve_dsv4_hc_pre_impl(sess, src0, src1, dst, node.kparams<htp_dsv4_hc_pre_kernel_params>()); + } + if (node.opcode == HTP_OP_DSV4_HC_POST) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + return htp_kparams_solve_dsv4_hc_post_impl(sess, src0, src1, src2, src3, dst, node.kparams<htp_dsv4_hc_post_kernel_params>()); + } + if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + return htp_kparams_solve_lightning_indexer_impl(sess, src0, src1, src2, src3, dst, node.kparams<htp_lid_kernel_params>()); + } + + return htp_kparams_solve(sess, node.node, node.kernel_params); +} + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + void * kparams +) { + const htp_op_code opcode = htp_op_remap(op); + if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode) || opcode == HTP_OP_UPSCALE) { + htp_opnode tmp(opcode, op); + if (!htp_kparams_solve(sess, tmp)) { + return false; + } + if (kparams) { + memcpy(kparams, tmp.kernel_params, sizeof(tmp.kernel_params)); + } + return true; + } + + switch (opcode) { + case HTP_OP_MUL_MAT: + case HTP_OP_MUL_MAT_ID: + return htp_kparams_invoke<htp_kparams_solve_matmul>(sess, op, kparams); + + case HTP_OP_MUL: + case HTP_OP_ADD: + case HTP_OP_ADD_ID: + case HTP_OP_SUB: + case HTP_OP_DIV: + return htp_kparams_invoke<htp_kparams_solve_binary>(sess, op, kparams); + + case HTP_OP_FLASH_ATTN_EXT: + return htp_kparams_invoke<htp_kparams_solve_flash_attn>(sess, op, kparams); + + case HTP_OP_GET_ROWS: + return htp_kparams_invoke<htp_kparams_solve_get_rows>(sess, op, kparams); + + case HTP_OP_SET_ROWS: + return htp_kparams_invoke<htp_kparams_solve_set_rows>(sess, op, kparams); + + case HTP_OP_ROPE: + return htp_kparams_invoke<htp_kparams_solve_rope>(sess, op, kparams); + + case HTP_OP_SSM_CONV: + return htp_kparams_invoke<htp_kparams_solve_ssm_conv>(sess, op, kparams); + + case HTP_OP_SOFTMAX: + return htp_kparams_invoke<htp_kparams_solve_softmax>(sess, op, kparams); + + case HTP_OP_GATED_DELTA_NET: + return htp_kparams_invoke<htp_kparams_solve_gated_delta_net>(sess, op, kparams); + + case HTP_OP_ARGSORT: + case HTP_OP_TOP_K: + return htp_kparams_invoke<htp_kparams_solve_sort>(sess, op, kparams); + + case HTP_OP_POOL_2D: + case HTP_OP_POOL_1D: + return htp_kparams_invoke<htp_kparams_solve_pool_2d>(sess, op, kparams); + + case HTP_OP_CONCAT: + return htp_kparams_invoke<htp_kparams_solve_concat>(sess, op, kparams); + + case HTP_OP_CPY: + case HTP_OP_CPY_FENCE: + return htp_kparams_invoke<htp_kparams_solve_cpy>(sess, op, kparams); + + case HTP_OP_IM2COL: + return htp_kparams_invoke<htp_kparams_solve_im2col>(sess, op, kparams); + + case HTP_OP_IM2COL_3D: + return htp_kparams_invoke<htp_kparams_solve_im2col_3d>(sess, op, kparams); + + case HTP_OP_CONV_2D: + return htp_kparams_invoke<htp_kparams_solve_conv2d>(sess, op, kparams); + + case HTP_OP_CONV_2D_DW: + return htp_kparams_invoke<htp_kparams_solve_conv2d_dw>(sess, op, kparams); + + case HTP_OP_CONV_3D: + return htp_kparams_invoke<htp_kparams_solve_conv3d>(sess, op, kparams); + + case HTP_OP_SUM_ROWS: + case HTP_OP_SUM: + case HTP_OP_ARGMAX: + return htp_kparams_invoke<htp_kparams_solve_sum_rows>(sess, op, kparams); + + case HTP_OP_REPEAT: + return htp_kparams_invoke<htp_kparams_solve_repeat>(sess, op, kparams); + + case HTP_OP_PAD: + return htp_kparams_invoke<htp_kparams_solve_pad>(sess, op, kparams); + + case HTP_OP_CUMSUM: + return htp_kparams_invoke<htp_kparams_solve_cumsum>(sess, op, kparams); + + case HTP_OP_TOP_K_MASK: + return htp_kparams_invoke<htp_kparams_solve_top_k_mask>(sess, op, kparams); + + case HTP_OP_FILL: + return htp_kparams_invoke<htp_kparams_solve_fill>(sess, op, kparams); + + case HTP_OP_ROLL: + return htp_kparams_invoke<htp_kparams_solve_roll>(sess, op, kparams); + + case HTP_OP_LIGHTNING_INDEXER: + return htp_kparams_invoke<htp_kparams_solve_lightning_indexer>(sess, op, kparams); + + case HTP_OP_DIAG: + return htp_kparams_invoke<htp_kparams_solve_diag>(sess, op, kparams); + + case HTP_OP_SOLVE_TRI: + return htp_kparams_invoke<htp_kparams_solve_solve_tri>(sess, op, kparams); + + case HTP_OP_DSV4_HC_PRE: + return htp_kparams_invoke<htp_kparams_solve_dsv4_hc_pre>(sess, op, kparams); + + case HTP_OP_DSV4_HC_POST: + return htp_kparams_invoke<htp_kparams_solve_dsv4_hc_post>(sess, op, kparams); + + default: + if (htp_op_is_unary(opcode)) { + return htp_kparams_invoke<htp_kparams_solve_unary>(sess, op, kparams); + } + if (htp_op_is_glu(opcode)) { + return htp_kparams_invoke<htp_kparams_solve_glu>(sess, op, kparams); + } + return false; + } +} diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h new file mode 100644 index 000000000..653554be4 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-kparams.h @@ -0,0 +1,164 @@ +#ifndef HTP_KPARAMS_H +#define HTP_KPARAMS_H + +#include "htp-common.h" + +#include "htp/allreduce-ops.h" +#include "htp/argsort-ops.h" +#include "htp/binary-ops.h" +#include "htp/concat-ops.h" +#include "htp/conv2d-ops.h" +#include "htp/cpy-ops.h" +#include "htp/cumsum-ops.h" +#include "htp/diag-ops.h" +#include "htp/dsv4-hc-ops.h" +#include "htp/fill-ops.h" +#include "htp/flash-attn-ops.h" +#include "htp/gated-delta-net-ops.h" +#include "htp/get-rows-ops.h" +#include "htp/glu-ops.h" +#include "htp/im2col-ops.h" +#include "htp/lightning-indexer-ops.h" +#include "htp/matmul-ops.h" +#include "htp/pad-ops.h" +#include "htp/pool-ops.h" +#include "htp/repeat-ops.h" +#include "htp/roll-ops.h" +#include "htp/rope-ops.h" +#include "htp/set-rows-ops.h" +#include "htp/softmax-ops.h" +#include "htp/solve-tri-ops.h" +#include "htp/ssm-conv.h" +#include "htp/sum-rows-ops.h" +#include "htp/top-k-mask-ops.h" +#include "htp/unary-ops.h" +#include "htp/upscale-ops.h" + +#include <type_traits> +#include <utility> + +bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride); +bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst); +bool htp_matmul_add_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * src2, const struct ggml_tensor * dst); +ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t); + +template <typename... Args> +auto htp_kparams_type(bool (*)(Args...)) -> std::remove_pointer_t<std::remove_reference_t<decltype((std::declval<Args>(), ...))>>; + +template <auto SolveFn, typename... Args> +static inline bool htp_kparams_check(Args &&... args) { + decltype(htp_kparams_type(SolveFn)) kparams; + return SolveFn(std::forward<Args>(args)..., &kparams); +} + +struct htp_opnode; + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + struct htp_opnode & node +); + +void htp_opnode_collapse( + const struct ggml_hexagon_session * sess, + struct htp_opnode & node +); + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + void * kparams = nullptr +); + +static inline bool htp_kparams_check( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op +) { + return htp_kparams_solve(sess, op, nullptr); +} + +bool htp_kparams_solve_allreduce( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * dst, + uint32_t rank, + uint32_t n_ranks, + bool has_add, + bool is_row_bcast, + bool is_shard_ok, + struct htp_allreduce_kernel_params * kparams +); + +bool htp_kparams_solve_rms_norm_mul( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +); + +bool htp_kparams_solve_rms_norm_mul_add( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +); + +bool htp_kparams_solve_matmul_add( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * src2, + const struct ggml_tensor * dst, + struct htp_mm_kernel_params * kparams +); + +bool htp_kparams_solve_mmnx( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + int32_t n_weights, + struct htp_mm_kernel_params * kparams +); + +bool htp_kparams_solve_mmidnx( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + const struct ggml_tensor * dst, + int32_t n_weights, + struct htp_mm_kernel_params * kparams +); + +bool htp_kparams_solve_repeat_get_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_get_rows_kernel_params * kparams +); + +bool htp_kparams_solve_unary_mul_glu( + const struct ggml_hexagon_session * sess, + uint32_t op, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_glu_kernel_params * kparams +); + +bool htp_kparams_solve_upscale( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * dst, + struct htp_upscale_kernel_params * kparams +); + +static inline bool htp_kparams_solve_upscale( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_upscale_kernel_params * kparams +) { + return htp_kparams_solve_upscale(sess, op->src[0], op, kparams); +} + +#endif // HTP_KPARAMS_H diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp new file mode 100644 index 000000000..8dffedc4b --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-opnode.cpp @@ -0,0 +1,304 @@ +#include "htp-opnode.h" + +#include "htp/allreduce-ops.h" +#include "htp/argsort-ops.h" +#include "htp/binary-ops.h" +#include "htp/cumsum-ops.h" +#include "htp/diag-ops.h" +#include "htp/dsv4-hc-ops.h" +#include "htp/fill-ops.h" +#include "htp/flash-attn-ops.h" +#include "htp/gated-delta-net-ops.h" +#include "htp/get-rows-ops.h" +#include "htp/glu-ops.h" +#include "htp/lightning-indexer-ops.h" +#include "htp/matmul-ops.h" +#include "htp/pad-ops.h" +#include "htp/repeat-ops.h" +#include "htp/roll-ops.h" +#include "htp/softmax-ops.h" +#include "htp/solve-tri-ops.h" +#include "htp/ssm-conv.h" +#include "htp/sum-rows-ops.h" +#include "htp/top-k-mask-ops.h" +#include "htp/unary-ops.h" +#include "htp/upscale-ops.h" + +#include <stdio.h> + +std::string htp_opnode::name() const { + switch (opcode) { + case HTP_OP_MDEV_GROUP: return "MDEV_GROUP"; + case HTP_OP_CPY_FENCE: return "CPY+FENCE"; + case HTP_OP_FENCE: return (node && node->op_params[1] == 0) ? "FENCE_WAIT" : "FENCE_SIGNAL"; + case HTP_OP_ALLREDUCE: return "ALLREDUCE"; + case HTP_OP_ALLREDUCE_ADD: return "ALLREDUCE+ADD"; + case HTP_OP_HADAMARD: return "MUL_MAT_HADAMARD"; + case HTP_OP_MUL_MAT_NX: return "MUL_MAT_NX"; + case HTP_OP_MUL_MAT_ID_NX: return "MUL_MAT_ID_NX"; + case HTP_OP_ADD: + case HTP_OP_SUB: + case HTP_OP_MUL: + case HTP_OP_DIV: + if (!fused.empty()) { + const auto * kp = kparams<htp_binary_kernel_params>(); + if (kp->kernel_type != HTP_BINARY_KERNEL_SCALAR) { + switch (opcode) { + case HTP_OP_ADD: return "ADD_NX"; + case HTP_OP_SUB: return "SUB_NX"; + case HTP_OP_MUL: return "MUL_NX"; + case HTP_OP_DIV: return "DIV_NX"; + default: break; + } + } + } + break; + default: + break; + } + + if (!node) { + return ""; + } + if (htp_is_hadamard(node)) { + return "MUL_MAT_HADAMARD"; + } + + std::string res = ggml_op_desc(node); + for (const auto * t : fused) { + res += "+"; + res += ggml_op_desc(t); + } + return res; +} + +static const char * format_tensor_name(const struct ggml_tensor * t) { + return t ? t->name : "NONE"; +} + +static std::string format_tensor_dims(const struct ggml_tensor * t) { + if (!t) { + return "NONE"; + } + char buf[64]; + if (t->ne[2] == 1 && t->ne[3] == 1) { + snprintf(buf, sizeof(buf), "%d:%d", (int) t->ne[0], (int) t->ne[1]); + } else { + snprintf(buf, sizeof(buf), "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]); + } + return buf; +} + +static const char * format_tensor_type(const struct ggml_tensor * t) { + return t ? ggml_type_name(t->type) : "NONE"; +} + +static std::string format_tensor_strides(const struct ggml_tensor * t) { + if (!t) { + return "NONE"; + } + char buf[64]; + const char * c = ggml_is_contiguous(t) ? "" : "!"; + if (t->ne[2] == 1 && t->ne[3] == 1) { + snprintf(buf, sizeof(buf), "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c); + } else { + snprintf(buf, sizeof(buf), "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c); + } + return buf; +} + +static const char * format_tensor_buff(const struct ggml_tensor * t) { + if (t && t->buffer) { + return ggml_backend_buffer_name(t->buffer); + } + return "NONE"; +} + +template <typename Fn> +static std::string format_op_tensors(const htp_opnode & node, Fn fn) { + std::string res; + const auto & inputs = node.get_inputs(); + const auto & outputs = node.get_outputs(); + + if (!inputs.empty()) { + for (size_t i = 0; i < inputs.size(); i++) { + if (i > 0) res += " x "; + res += fn(inputs[i]); + } + res += " -> "; + } + + if (outputs.empty()) { + res += fn(node.dst()); + } else { + for (size_t i = 0; i < outputs.size(); i++) { + if (i > 0) res += " x "; + res += fn(outputs[i]); + } + } + return res; +} + +static std::string format_kernel_params(const htp_opnode & node) { + char str[128]; + const size_t max_size = sizeof(str); + + if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || + node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX || + node.opcode == HTP_OP_MUL_MAT_ADD) { + const auto * kparams = node.kparams<htp_mm_kernel_params>(); + const char * path = "unknown"; + int32_t type = kparams->kernel_type; + if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_BATCHED) { + path = (kparams->tile_size > 0) ? "hmx-tiled" : "hmx-flat"; + } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM || + type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM) { + path = "hvx-flat"; + } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { + path = "hvx-tiled"; + } + snprintf(str, max_size, "%s nhvx %u vtcm %u", path, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { + const auto * kparams = node.kparams<htp_fa_kernel_params>(); + const char * path = "unknown"; + int32_t type = kparams->kernel_type; + if (type == HTP_FA_KERNEL_HMX) { + path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq"; + } else if (type == HTP_FA_KERNEL_HVX) { + path = "hvx"; + } + snprintf(str, max_size, "%s nhvx %u vtcm %u", path, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (htp_op_is_unary(node.opcode)) { + const auto * kparams = node.kparams<htp_unary_kernel_params>(); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->col_tile ? "wide-row" : "row-block", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) { + snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]); + } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) { + snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]); + } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) { + snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]); + } else if (node.opcode == HTP_OP_SSM_CONV) { + const auto * kparams = node.kparams<htp_ssm_conv_kernel_params>(); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->n_t == 1 ? "decode" : "prefill", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_SOFTMAX) { + const auto * kparams = node.kparams<htp_softmax_kernel_params>(); + snprintf(str, max_size, "k%d nhvx %u vtcm %u", (int) kparams->kernel_type, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { + const auto * kparams = node.kparams<htp_gdn_kernel_params>(); + const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent"; + snprintf(str, max_size, "%s-%s nhvx %u vtcm %u", path, kparams->kda ? "kda" : "scalar", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_DSV4_HC_PRE) { + const auto * kparams = node.kparams<htp_dsv4_hc_pre_kernel_params>(); + const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_PRE_KERNEL_GATED ? "gated" : "plain"; + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_DSV4_HC_POST) { + const auto * kparams = node.kparams<htp_dsv4_hc_post_kernel_params>(); + const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity"; + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + const auto * kparams = node.kparams<htp_lid_kernel_params>(); + snprintf(str, max_size, "%s nhvx %u tok %u bc %u vtcm %u", + kparams->kernel_type == HTP_LID_KERNEL_HMX ? "hmx" : "hvx", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->n_tokens_tile, + (unsigned int) kparams->kv_block, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || + node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { + const auto * kparams = node.kparams<htp_binary_kernel_params>(); + snprintf(str, max_size, "nhvx %u vtcm %u", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) { + const auto * kparams = node.kparams<htp_sort_kernel_params>(); + snprintf(str, max_size, "%s nhvx %u nchk %u chk %u vtcm %u", + node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->n_chunks, + (unsigned int) kparams->chunk_elems, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_GET_ROWS) { + const auto * kparams = node.kparams<htp_get_rows_kernel_params>(); + const char * ktype_str = "unknown"; + switch (kparams->kernel_type) { + case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break; + case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break; + case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break; + } + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (htp_op_is_glu(node.opcode)) { + const auto * kparams = node.kparams<htp_glu_kernel_params>(); + snprintf(str, max_size, "nhvx %u vtcm %u", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_PAD) { + const auto * kparams = node.kparams<htp_pad_kernel_params>(); + snprintf(str, max_size, "k%d nhvx %u vtcm %u", (int) kparams->kernel_type, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_CUMSUM) { + const auto * kparams = node.kparams<htp_cumsum_kernel_params>(); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->use_dma ? "dma" : "hvx", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_TOP_K_MASK) { + const auto * kparams = node.kparams<htp_top_k_mask_kernel_params>(); + const char * ktype_str = "?"; + switch (kparams->kernel_type) { + case HTP_TOP_K_MASK_KERNEL_ROWS: ktype_str = "rows"; break; + case HTP_TOP_K_MASK_KERNEL_PICKS: ktype_str = "picks"; break; + } + snprintf(str, max_size, "%s nhvx %u tile %u x%u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->tile_cols, (unsigned int) kparams->n_col_tiles, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_FILL) { + const auto * kparams = node.kparams<htp_fill_kernel_params>(); + snprintf(str, max_size, "%s nhvx %u", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (unsigned int) kparams->n_hvx); + } else if (node.opcode == HTP_OP_ROLL) { + const auto * kparams = node.kparams<htp_roll_kernel_params>(); + snprintf(str, max_size, "k%d nhvx %u", (int) kparams->kernel_type, (unsigned int) kparams->n_hvx); + } else if (node.opcode == HTP_OP_DIAG) { + const auto * kparams = node.kparams<htp_diag_kernel_params>(); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->use_dma ? "dma" : "hvx", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_SOLVE_TRI) { + const auto * kparams = node.kparams<htp_solve_tri_kernel_params>(); + snprintf(str, max_size, "%s nhvx %u", kparams->batched ? "batched" : "chunked", (unsigned int) kparams->n_hvx); + } else if (node.opcode == HTP_OP_UPSCALE) { + const auto * kparams = node.kparams<htp_upscale_kernel_params>(); + const char * ktype_str = "unknown"; + switch (kparams->kernel_type) { + case HTP_UPSCALE_KERNEL_NEAREST: ktype_str = "nearest"; break; + case HTP_UPSCALE_KERNEL_NEAREST_2X: ktype_str = "nearest-2x"; break; + case HTP_UPSCALE_KERNEL_NEAREST_4X: ktype_str = "nearest-4x"; break; + case HTP_UPSCALE_KERNEL_BILINEAR: ktype_str = kparams->align_corners ? "bilinear-ac" : "bilinear"; break; + case HTP_UPSCALE_KERNEL_BILINEAR_2X: ktype_str = "bilinear-2x"; break; + case HTP_UPSCALE_KERNEL_BILINEAR_AA: ktype_str = "bilinear-aa"; break; + case HTP_UPSCALE_KERNEL_BICUBIC: ktype_str = "bicubic"; break; + } + snprintf(str, max_size, "%s nhvx %u chk %u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size); + } else { + snprintf(str, max_size, "----"); + } + return str; +} + +static void append_section(std::string & res, const std::string & sec) { + if (!res.empty()) { + res += "|"; + } + res += sec; +} + +std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags) { + std::string res; + if (flags & HTP_OPFMT_NAMES) append_section(res, format_tensor_name(t)); + if (flags & HTP_OPFMT_DIMS) append_section(res, format_tensor_dims(t)); + if (flags & HTP_OPFMT_TYPES) append_section(res, format_tensor_type(t)); + if (flags & HTP_OPFMT_STRIDES) append_section(res, format_tensor_strides(t)); + if (flags & HTP_OPFMT_BUFFS) append_section(res, format_tensor_buff(t)); + return res; +} + +std::string htp_format_op(const htp_opnode & node, uint32_t flags) { + std::string res; + if (flags & HTP_OPFMT_OP) append_section(res, node.name()); + if (flags & HTP_OPFMT_NAMES) append_section(res, format_op_tensors(node, format_tensor_name)); + if (flags & HTP_OPFMT_DIMS) append_section(res, format_op_tensors(node, format_tensor_dims)); + if (flags & HTP_OPFMT_TYPES) append_section(res, format_op_tensors(node, format_tensor_type)); + if (flags & HTP_OPFMT_STRIDES) append_section(res, format_op_tensors(node, format_tensor_strides)); + if (flags & HTP_OPFMT_BUFFS) append_section(res, format_op_tensors(node, format_tensor_buff)); + if (flags & HTP_OPFMT_KPARAMS) append_section(res, format_kernel_params(node)); + return res; +} diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index 8ff38545c..a30c5b600 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -1,38 +1,74 @@ #ifndef HTP_OPNODE_H #define HTP_OPNODE_H -#define GGML_COMMON_IMPL_CPP #include "ggml-backend-impl.h" -#include "ggml-common.h" +#include "ggml-impl.h" +#include "htp/htp-ops.h" -#include <algorithm> +#include <cstring> +#include <initializer_list> +#include <memory> #include <string> #include <vector> -#include <memory> -#include <stdio.h> -#include "htp-ops.h" -#include "htp/matmul-ops.h" -#include "htp/flash-attn-ops.h" -#include "htp/unary-ops.h" -#include "htp/binary-ops.h" -#include "htp/allreduce-ops.h" -#include "htp/ssm-conv.h" -#include "htp/gated-delta-net-ops.h" -#include "htp/softmax-ops.h" -#include "htp/argsort-ops.h" -#include "htp/get-rows-ops.h" + +#define HTP_OP_MAX_FUSED 16 + +// local std::inplace_vector replacement +template <typename T, size_t MAX_N> +struct htp_opvec { + T data[MAX_N] {}; + size_t n = 0; + + htp_opvec() = default; + htp_opvec(std::initializer_list<T> init) { + GGML_ASSERT(init.size() <= MAX_N); + for (const auto & v : init) { + data[n++] = v; + } + } + + size_t size() const { return n; } + bool empty() const { return n == 0; } + void clear() { n = 0; } + T * begin() { return data; } + T * end() { return data + n; } + const T * begin() const { return data; } + const T * end() const { return data + n; } + T & back() { return data[n - 1]; } + const T & back() const { return data[n - 1]; } + T & operator[](size_t i) { return data[i]; } + const T & operator[](size_t i) const { return data[i]; } + + void push_back(T val) { + GGML_ASSERT(n < MAX_N); + data[n++] = val; + } + + void insert(T * pos, T val) { + GGML_ASSERT(n < MAX_N); + for (T * p = data + n; p > pos; --p) { + *p = *(p - 1); + } + *pos = val; + n++; + } +}; + +static inline bool htp_is_hadamard(const struct ggml_tensor * t) { + return t && t->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD; +} struct htp_opnode { - ggml_tensor * node { nullptr }; - htp_op_code opcode { HTP_OP_INVALID }; - int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0}; + const ggml_tensor * node { nullptr }; + htp_op_code opcode { HTP_OP_INVALID }; + int32_t params[HTP_OP_MAX_PARAMS] {0}; + int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0}; - std::vector<ggml_tensor *> fused; - std::vector<std::shared_ptr<ggml_tensor>> dummy; + std::vector<std::shared_ptr<ggml_tensor>> dummy; - std::vector<const ggml_tensor *> inputs; - std::vector<const ggml_tensor *> outputs; - std::string name; + htp_opvec<const ggml_tensor *, HTP_OP_MAX_FUSED> fused; + htp_opvec<const ggml_tensor *, HTP_OP_MAX_INPUTS> inputs; + htp_opvec<const ggml_tensor *, HTP_OP_MAX_OUTPUTS> outputs; int n_active_src(const ggml_tensor * t) const { if (!t) return 0; @@ -44,28 +80,36 @@ struct htp_opnode { return 0; } - void init(ggml_tensor * node) { + void init(const ggml_tensor * node) { + static_assert(sizeof(this->params) == sizeof(this->node->op_params), "params size mismatch"); + this->node = node; - if (this->node) { - this->name = ggml_op_desc(this->node); + if (!node) return; - // Build inputs (preserving optional nullptrs) + memcpy(this->params, this->node->op_params, sizeof(this->params)); + + // Build inputs (preserving optional nullptrs) + if (htp_is_hadamard(this->node)) { + this->inputs.push_back(this->node->src[1]); + } else { int n_inputs = n_active_src(this->node); - this->inputs.resize(n_inputs, nullptr); for (int i = 0; i < n_inputs; i++) { - this->inputs[i] = this->node->src[i]; + this->inputs.push_back(this->node->src[i]); } - - // Build outputs - this->outputs.push_back(this->dst()); } + + // Build outputs + this->outputs.push_back(this->dst()); } - htp_opnode(htp_op_code opcode = HTP_OP_INVALID, ggml_tensor * node = nullptr) : opcode(opcode) { + htp_opnode(htp_op_code opcode = HTP_OP_INVALID, const ggml_tensor * node = nullptr) : opcode(opcode) { + init(node); + } + + htp_opnode(const ggml_tensor * node) : opcode(HTP_OP_INVALID) { init(node); } - ggml_op op() const { return node->op; } const ggml_tensor * src0() const { return node->src[0]; } const ggml_tensor * src1() const { return node->src[1]; } const ggml_tensor * dst() const { return outputs.empty() ? node : outputs.back(); } @@ -75,337 +119,54 @@ struct htp_opnode { return dummy.back().get(); } - void add_fused(ggml_tensor * t, bool extra_dst = false) { - fused.push_back(t); - - name += "+"; - name += ggml_op_desc(t); - - if (extra_dst) { - outputs.push_back(t); - } else { - outputs.clear(); - outputs.push_back(t); - } - - // Remove the newly fused intermediate output tensor t from inputs (if it was there) - inputs.erase(std::remove(inputs.begin(), inputs.end(), t), inputs.end()); - - // Append new inputs from t, preserving middle nullptrs - int n_inputs = n_active_src(t); - for (int i = 0; i < n_inputs; i++) { - const auto * src = t->src[i]; - if (!src) { - inputs.push_back(nullptr); - } else if (src != node && - std::find(fused.begin(), fused.end(), src) == fused.end() && - std::find(inputs.begin(), inputs.end(), src) == inputs.end()) { - inputs.push_back(src); - } - } - } - - const std::vector<const ggml_tensor *> & get_inputs() const { + const htp_opvec<const ggml_tensor *, HTP_OP_MAX_INPUTS> & get_inputs() const { return inputs; } - const std::vector<const ggml_tensor *> & get_outputs() const { + const htp_opvec<const ggml_tensor *, HTP_OP_MAX_OUTPUTS> & get_outputs() const { return outputs; } - std::string op_name() const { - return name; + template <typename T> + T * kparams() { + static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS"); + static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t"); + return reinterpret_cast<T *>(kernel_params); } - bool is_empty() const { - return ggml_op_is_empty(node->op); + template <typename T> + const T * kparams() const { + static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS"); + static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t"); + return reinterpret_cast<const T *>(kernel_params); } - bool stackable() const { - switch (this->op()) { - case GGML_OP_MUL_MAT: - case GGML_OP_MUL_MAT_ID: - return ggml_is_quantized(this->src0()->type); - default: - return false; - } + template <typename T> + void set_kparams(const T & kp) { + static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS"); + static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t"); + memcpy(kernel_params, &kp, sizeof(T)); } - bool same_input(const htp_opnode& n) const { - return n.src1() == this->src1(); - } + std::string name() const; }; -struct htp_opformat { - char strides[64 * GGML_MAX_SRC]; - char dims[64 * GGML_MAX_SRC]; - char types[16 * GGML_MAX_SRC]; - char buffs[64 * GGML_MAX_SRC]; - char names[64 * GGML_MAX_SRC]; - char kparams[128]; - - int format_tensor_dims(char * str, size_t max_size, const struct ggml_tensor * t) { - if (!t) { - return snprintf(str, max_size, "NONE"); - } - if (t->ne[2] == 1 && t->ne[3] == 1) { - return snprintf(str, max_size, "%d:%d", (int) t->ne[0], (int) t->ne[1]); - } else { - return snprintf(str, max_size, "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]); - } - } - - void format_op_dims(char * str, size_t max_size, const htp_opnode & node) { - char * p = str; - char * p_end = str + max_size; - auto inputs = node.get_inputs(); - - if (!inputs.empty()) { - p += std::min((size_t)format_tensor_dims(p, p_end - p, inputs[0]), (size_t)(p_end - p)); - - for (size_t i = 1; i < inputs.size(); i++) { - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p)); - } - if (p < p_end) { - p += std::min((size_t)format_tensor_dims(p, p_end - p, inputs[i]), (size_t)(p_end - p)); - } - } - - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p)); - } - } - - char self[64]; - format_tensor_dims(self, sizeof(self), node.dst()); - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", self), (size_t)(p_end - p)); - } - } - - int format_tensor_strides(char * str, size_t max_size, const struct ggml_tensor * t) { - if (!t) { - return snprintf(str, max_size, "NONE"); - } - const char * c = ggml_is_contiguous(t) ? "" : "!"; - - if (t->ne[2] == 1 && t->ne[3] == 1) { - return snprintf(str, max_size, "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c); - } else { - return snprintf(str, max_size, "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c); - } - } - - void format_op_strides(char * str, size_t max_size, const htp_opnode & node) { - char * p = str; - char * p_end = str + max_size; - auto inputs = node.get_inputs(); - - if (!inputs.empty()) { - p += std::min((size_t)format_tensor_strides(p, p_end - p, inputs[0]), (size_t)(p_end - p)); - - for (size_t i = 1; i < inputs.size(); i++) { - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p)); - } - if (p < p_end) { - p += std::min((size_t)format_tensor_strides(p, p_end - p, inputs[i]), (size_t)(p_end - p)); - } - } - - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p)); - } - } - - char self[64]; - format_tensor_strides(self, sizeof(self), node.dst()); - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", self), (size_t)(p_end - p)); - } - } - - void format_op_types(char * str, size_t max_size, const htp_opnode & node) { - char * p = str; - char * p_end = str + max_size; - auto inputs = node.get_inputs(); - - if (!inputs.empty()) { - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[0] ? ggml_type_name(inputs[0]->type) : "NONE"), (size_t)(p_end - p)); - } - - for (size_t i = 1; i < inputs.size(); i++) { - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p)); - } - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[i] ? ggml_type_name(inputs[i]->type) : "NONE"), (size_t)(p_end - p)); - } - } - - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p)); - } - } - - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", ggml_type_name(node.dst()->type)), (size_t)(p_end - p)); - } - } - - const char * tensor_buff_name(const struct ggml_tensor * t) { - if (t && t->buffer) { - return ggml_backend_buffer_name(t->buffer); - } - return "NONE"; - } - - void format_op_buffs(char * str, size_t max_size, const htp_opnode & node) { - char * p = str; - char * p_end = str + max_size; - auto inputs = node.get_inputs(); - - if (!inputs.empty()) { - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(inputs[0])), (size_t)(p_end - p)); - } - - for (size_t i = 1; i < inputs.size(); i++) { - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p)); - } - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(inputs[i])), (size_t)(p_end - p)); - } - } - - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p)); - } - } - - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(node.dst())), (size_t)(p_end - p)); - } - } - - void format_op_names(char * str, size_t max_size, const htp_opnode & node) { - char * p = str; - char * p_end = str + max_size; - auto inputs = node.get_inputs(); - - if (!inputs.empty()) { - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[0] ? inputs[0]->name : "NONE"), (size_t)(p_end - p)); - } - - for (size_t i = 1; i < inputs.size(); i++) { - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p)); - } - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[i] ? inputs[i]->name : "NONE"), (size_t)(p_end - p)); - } - } - - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p)); - } - } - - if (p < p_end) { - p += std::min((size_t)snprintf(p, p_end - p, "%s", node.dst()->name), (size_t)(p_end - p)); - } - } - void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) { - if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || - node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX || - node.opcode == HTP_OP_MUL_MAT_ADD) { - const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params; - const char * path = "unknown"; - int32_t type = kparams->kernel_type; - if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_F16_BATCHED) { - path = "hmx-tiled"; - } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM || - type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { - path = "hvx-tiled"; - } - snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); - } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { - const auto * kparams = (const struct htp_fa_kernel_params *) node.kernel_params; - const char * path = "unknown"; - int32_t type = kparams->kernel_type; - if (type == HTP_FA_KERNEL_HMX) { - path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq"; - } else if (type == HTP_FA_KERNEL_HVX) { - path = "hvx"; - } - snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); - } else if (htp_op_is_unary(node.opcode)) { - const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params; - snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size); - } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) { - snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]); - } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) { - snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]); - } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) { - snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]); - } else if (node.opcode == HTP_OP_SSM_CONV) { - const auto * kparams = (const struct htp_ssm_conv_kernel_params *) node.kernel_params; - snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size); - } else if (node.opcode == HTP_OP_SOFTMAX) { - const auto * kparams = (const struct htp_softmax_kernel_params *) node.kernel_params; - snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_id, (int) kparams->n_threads, (int) kparams->vtcm_size); - } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { - const auto * kparams = (const struct htp_gdn_kernel_params *) node.kernel_params; - const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent"; - snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar", - (unsigned int) (kparams->vtcm_size ? kparams->vtcm_size : kparams->vtcm_per_thread * kparams->n_threads)); - } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || - node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { - const auto * kparams = (const struct htp_binary_kernel_params *) node.kernel_params; - snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size); - } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) { - const auto * kparams = (const struct htp_sort_kernel_params *) node.kernel_params; - snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d", - node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort", - (int) kparams->n_threads, (int) kparams->n_chunks, - (int) kparams->chunk_elems, (int) kparams->vtcm_size); - } else if (node.opcode == HTP_OP_GET_ROWS) { - const auto * kparams = (const struct htp_get_rows_kernel_params *) node.kernel_params; - const char * ktype_str = "unknown"; - switch (kparams->kernel_type) { - case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break; - case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break; - case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break; - } - snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size); - } else { - snprintf(str, max_size, "----"); - } - - } - - void format(const htp_opnode & node) { - format_op_dims(dims, sizeof(dims), node); - format_op_strides(strides, sizeof(strides), node); - format_op_types(types, sizeof(types), node); - format_op_buffs(buffs, sizeof(buffs), node); - format_op_names(names, sizeof(names), node); - format_kernel_params(kparams, sizeof(kparams), node); - } - - htp_opformat() { - strides[0] = '\0'; - dims[0] = '\0'; - types[0] = '\0'; - buffs[0] = '\0'; - names[0] = '\0'; - kparams[0] = '\0'; - } - htp_opformat(const htp_opnode & node) { format(node); } +enum htp_opformat_flags { + HTP_OPFMT_OP = (1 << 0), + HTP_OPFMT_NAMES = (1 << 1), + HTP_OPFMT_DIMS = (1 << 2), + HTP_OPFMT_TYPES = (1 << 3), + HTP_OPFMT_STRIDES = (1 << 4), + HTP_OPFMT_BUFFS = (1 << 5), + HTP_OPFMT_KPARAMS = (1 << 6), + + HTP_OPFMT_EXEC = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS | HTP_OPFMT_KPARAMS, + HTP_OPFMT_SUPP = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS, + HTP_OPFMT_PROF = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_KPARAMS, + HTP_OPFMT_TENSOR = HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES, }; +std::string htp_format_op(const htp_opnode & node, uint32_t flags = HTP_OPFMT_EXEC); +std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags = HTP_OPFMT_TENSOR); + #endif // HTP_OPNODE_H diff --git a/ggml/src/ggml-hexagon/htp-repack.cpp b/ggml/src/ggml-hexagon/htp-repack.cpp new file mode 100644 index 000000000..b5413791c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-repack.cpp @@ -0,0 +1,1805 @@ +#include "htp-repack.h" + +#define GGML_COMMON_IMPL_CPP +#include "ggml-common.h" +#include "ggml-impl.h" +#include "ggml-quants.h" + +#include <algorithm> +#include <cmath> +#include <cstring> +#include <initializer_list> +#include <thread> + +#if defined(__ARM_NEON) || defined(__aarch64__) +# include <arm_neon.h> +#endif + +// --- 4-bit block-32 nibble helpers (shared by Q4_0, IQ4_NL, Q4_1, Q5_0, Q5_1, MXFP4) --- + +static inline void unpack_nibbles_32(uint8_t * dst, const uint8_t * qs) { + for (unsigned int i = 0; i < 16; ++i) { + dst[i + 0] = qs[i] & 0x0F; + dst[i + 16] = qs[i] >> 4; + } +} + +static inline void pack_nibbles_32(uint8_t * qs, const uint8_t * src) { + for (unsigned int i = 0; i < 16; ++i) { + qs[i] = src[i + 0] | (src[i + 16] << 4); + } +} + +static inline void transpose_nibbles_to_tile(uint8_t * tile_dst, const uint8_t tile_quants[32][32]) { + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } +} + +static inline void transpose_tile_to_nibbles(uint8_t tile_quants[32][32], const uint8_t * tile_src) { + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + const uint8_t val = tile_src[cp * 32 + row]; + tile_quants[row][2 * cp + 0] = val & 0x0F; + tile_quants[row][2 * cp + 1] = val >> 4; + } + } +} + +// 5th-bit plane of one row (qh bit k = quant k), same bit layout as the Q5_K plane +static inline void pack_q5_plane(uint8_t * plane, int row, uint32_t qh) { + for (int c = 0; c < 4; c++) { + uint32_t v = 0; + for (int i = 0; i < 4; i++) { + v |= ((qh >> (8 * i + 2 * c)) & 3) << (2 * i); + } + plane[c * 32 + row] = (uint8_t) v; + } +} + +static inline uint32_t unpack_q5_plane(const uint8_t * plane, int row) { + uint32_t qh = 0; + for (int c = 0; c < 4; c++) { + const uint32_t v = plane[c * 32 + row]; + for (int i = 0; i < 4; i++) { + qh |= ((v >> (2 * i)) & 3) << (8 * i + 2 * c); + } + } + return qh; +} + +#if defined(__ARM_NEON) || defined(__aarch64__) +static void repack_neon_transpose8x8(const uint8x8_t rows[8], uint8x8_t columns[8]) { + const uint8x8x2_t t0 = vtrn_u8(rows[0], rows[1]); + const uint8x8x2_t t1 = vtrn_u8(rows[2], rows[3]); + const uint8x8x2_t t2 = vtrn_u8(rows[4], rows[5]); + const uint8x8x2_t t3 = vtrn_u8(rows[6], rows[7]); + const uint16x4x2_t t4 = vtrn_u16(vreinterpret_u16_u8(t0.val[0]), vreinterpret_u16_u8(t1.val[0])); + const uint16x4x2_t t5 = vtrn_u16(vreinterpret_u16_u8(t0.val[1]), vreinterpret_u16_u8(t1.val[1])); + const uint16x4x2_t t6 = vtrn_u16(vreinterpret_u16_u8(t2.val[0]), vreinterpret_u16_u8(t3.val[0])); + const uint16x4x2_t t7 = vtrn_u16(vreinterpret_u16_u8(t2.val[1]), vreinterpret_u16_u8(t3.val[1])); + const uint32x2x2_t t8 = vtrn_u32(vreinterpret_u32_u16(t4.val[0]), vreinterpret_u32_u16(t6.val[0])); + const uint32x2x2_t t9 = vtrn_u32(vreinterpret_u32_u16(t4.val[1]), vreinterpret_u32_u16(t6.val[1])); + const uint32x2x2_t ta = vtrn_u32(vreinterpret_u32_u16(t5.val[0]), vreinterpret_u32_u16(t7.val[0])); + const uint32x2x2_t tb = vtrn_u32(vreinterpret_u32_u16(t5.val[1]), vreinterpret_u32_u16(t7.val[1])); + columns[0] = vreinterpret_u8_u32(t8.val[0]); + columns[1] = vreinterpret_u8_u32(ta.val[0]); + columns[2] = vreinterpret_u8_u32(t9.val[0]); + columns[3] = vreinterpret_u8_u32(tb.val[0]); + columns[4] = vreinterpret_u8_u32(t8.val[1]); + columns[5] = vreinterpret_u8_u32(ta.val[1]); + columns[6] = vreinterpret_u8_u32(t9.val[1]); + columns[7] = vreinterpret_u8_u32(tb.val[1]); +} + +static uint8x8_t repack_neon_pack_4bit_pairs(uint8x8_t first, uint8x8_t second, bool high) { + const uint8x8x2_t separated = vuzp_u8(first, second); + const uint8x8_t mask = vdup_n_u8(0x0F); + uint8x8_t even = separated.val[0]; + uint8x8_t odd = separated.val[1]; + + if (high) { + even = vshr_n_u8(even, 4); + odd = vshr_n_u8(odd, 4); + } + + even = vand_u8(even, mask); + odd = vand_u8(odd, mask); + return vorr_u8(vshl_n_u8(odd, 4), even); +} + +static void repack_neon_4bit_rows(const uint8_t * qs[8], uint8_t * tile_dst, int row0) { + uint8x8_t low_rows[8]; + uint8x8_t high_rows[8]; + uint8x8_t columns[8]; + + for (int rr = 0; rr < 8; ++rr) { + const uint8x16_t packed = vld1q_u8(qs[rr]); + const uint8x8_t first = vget_low_u8(packed); + const uint8x8_t second = vget_high_u8(packed); + low_rows[rr] = repack_neon_pack_4bit_pairs(first, second, false); + high_rows[rr] = repack_neon_pack_4bit_pairs(first, second, true); + } + + repack_neon_transpose8x8(low_rows, columns); + for (int cp = 0; cp < 8; ++cp) { + vst1_u8(tile_dst + cp * 32 + row0, columns[cp]); + } + + repack_neon_transpose8x8(high_rows, columns); + for (int cp = 0; cp < 8; ++cp) { + vst1_u8(tile_dst + (cp + 8) * 32 + row0, columns[cp]); + } +} + +static void repack_neon_k4_rows(const uint8_t * qs[8], uint8_t * tile_dst, int row0, bool high) { + uint8x8_t first_rows[8]; + uint8x8_t second_rows[8]; + uint8x8_t columns[8]; + + for (int rr = 0; rr < 8; ++rr) { + first_rows[rr] = repack_neon_pack_4bit_pairs(vld1_u8(qs[rr]), vld1_u8(qs[rr] + 8), high); + second_rows[rr] = repack_neon_pack_4bit_pairs(vld1_u8(qs[rr] + 16), vld1_u8(qs[rr] + 24), high); + } + + repack_neon_transpose8x8(first_rows, columns); + for (int cp = 0; cp < 8; ++cp) { + vst1_u8(tile_dst + cp * 32 + row0, columns[cp]); + } + + repack_neon_transpose8x8(second_rows, columns); + for (int cp = 0; cp < 8; ++cp) { + vst1_u8(tile_dst + (cp + 8) * 32 + row0, columns[cp]); + } +} + +static void repack_neon_q8_rows(const uint8_t * qs[8], uint8_t * tile_dst, int row0) { + uint8x8_t rows[8]; + uint8x8_t columns[8]; + + for (int cp = 0; cp < 16; cp += 4) { + for (int rr = 0; rr < 8; ++rr) { + rows[rr] = vld1_u8(qs[rr] + 2 * cp); + } + repack_neon_transpose8x8(rows, columns); + + for (int pair = 0; pair < 4; ++pair) { + const uint8x8x2_t interleaved = vzip_u8(columns[2 * pair], columns[2 * pair + 1]); + uint8_t * dst = tile_dst + (cp + pair) * 64 + 2 * row0; + vst1_u8(dst, interleaved.val[0]); + vst1_u8(dst + 8, interleaved.val[1]); + } + } +} + +static inline void repack_store_u32(uint8_t * dst, uint32_t v) { + memcpy(dst, &v, sizeof(v)); +} + +static inline uint32x4_t repack_neon_transpose_2bit_4x4(uint32x4_t w) { + uint32x4_t t = vandq_u32(veorq_u32(w, vshrq_n_u32(w, 6)), vdupq_n_u32(0x00CC00CCu)); + w = veorq_u32(w, veorq_u32(t, vshlq_n_u32(t, 6))); + t = vandq_u32(veorq_u32(w, vshrq_n_u32(w, 12)), vdupq_n_u32(0x0000F0F0u)); + return veorq_u32(w, veorq_u32(t, vshlq_n_u32(t, 12))); +} + +static inline void repack_neon_q2_0_block(const uint8_t * qs, uint8_t * tile, int row) { + const uint32x4_t w = repack_neon_transpose_2bit_4x4(vreinterpretq_u32_u8(vld1q_u8(qs))); + repack_store_u32(tile + 0 * 128 + 4 * row, vgetq_lane_u32(w, 0)); + repack_store_u32(tile + 1 * 128 + 4 * row, vgetq_lane_u32(w, 1)); + repack_store_u32(tile + 2 * 128 + 4 * row, vgetq_lane_u32(w, 2)); + repack_store_u32(tile + 3 * 128 + 4 * row, vgetq_lane_u32(w, 3)); +} + +static inline void repack_neon_q1_0_block(const uint8_t * qs, uint8_t * tile, int row) { + uint32x4_t w = vreinterpretq_u32_u8(vld1q_u8(qs)); + uint32x4_t t = vandq_u32(veorq_u32(w, vshrq_n_u32(w, 1)), vdupq_n_u32(0x22222222u)); + w = veorq_u32(w, veorq_u32(t, vshlq_n_u32(t, 1))); + t = vandq_u32(veorq_u32(w, vshrq_n_u32(w, 3)), vdupq_n_u32(0x0A0A0A0Au)); + w = veorq_u32(w, veorq_u32(t, vshlq_n_u32(t, 3))); + w = repack_neon_transpose_2bit_4x4(w); + repack_store_u32(tile + 0 * 128 + 4 * row, vgetq_lane_u32(w, 0)); + repack_store_u32(tile + 1 * 128 + 4 * row, vgetq_lane_u32(w, 1)); + repack_store_u32(tile + 2 * 128 + 4 * row, vgetq_lane_u32(w, 2)); + repack_store_u32(tile + 3 * 128 + 4 * row, vgetq_lane_u32(w, 3)); +} + +static inline void repack_neon_pack_2bit_plane(uint8x16_t v0, uint8x16_t v1, uint8_t * dst, int row) { + const uint32x4x2_t z = vuzpq_u32(vreinterpretq_u32_u8(v0), vreinterpretq_u32_u8(v1)); + const uint32x4_t p01 = vorrq_u32(z.val[0], vshlq_n_u32(z.val[1], 2)); + const uint32x2x2_t u = vuzp_u32(vget_low_u32(p01), vget_high_u32(p01)); + const uint32x2_t out = vorr_u32(u.val[0], vshl_n_u32(u.val[1], 4)); + repack_store_u32(dst + 0 * 128 + row * 4, vget_lane_u32(out, 0)); + repack_store_u32(dst + 1 * 128 + row * 4, vget_lane_u32(out, 1)); +} + +static inline void repack_neon_q2_3_k_low2(const uint8_t * qs, int kt_local, uint8_t * lo_pl, int row) { + const uint8_t * qs_sub = qs + (kt_local / 4) * 32; + const int8x16_t shift = vdupq_n_s8(-2 * (kt_local % 4)); + const uint8x16_t mask = vdupq_n_u8(0x03); + const uint8x16_t v0 = vandq_u8(vshlq_u8(vld1q_u8(qs_sub + 0), shift), mask); + const uint8x16_t v1 = vandq_u8(vshlq_u8(vld1q_u8(qs_sub + 16), shift), mask); + repack_neon_pack_2bit_plane(v0, v1, lo_pl, row); +} + +static inline void repack_neon_q3_k_hmask(const uint8_t * hmask, int kt_local, uint8_t * neg_pl, int row) { + static const uint32_t mask_words[4] = { 0x01010101u, 0x02020202u, 0x04040404u, 0x08080808u }; + const uint8x16_t m0 = vreinterpretq_u8_u32(vld1q_u32(mask_words)); + const uint8x16_t m1 = vshlq_n_u8(m0, 4); + const uint8x16_t hbit = vdupq_n_u8((uint8_t) (1u << kt_local)); + const uint8x16_t b0 = vbicq_u8(m0, vtstq_u8(vld1q_u8(hmask + 0), hbit)); + const uint8x16_t b1 = vbicq_u8(m1, vtstq_u8(vld1q_u8(hmask + 16), hbit)); + const uint32x4_t b01 = vreinterpretq_u32_u8(vorrq_u8(b0, b1)); + const uint32x2_t f0 = vorr_u32(vget_low_u32(b01), vget_high_u32(b01)); + const uint32x2_t f1 = vpadd_u32(f0, f0); + repack_store_u32(neg_pl + row * 4, vget_lane_u32(f1, 0)); +} + +static inline void repack_neon_q6_k(const block_q6_K * b, int kt_local, uint8_t * lo_pl, uint8_t * hi_pl, int row) { + const int c = kt_local / 4; + const int g_sb = kt_local % 4; + const uint8_t * ql = b->ql + c * 64 + (g_sb & 1) * 32; + const uint8_t * qh = b->qh + c * 32; + const int8x16_t lshift = vdupq_n_s8((g_sb >= 2) ? -4 : 0); + const int8x16_t hshift = vdupq_n_s8(-2 * g_sb); + const uint8x16_t mask4 = vdupq_n_u8(0x0F); + const uint8x16_t mask2 = vdupq_n_u8(0x03); + + const uint8x16_t lo0 = vandq_u8(vshlq_u8(vld1q_u8(ql + 0), lshift), mask4); + const uint8x16_t lo1 = vandq_u8(vshlq_u8(vld1q_u8(ql + 16), lshift), mask4); + const uint32x4x2_t lz = vuzpq_u32(vreinterpretq_u32_u8(lo0), vreinterpretq_u32_u8(lo1)); + const uint32x4_t lout = vorrq_u32(lz.val[0], vshlq_n_u32(lz.val[1], 4)); + repack_store_u32(lo_pl + 0 * 128 + row * 4, vgetq_lane_u32(lout, 0)); + repack_store_u32(lo_pl + 1 * 128 + row * 4, vgetq_lane_u32(lout, 1)); + repack_store_u32(lo_pl + 2 * 128 + row * 4, vgetq_lane_u32(lout, 2)); + repack_store_u32(lo_pl + 3 * 128 + row * 4, vgetq_lane_u32(lout, 3)); + + const uint8x16_t hi0 = vandq_u8(vshlq_u8(vld1q_u8(qh + 0), hshift), mask2); + const uint8x16_t hi1 = vandq_u8(vshlq_u8(vld1q_u8(qh + 16), hshift), mask2); + repack_neon_pack_2bit_plane(hi0, hi1, hi_pl, row); +} + +static inline void repack_neon_q5_k_qh(const uint8_t * qh, uint8_t hbit, uint8_t * plane, int row) { + static const uint8_t mask_bytes[16] = { 1, 2, 1, 2, 1, 2, 1, 2, 4, 8, 4, 8, 4, 8, 4, 8 }; + const uint8x16_t m0 = vld1q_u8(mask_bytes); + const uint8x16_t m1 = vshlq_n_u8(m0, 4); + const uint8x16_t vbit = vdupq_n_u8(hbit); + const uint8x16_t b0 = vandq_u8(vtstq_u8(vld1q_u8(qh + 0), vbit), m0); + const uint8x16_t b1 = vandq_u8(vtstq_u8(vld1q_u8(qh + 16), vbit), m1); + const uint8x16_t b01 = vorrq_u8(b0, b1); + const uint8x8_t fold = vorr_u8(vget_low_u8(b01), vget_high_u8(b01)); + const uint8x8_t out = vpadd_u8(fold, fold); + plane[0 * 32 + row] = vget_lane_u8(out, 0); + plane[1 * 32 + row] = vget_lane_u8(out, 1); + plane[2 * 32 + row] = vget_lane_u8(out, 2); + plane[3 * 32 + row] = vget_lane_u8(out, 3); +} + +static inline uint16x8_t repack_neon_load_f16x8(const uint16_t * src) { + return vld1q_u16(src); +} + +#if defined(__aarch64__) || defined(__ARM_FP16_FORMAT_IEEE) +static inline uint16x8_t repack_neon_load_f16x8(const float * src) { + const float16x4_t lo = vcvt_f16_f32(vld1q_f32(src)); + const float16x4_t hi = vcvt_f16_f32(vld1q_f32(src + 4)); + return vreinterpretq_u16_f16(vcombine_f16(lo, hi)); +} + +static inline uint16x8_t repack_neon_load_f16x8(const ggml_bf16_t * src) { + const uint16x8_t bf = vld1q_u16((const uint16_t *) src); + const float32x4_t f_lo = vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)); + const float32x4_t f_hi = vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)); + const float16x4_t lo = vcvt_f16_f32(f_lo); + const float16x4_t hi = vcvt_f16_f32(f_hi); + return vreinterpretq_u16_f16(vcombine_f16(lo, hi)); +} +#else +static inline uint16x8_t repack_neon_load_f16x8(const float * src) { + uint16_t tmp[8]; + for (int i = 0; i < 8; ++i) { + tmp[i] = GGML_FP32_TO_FP16(src[i]); + } + return vld1q_u16(tmp); +} + +static inline uint16x8_t repack_neon_load_f16x8(const ggml_bf16_t * src) { + uint16_t tmp[8]; + for (int i = 0; i < 8; ++i) { + tmp[i] = GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(src[i])); + } + return vld1q_u16(tmp); +} +#endif + +template <typename SrcT> +static void repack_neon_f16_tile(uint16_t * tile_dst, const SrcT * src_slice, int64_t row_stride, int ct, int kt) { + for (int row0 = 0; row0 < 32; row0 += 4) { + const SrcT * r0 = src_slice + (int64_t) (ct * 32 + row0 + 0) * row_stride + kt * 32; + const SrcT * r1 = src_slice + (int64_t) (ct * 32 + row0 + 1) * row_stride + kt * 32; + const SrcT * r2 = src_slice + (int64_t) (ct * 32 + row0 + 2) * row_stride + kt * 32; + const SrcT * r3 = src_slice + (int64_t) (ct * 32 + row0 + 3) * row_stride + kt * 32; + for (int cp = 0; cp < 16; cp += 4) { + const uint32x4_t v0 = vreinterpretq_u32_u16(repack_neon_load_f16x8(r0 + 2 * cp)); + const uint32x4_t v1 = vreinterpretq_u32_u16(repack_neon_load_f16x8(r1 + 2 * cp)); + const uint32x4_t v2 = vreinterpretq_u32_u16(repack_neon_load_f16x8(r2 + 2 * cp)); + const uint32x4_t v3 = vreinterpretq_u32_u16(repack_neon_load_f16x8(r3 + 2 * cp)); + + const uint32x2x2_t t0 = vtrn_u32(vget_low_u32(v0), vget_low_u32(v1)); + const uint32x2x2_t t1 = vtrn_u32(vget_high_u32(v0), vget_high_u32(v1)); + const uint32x2x2_t t2 = vtrn_u32(vget_low_u32(v2), vget_low_u32(v3)); + const uint32x2x2_t t3 = vtrn_u32(vget_high_u32(v2), vget_high_u32(v3)); + + vst1q_u16(tile_dst + (cp + 0) * 64 + row0 * 2, vreinterpretq_u16_u32(vcombine_u32(t0.val[0], t2.val[0]))); + vst1q_u16(tile_dst + (cp + 1) * 64 + row0 * 2, vreinterpretq_u16_u32(vcombine_u32(t0.val[1], t2.val[1]))); + vst1q_u16(tile_dst + (cp + 2) * 64 + row0 * 2, vreinterpretq_u16_u32(vcombine_u32(t1.val[0], t3.val[0]))); + vst1q_u16(tile_dst + (cp + 3) * 64 + row0 * 2, vreinterpretq_u16_u32(vcombine_u32(t1.val[1], t3.val[1]))); + } + } +} +#endif + +// --- Generic 32x32 tile grid walkers for block-32 and float types --- + +template <typename Fn> +static void repack_parallel_for_ct(size_t slice_bytes, int n_col_tiles, Fn && fn) { + int n_threads = 1; + if (slice_bytes > 128 * 1024) { + for (int nt : { 4, 3, 2 }) { + if (n_col_tiles % nt == 0) { + n_threads = nt; + break; + } + } + } + + if (n_threads <= 1) { + fn(0, n_col_tiles); + return; + } + + const int chunk = n_col_tiles / n_threads; + std::thread workers[3]; + for (int i = 0; i < n_threads - 1; ++i) { + workers[i] = std::thread([&fn, i, chunk]() { + fn(i * chunk, (i + 1) * chunk); + }); + } + fn((n_threads - 1) * chunk, n_col_tiles); + for (int i = 0; i < n_threads - 1; ++i) { + workers[i].join(); + } +} + +template <typename SrcElemT, int ELEMS_PER_KTILE, typename PackTileFn> +static void repack_to_tiled_grid_32(ggml_tensor * t, const void * data, size_t offset, size_t size, + size_t tile_size, PackTileFn && pack_tile) { + const SrcElemT * src_matrix = (const SrcElemT *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = hex_round_up(ne0, 32) / 32; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + + const size_t slice_size = (size_t) ne1 * ggml_row_size(t->type, ne0); + const int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } + + const int64_t row_stride = (ne0 / (32 / ELEMS_PER_KTILE)); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const SrcElemT * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * row_stride); + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; + + repack_parallel_for_ct(matrix_size, n_col_tiles, [&](int ct0, int ct1) { + for (int ct = ct0; ct < ct1; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + pack_tile(tile_dst, src_slice, ne0, ne1, row_stride, ct, kt); + } + } + }); + } +} + +template <typename DstElemT, int ELEMS_PER_KTILE, typename UnpackTileFn> +static void unpack_from_tiled_grid_32(void * data, const ggml_tensor * t, size_t offset, size_t size, + size_t tile_size, UnpackTileFn && unpack_tile) { + uint8_t * dst_bytes = (uint8_t *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = hex_round_up(ne0, 32) / 32; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + + const size_t row_size_bytes = ggml_row_size(t->type, ne0); + const size_t slice_size = (size_t) ne1 * row_size_bytes; + const int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } + + const int64_t row_stride = (ne0 / (32 / ELEMS_PER_KTILE)); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + const size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; + const size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; + + const int64_t start_row = slice_offset_start / row_size_bytes; + const int64_t end_row = (std::min)((int64_t) ((slice_offset_end + row_size_bytes - 1) / row_size_bytes), ne1); + + const int start_ct = (int) (start_row / 32); + const int end_ct = (std::min)((int) ((end_row + 31) / 32), n_col_tiles); + + DstElemT * dst_slice = (DstElemT *) (dst_bytes + (cur_start_byte - offset)); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_ct; ct < end_ct; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; + unpack_tile(dst_slice, tile_src, ne0, start_row, end_row, row_stride, ct, kt); + } + } + } +} + +// --- Block-32 Quantized Types (Q4_0, IQ4_NL, Q4_1, Q5_0, Q5_1, Q8_0, MXFP4) --- + +static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32<block_q4_0, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_0, + [](uint8_t * tile_dst, const block_q4_0 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + scale_dst[row] = src_slice[(ct * 32 + row) * row_stride + kt].d; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_q4_0 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + scale_dst[row] = b.d; + } else { + memset(tile_quants[row], 8, 32); + scale_dst[row] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32<block_q4_0, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_0, + [](block_q4_0 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q4_0 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + b.d = scale_src[row]; + } + } + }); +} + +static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32<block_q4_1, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_1, + [](uint8_t * tile_dst, const block_q4_1 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + const block_q4_1 & src_block = src_slice[(ct * 32 + row) * row_stride + kt]; + scale_dst[2 * row + 0] = src_block.d; + scale_dst[2 * row + 1] = src_block.m; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_q4_1 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + scale_dst[2 * row + 0] = b.d; + scale_dst[2 * row + 1] = b.m; + } else { + memset(tile_quants[row], 0, 32); + scale_dst[2 * row + 0] = 0; + scale_dst[2 * row + 1] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32<block_q4_1, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_1, + [](block_q4_1 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q4_1 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + b.d = scale_src[2 * row + 0]; + b.m = scale_src[2 * row + 1]; + } + } + }); +} + +static void repack_q5_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32<block_q5_0, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q5_0, + [](uint8_t * tile_dst, const block_q5_0 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + uint8_t * plane_dst = tile_dst + 512; + ggml_half * scale_dst = (ggml_half *) (tile_dst + 640); +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + const block_q5_0 & src_block = src_slice[(ct * 32 + row) * row_stride + kt]; + uint32_t qh; + memcpy(&qh, src_block.qh, sizeof(qh)); + pack_q5_plane(plane_dst, row, ~qh); + scale_dst[row] = src_block.d; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_q5_0 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + uint32_t qh; + memcpy(&qh, b.qh, sizeof(qh)); + pack_q5_plane(plane_dst, row, ~qh); + scale_dst[row] = b.d; + } else { + memset(tile_quants[row], 0, 32); + pack_q5_plane(plane_dst, row, 0); + scale_dst[row] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_q5_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32<block_q5_0, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q5_0, + [](block_q5_0 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const uint8_t * plane_src = tile_src + 512; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 640); + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q5_0 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + const uint32_t qh = ~unpack_q5_plane(plane_src, row); + memcpy(b.qh, &qh, sizeof(qh)); + b.d = scale_src[row]; + } + } + }); +} + +// Q5_1 uses the Q5_K tile layout +static void repack_q5_1_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32<block_q5_1, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q5_1, + [](uint8_t * tile_dst, const block_q5_1 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + uint8_t * plane_dst = tile_dst + 640; +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + const block_q5_1 & src_block = src_slice[(ct * 32 + row) * row_stride + kt]; + uint32_t qh; + memcpy(&qh, src_block.qh, sizeof(qh)); + pack_q5_plane(plane_dst, row, qh); + scale_dst[2 * row + 0] = src_block.d; + scale_dst[2 * row + 1] = src_block.m; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_q5_1 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + uint32_t qh; + memcpy(&qh, b.qh, sizeof(qh)); + pack_q5_plane(plane_dst, row, qh); + scale_dst[2 * row + 0] = b.d; + scale_dst[2 * row + 1] = b.m; + } else { + memset(tile_quants[row], 0, 32); + pack_q5_plane(plane_dst, row, 0); + scale_dst[2 * row + 0] = 0; + scale_dst[2 * row + 1] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_q5_1(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32<block_q5_1, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q5_1, + [](block_q5_1 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + const uint8_t * plane_src = tile_src + 640; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q5_1 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + const uint32_t qh = unpack_q5_plane(plane_src, row); + memcpy(b.qh, &qh, sizeof(qh)); + b.d = scale_src[2 * row + 0]; + b.m = scale_src[2 * row + 1]; + } + } + }); +} + +static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32<block_mxfp4, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_MXFP4, + [](uint8_t * tile_dst, const block_mxfp4 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + uint8_t * scale_dst = tile_dst + 512; +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + scale_dst[row] = src_slice[(ct * 32 + row) * row_stride + kt].e; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_mxfp4 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + scale_dst[row] = b.e; + } else { + memset(tile_quants[row], 0, 32); + scale_dst[row] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32<block_mxfp4, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_MXFP4, + [](block_mxfp4 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const uint8_t * scale_src = tile_src + 512; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_mxfp4 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + b.e = scale_src[row]; + } + } + }); +} + +static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32<block_q8_0, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q8_0, + [](uint8_t * tile_dst, const block_q8_0 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + ggml_half * scale_dst = (ggml_half *) (tile_dst + 1024); +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = (const uint8_t *) src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_q8_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + scale_dst[row] = src_slice[(ct * 32 + row) * row_stride + kt].d; + } + return; + } +#endif + for (int cp = 0; cp < 16; cp++) { + const int col0 = cp * 2; + const int col1 = col0 + 1; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_slice[r * row_stride + kt] : nullptr; + tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0; + tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0; + } + } + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * row_stride + kt].d : 0; + } + }); +} + +static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32<block_q8_0, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q8_0, + [](block_q8_0 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + for (int cp = 0; cp < 16; cp++) { + const int col0 = cp * 2; + const int col1 = col0 + 1; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q8_0 & b = dst_slice[(r - start_row) * row_stride + kt]; + b.qs[col0] = tile_src[cp * 64 + 2 * row + 0]; + b.qs[col1] = tile_src[cp * 64 + 2 * row + 1]; + } + } + } + const ggml_half * scale_src = (const ggml_half *) (tile_src + 1024); + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + dst_slice[(r - start_row) * row_stride + kt].d = scale_src[row]; + } + } + }); +} + +// --- Float Types (F16, F32, BF16 -> f16_tiled HMX layout) --- + +static void repack_q2_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + GGML_ASSERT(offset == 0); + GGML_ASSERT(t->ne[0] % 64 == 0); + GGML_ASSERT(size == ggml_nbytes(t)); + + const block_q2_0 * src = (const block_q2_0 *) data; + const int64_t blocks_per_row = t->ne[0] / 64; + const int64_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32; + const int64_t n_weight_tiles = blocks_per_row; + const size_t matrix_size = (size_t) n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0; + + for (int64_t slice = 0; slice < t->ne[2] * t->ne[3]; ++slice) { + const block_q2_0 * src_slice = src + slice * t->ne[1] * blocks_per_row; + uint8_t * dst_slice = (uint8_t *) t->data + slice * matrix_size; + const size_t ct_bytes = (size_t) n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0; + + repack_parallel_for_ct(matrix_size, (int) n_col_tiles, [&](int ct0, int ct1) { + memset(dst_slice + (size_t) ct0 * ct_bytes, 0, (size_t) (ct1 - ct0) * ct_bytes); + const int64_t r0 = (int64_t) ct0 * 32; + const int64_t r1 = (std::min)((int64_t) ct1 * 32, t->ne[1]); + for (int64_t row_index = r0; row_index < r1; ++row_index) { + const int64_t ct = row_index / 32; + const int row = (int) (row_index % 32); + for (int64_t kt = 0; kt < n_weight_tiles; ++kt) { + const block_q2_0 * block = src_slice + row_index * blocks_per_row + kt; + uint8_t * tile = dst_slice + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0; +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q2_0_block(block->qs, tile, row); +#else + for (int group = 0; group < 16; ++group) { + for (int byte = 0; byte < 4; ++byte) { + const int k = 4 * group + byte; + const uint8_t code = (block->qs[k / 4] >> (2 * (k % 4))) & 3; + tile[(group >> 2) * 128 + 4 * row + byte] |= code << (2 * (group & 3)); + } + } +#endif + memcpy(tile + 512 + 2 * row, &block->d, sizeof(block->d)); + } + } + }); + } +} + +static void repack_tiled_q2_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + GGML_ASSERT(t->ne[0] % 64 == 0); + GGML_ASSERT(offset + size <= ggml_nbytes(t)); + + const size_t first_block = offset / sizeof(block_q2_0); + const size_t end_block = (offset + size + sizeof(block_q2_0) - 1) / sizeof(block_q2_0); + const size_t blocks_per_row = t->ne[0] / 64; + const size_t blocks_per_slice = t->ne[1] * blocks_per_row; + const size_t n_weight_tiles = blocks_per_row; + const size_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32; + const size_t matrix_size = n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0; + + for (size_t block_index = first_block; block_index < end_block; ++block_index) { + const size_t slice = block_index / blocks_per_slice; + const size_t in_slice = block_index % blocks_per_slice; + const size_t row_index = in_slice / blocks_per_row; + const size_t kt = in_slice % blocks_per_row; + const size_t ct = row_index / 32; + const size_t row = row_index % 32; + const uint8_t * tile = (const uint8_t *) t->data + slice * matrix_size + + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0; + + block_q2_0 block = {}; + memcpy(&block.d, tile + 512 + 2 * row, sizeof(block.d)); + for (int group = 0; group < 16; ++group) { + for (int byte = 0; byte < 4; ++byte) { + const uint8_t packed = tile[(group >> 2) * 128 + 4 * row + byte]; + const uint8_t code = (packed >> (2 * (group & 3))) & 3; + block.qs[group] |= code << (2 * byte); + } + } + + const size_t block_begin = block_index * sizeof(block_q2_0); + const size_t copy_begin = (std::max)(block_begin, offset); + const size_t copy_end = (std::min)(block_begin + sizeof(block_q2_0), offset + size); + memcpy((uint8_t *) data + copy_begin - offset, (const uint8_t *) &block + copy_begin - block_begin, copy_end - copy_begin); + } +} + +template <typename SrcT, typename ToFp16Fn> +static void repack_float_to_f16_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size, ToFp16Fn && to_fp16) { + repack_to_tiled_grid_32<SrcT, 32>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_F16, + [&](uint8_t * tile_bytes, const SrcT * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + uint16_t * tile_dst = (uint16_t *) tile_bytes; +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt * 32 + 32 <= ne0) { + repack_neon_f16_tile(tile_dst, src_slice, row_stride, ct, kt); + return; + } +#endif + for (int cp = 0; cp < 16; cp++) { + const int64_t c0 = kt * 32 + cp * 2; + const int64_t c1 = c0 + 1; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + const SrcT * r_ptr = (r < ne1) ? &src_slice[r * row_stride] : nullptr; + tile_dst[cp * 64 + row * 2 + 0] = (r_ptr && c0 < ne0) ? to_fp16(r_ptr[c0]) : 0; + tile_dst[cp * 64 + row * 2 + 1] = (r_ptr && c1 < ne0) ? to_fp16(r_ptr[c1]) : 0; + } + } + }); +} + +template <typename DstT, typename FromFp16Fn> +static void unpack_f16_tiled_to_float(void * data, const ggml_tensor * t, size_t offset, size_t size, FromFp16Fn && from_fp16) { + unpack_from_tiled_grid_32<DstT, 32>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_F16, + [&](DstT * dst_slice, const uint8_t * tile_bytes, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + const uint16_t * tile_src = (const uint16_t *) tile_bytes; + for (int cp = 0; cp < 16; cp++) { + const int64_t c0 = kt * 32 + cp * 2; + const int64_t c1 = c0 + 1; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row) { + DstT * r_ptr = dst_slice + (r - start_row) * row_stride; + if (c0 < ne0) r_ptr[c0] = from_fp16(tile_src[cp * 64 + row * 2 + 0]); + if (c1 < ne0) r_ptr[c1] = from_fp16(tile_src[cp * 64 + row * 2 + 1]); + } + } + } + }); +} + +// --- Generic K-Quant (super-block 256) grid walkers --- + +template <typename BlockT, typename PackKtFn> +static void repack_kquant_to_tiled_grid(ggml_tensor * t, const void * data, size_t offset, size_t size, + size_t tile_size, PackKtFn && pack_kt) { + GGML_ASSERT(offset == 0); + GGML_UNUSED(size); + + const BlockT * src_matrix = (const BlockT *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + GGML_ASSERT(ne0 % QK_K == 0); + + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = hex_round_up(ne0, 32) / 32; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const int64_t sb_per_row = ne0 / QK_K; + + for (int64_t slice_idx = 0; slice_idx < ne2 * ne3; slice_idx++) { + const BlockT * src_slice = src_matrix + slice_idx * (ne1 * sb_per_row); + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; + const size_t ct_bytes = (size_t) n_k_tiles * tile_size; + + repack_parallel_for_ct(matrix_size, n_col_tiles, [&](int ct0, int ct1) { + // Padding rows and OR-ed bit planes require zeroed tiles + memset(matrix_dst + (size_t) ct0 * ct_bytes, 0, (size_t) (ct1 - ct0) * ct_bytes); + + const int64_t r0 = (int64_t) ct0 * 32; + const int64_t r1 = (std::min)((int64_t) ct1 * 32, ne1); + for (int64_t r = r0; r < r1; r++) { + const int ct = (int) (r / 32); + const int row = (int) (r % 32); + const BlockT * src_row = src_slice + r * sb_per_row; + + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + pack_kt(tile, &src_row[kt / 8], row, kt % 8); + } + } + }); + } +} + +template <typename BlockT, typename UnpackSbFn> +static void unpack_kquant_from_tiled_grid(void * data, const ggml_tensor * t, size_t offset, size_t size, + size_t tile_size, UnpackSbFn && unpack_sb) { + uint8_t * dst_bytes = (uint8_t *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + GGML_ASSERT(ne0 % QK_K == 0); + + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = hex_round_up(ne0, 32) / 32; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const int64_t sb_per_row = ne0 / QK_K; + + const size_t row_size_bytes = ggml_row_size(t->type, ne0); + const size_t slice_size = (size_t) ne1 * row_size_bytes; + const int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + const size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; + const size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; + + const int64_t start_row = slice_offset_start / row_size_bytes; + const int64_t end_row = (std::min)((int64_t) ((slice_offset_end + row_size_bytes - 1) / row_size_bytes), ne1); + + BlockT * dst_slice = (BlockT *) (dst_bytes + (cur_start_byte - offset)); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int64_t r = start_row; r < end_row; r++) { + const int ct = (int) (r / 32); + const int row = (int) (r % 32); + BlockT * dst_row = dst_slice + (r - start_row) * sb_per_row; + + for (int64_t sb = 0; sb < sb_per_row; sb++) { + BlockT * b = &dst_row[sb]; + memset(b, 0, sizeof(BlockT)); + const uint8_t * sb_tiles = matrix_src + ((size_t) ct * n_k_tiles + sb * 8) * tile_size; + unpack_sb(b, sb_tiles, tile_size, row); + } + } + } +} + +// --- K-Quant Helpers & Per-Type Implementations (Q2_K, Q3_K, Q4_K, Q5_K, Q6_K) --- + +#if !defined(__ARM_NEON) && !defined(__aarch64__) +// unsigned 6-bit value (0..63) of element e of a Q6_K block, same bit layout as dequantize_row_q6_K +static inline uint8_t q6_K_get_quant(const block_q6_K * b, int e) { + const int c = e / 128; + const int w = e % 128; + const int g = w / 32; + const int l = w % 32; + const uint8_t * ql = b->ql + c * 64; + const uint8_t * qh = b->qh + c * 32; + uint8_t lo, hi; + switch (g) { + case 0: lo = ql[l] & 0xF; hi = (qh[l] >> 0) & 3; break; + case 1: lo = ql[l + 32] & 0xF; hi = (qh[l] >> 2) & 3; break; + case 2: lo = ql[l] >> 4; hi = (qh[l] >> 4) & 3; break; + default: lo = ql[l + 32] >> 4; hi = (qh[l] >> 6) & 3; break; + } + return (uint8_t) (lo | (hi << 4)); +} +#endif + +static void repack_q6_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_kquant_to_tiled_grid<block_q6_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q6_K, + [](uint8_t * tile, const block_q6_K * b, int row, int kt_local) { + const float d = GGML_FP16_TO_FP32(b->d); + uint8_t * lo_pl = tile; + uint8_t * hi_pl = tile + 512; + ggml_half * sc_pl = (ggml_half *) (tile + 768); + +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q6_k(b, kt_local, lo_pl, hi_pl, row); +#else + for (int lk = 0; lk < 32; lk++) { + const uint8_t q6 = q6_K_get_quant(b, kt_local * 32 + lk); + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + lo_pl[(g >> 1) * 128 + pos] |= (uint8_t) ((q6 & 0xF) << ((g & 1) * 4)); + hi_pl[(g >> 2) * 128 + pos] |= (uint8_t) ((q6 >> 4) << ((g & 3) * 2)); + } +#endif + for (int sub = 0; sub < 2; sub++) { + sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) b->scales[kt_local * 2 + sub]); + } + }); +} + +static void repack_tiled_q6_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid<block_q6_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q6_K, + [](block_q6_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + float sub_scales[16]; + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile = sb_tiles + kt_local * tile_size; + const uint8_t * lo_pl = tile; + const uint8_t * hi_pl = tile + 512; + const ggml_half * sc_pl = (const ggml_half *) (tile + 768); + + const int c = kt_local / 4; + const int g = kt_local % 4; + uint8_t * ql = b->ql + c * 64; + uint8_t * qh = b->qh + c * 32; + + for (int lk = 0; lk < 32; lk++) { + const int g_tile = lk >> 2; + const int pos = row * 4 + (lk & 3); + const uint8_t lo = (lo_pl[(g_tile >> 1) * 128 + pos] >> ((g_tile & 1) * 4)) & 0xF; + const uint8_t hi = (hi_pl[(g_tile >> 2) * 128 + pos] >> ((g_tile & 3) * 2)) & 3; + + switch (g) { + case 0: ql[lk] |= lo; qh[lk] |= (hi << 0); break; + case 1: ql[lk + 32] |= lo; qh[lk] |= (hi << 2); break; + case 2: ql[lk] |= (lo << 4); qh[lk] |= (hi << 4); break; + default: ql[lk + 32] |= (lo << 4); qh[lk] |= (hi << 6); break; + } + } + + for (int sub = 0; sub < 2; sub++) { + sub_scales[kt_local * 2 + sub] = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]); + } + } + + float max_abs_scale = 0.0f; + for (int s = 0; s < 16; s++) { + max_abs_scale = (std::max)(max_abs_scale, fabsf(sub_scales[s])); + } + + if (max_abs_scale == 0.0f) { + b->d = GGML_FP32_TO_FP16(0.0f); + } else { + b->d = GGML_FP32_TO_FP16(max_abs_scale / 127.0f); + const float d_actual = GGML_FP16_TO_FP32(b->d); + const float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f; + for (int s = 0; s < 16; s++) { + const int sc = (int) roundf(sub_scales[s] * inv_d); + b->scales[s] = (int8_t) (std::max)(-128, (std::min)(127, sc)); + } + } + }); +} + +#if !defined(__ARM_NEON) && !defined(__aarch64__) +// low 2 bits (0..3) of element e of a Q2_K or Q3_K block, same bit layout as dequantize_row_q2_K / q3_K +static inline uint8_t q2_3_K_get_low2(const uint8_t * qs, int e) { + const int c = e / 128; + const int j = (e % 128) / 32; + const int l = e % 32; + return (qs[c * 32 + l] >> (2 * j)) & 3; +} + +// hmask bit of element e of a Q3_K block, same bit layout as dequantize_row_q3_K +static inline bool q3_K_get_hbit(const block_q3_K * b, int e) { + const int c = e / 128; + const int j = (e % 128) / 32; + const int l = e % 32; + return (b->hmask[l] >> (c * 4 + j)) & 1; +} +#endif + +// signed 6-bit scale j (-32..31) of a Q3_K block, same packing as quantize_row_q3_K_ref +static inline int q3_K_get_scale(const uint8_t * scales, int j) { + const int lo = (j < 8) ? (scales[j] & 0xF) : (scales[j - 8] >> 4); + const int hi = (scales[8 + j % 4] >> (2 * (j / 4))) & 3; + return (lo | (hi << 4)) - 32; +} + +// read-back: find fp16 d and l[j] in [lmin, lmax] with fp16(d * l[j]) == prod[j] for all j, false if none +static bool hexagon_recover_k_scales(const ggml_half * prod, int n, int lmin, int lmax, ggml_half * d_out, int * l_out) { + int jmax = 0; + for (int j = 1; j < n; j++) { + if (fabsf(GGML_FP16_TO_FP32(prod[j])) > fabsf(GGML_FP16_TO_FP32(prod[jmax]))) { + jmax = j; + } + } + const float pmax = GGML_FP16_TO_FP32(prod[jmax]); + if (pmax == 0.0f) { + *d_out = GGML_FP32_TO_FP16(0.0f); + for (int j = 0; j < n; j++) { + l_out[j] = 0; + } + return true; + } + + // the quantizers put the largest scale at or near the range end, so try large |l| first + const int lext = (std::max)(-lmin, lmax); + for (int a = lext; a >= 1; a--) { + for (int sign : { 1, -1 }) { + const int lj = sign * a; + if (lj < lmin || lj > lmax) { + continue; + } + const ggml_half d0 = GGML_FP32_TO_FP16(pmax / (float) lj); + for (int ulp : { 0, -1, 1 }) { + ggml_half d = d0; + uint16_t bits; + memcpy(&bits, &d, sizeof(bits)); + bits = (uint16_t) (bits + ulp); + memcpy(&d, &bits, sizeof(bits)); + + const float df = GGML_FP16_TO_FP32(d); + if (!std::isfinite(df) || df == 0.0f) { + continue; + } + bool ok = true; + for (int j = 0; j < n && ok; j++) { + const int l = (int) roundf(GGML_FP16_TO_FP32(prod[j]) / df); + const ggml_half p = GGML_FP32_TO_FP16(df * (float) l); + ok = l >= lmin && l <= lmax && memcmp(&p, &prod[j], sizeof(p)) == 0; + l_out[j] = l; + } + if (ok) { + *d_out = d; + return true; + } + } + } + } + return false; +} + +static void repack_q3_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_kquant_to_tiled_grid<block_q3_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q3_K, + [](uint8_t * tile, const block_q3_K * b, int row, int kt_local) { + const float d = GGML_FP16_TO_FP32(b->d); + uint8_t * lo_pl = tile; + uint8_t * neg_pl = tile + 256; + ggml_half * sc_pl = (ggml_half *) (tile + 384); + +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q2_3_k_low2(b->qs, kt_local, lo_pl, row); + repack_neon_q3_k_hmask(b->hmask, kt_local, neg_pl, row); +#else + for (int lk = 0; lk < 32; lk++) { + const int e = kt_local * 32 + lk; + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, e) << ((g & 3) * 2)); + if (!q3_K_get_hbit(b, e)) { + neg_pl[pos] |= (uint8_t) (1 << g); + } + } +#endif + for (int sub = 0; sub < 2; sub++) { + sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) q3_K_get_scale(b->scales, kt_local * 2 + sub)); + } + }); +} + +static void repack_tiled_q3_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid<block_q3_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q3_K, + [](block_q3_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + ggml_half sub_scales[16]; + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile = sb_tiles + kt_local * tile_size; + const uint8_t * lo_pl = tile; + const uint8_t * neg_pl = tile + 256; + const ggml_half * sc_pl = (const ggml_half *) (tile + 384); + + for (int lk = 0; lk < 32; lk++) { + const int e = kt_local * 32 + lk; + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + const uint8_t lo = (lo_pl[(g >> 2) * 128 + pos] >> ((g & 3) * 2)) & 3; + + const int c = e / 128; + const int j = (e % 128) / 32; + const int l = e % 32; + b->qs[c * 32 + l] |= (uint8_t) (lo << (2 * j)); + if (!((neg_pl[pos] >> g) & 1)) { + b->hmask[l] |= (uint8_t) (1 << (c * 4 + j)); + } + } + + for (int sub = 0; sub < 2; sub++) { + sub_scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row]; + } + } + + int ls[16]; + if (!hexagon_recover_k_scales(sub_scales, 16, -32, 31, &b->d, ls)) { + float max_scale = 0.0f; + for (int s = 0; s < 16; s++) { + if (fabsf(GGML_FP16_TO_FP32(sub_scales[s])) > fabsf(max_scale)) { + max_scale = GGML_FP16_TO_FP32(sub_scales[s]); + } + } + b->d = GGML_FP32_TO_FP16(-max_scale / 32.0f); + const float d_actual = GGML_FP16_TO_FP32(b->d); + const float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f; + for (int s = 0; s < 16; s++) { + ls[s] = (std::max)(-32, (std::min)(31, (int) roundf(GGML_FP16_TO_FP32(sub_scales[s]) * inv_d))); + } + } + + for (int s = 0; s < 16; s++) { + const int l = ls[s] + 32; + if (s < 8) { + b->scales[s] = l & 0xF; + } else { + b->scales[s - 8] |= (uint8_t) ((l & 0xF) << 4); + } + b->scales[s % 4 + 8] |= (uint8_t) ((l >> 4) << (2 * (s / 4))); + } + }); +} + +static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_kquant_to_tiled_grid<block_q2_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q2_K, + [](uint8_t * tile, const block_q2_K * b, int row, int kt_local) { + uint8_t * lo_pl = tile; + uint8_t * sc_pl = tile + 256; + ggml_half * dd_pl = (ggml_half *) (tile + 320); + +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q2_3_k_low2(b->qs, kt_local, lo_pl, row); +#else + for (int lk = 0; lk < 32; lk++) { + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, kt_local * 32 + lk) << ((g & 3) * 2)); + } +#endif + for (int sub = 0; sub < 2; sub++) { + sc_pl[sub * 32 + row] = b->scales[kt_local * 2 + sub]; + } + if (row / 8 == kt_local % 4) { + dd_pl[row % 8] = kt_local < 4 ? b->d : (ggml_half) (b->dmin ^ 0x8000); + } + }); +} + +static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid<block_q2_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q2_K, + [](block_q2_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile = sb_tiles + kt_local * tile_size; + const uint8_t * lo_pl = tile; + const uint8_t * sc_pl = tile + 256; + const ggml_half * dd_pl = (const ggml_half *) (tile + 320); + + for (int lk = 0; lk < 32; lk++) { + const int e = kt_local * 32 + lk; + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + const uint8_t lo = (lo_pl[(g >> 2) * 128 + pos] >> ((g & 3) * 2)) & 3; + b->qs[(e / 128) * 32 + e % 32] |= (uint8_t) (lo << (2 * ((e % 128) / 32))); + } + + for (int sub = 0; sub < 2; sub++) { + b->scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row]; + } + if (row / 8 == kt_local % 4) { + if (kt_local < 4) { + b->d = dd_pl[row % 8]; + } else { + b->dmin = (ggml_half) (dd_pl[row % 8] ^ 0x8000); + } + } + } + }); +} + +static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j + 4] & 63; + } else { + *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + } +} + +static inline void pack_k4_scales_mins(uint8_t * scales, ggml_half * d, ggml_half * dmin, + const float * sub_scales, const float * sub_mins) { + float max_scale = 0.0f; + float max_min = 0.0f; + for (int j = 0; j < 8; j++) { + if (sub_scales[j] > max_scale) max_scale = sub_scales[j]; + if (sub_mins[j] > max_min) max_min = sub_mins[j]; + } + + float inv_scale = 0.0f; + if (max_scale > 0.0f) { + *d = GGML_FP32_TO_FP16(max_scale / 63.0f); + const float d_actual = GGML_FP16_TO_FP32(*d); + inv_scale = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f; + } else { + *d = GGML_FP32_TO_FP16(0.0f); + } + + float inv_min = 0.0f; + if (max_min > 0.0f) { + *dmin = GGML_FP32_TO_FP16(max_min / 63.0f); + const float dmin_actual = GGML_FP16_TO_FP32(*dmin); + inv_min = (dmin_actual > 0.0f) ? (1.0f / dmin_actual) : 0.0f; + } else { + *dmin = GGML_FP32_TO_FP16(0.0f); + } + + for (int j = 0; j < 8; j++) { + uint8_t ls = (std::min)((uint8_t) 63, (uint8_t) roundf(inv_scale * sub_scales[j])); + uint8_t lm = (std::min)((uint8_t) 63, (uint8_t) roundf(inv_min * sub_mins[j])); + if (j < 4) { + scales[j] = ls; + scales[j + 4] = lm; + } else { + scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); + scales[j - 4] |= ((ls >> 4) << 6); + scales[j - 0] |= ((lm >> 4) << 6); + } + } +} + +template <typename BlockT> +static inline void repack_k4_quants(uint8_t * tile_dst, const BlockT * b, int row, int kt_local, + int64_t sb_per_row, bool use_neon) { +#if defined(__ARM_NEON) || defined(__aarch64__) + if (use_neon) { + if ((row & 7) == 0) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = b[rr * sb_per_row].qs + (kt_local / 2) * 32; + } + repack_neon_k4_rows(qs, tile_dst, row, (kt_local & 1) != 0); + } + return; + } +#else + GGML_UNUSED(sb_per_row); + GGML_UNUSED(use_neon); +#endif + const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; + const int shift = (kt_local & 1) ? 4 : 0; + + for (int cp = 0; cp < 16; cp++) { + const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F; + const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F; + tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0); + } +} + +static void repack_q4_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + const int64_t sb_per_row = t->ne[0] / QK_K; + const bool use_neon = (t->ne[1] % 8 == 0); + repack_kquant_to_tiled_grid<block_q4_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_K, + [sb_per_row, use_neon](uint8_t * tile_dst, const block_q4_K * b, int row, int kt_local) { + const float d = GGML_FP16_TO_FP32(b->d); + const float dmin = GGML_FP16_TO_FP32(b->dmin); + + uint8_t sc, m; + get_scale_min_k4(kt_local, b->scales, &sc, &m); + + repack_k4_quants(tile_dst, b, row, kt_local, sb_per_row, use_neon); + + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(d * (float) sc); + scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(-dmin * (float) m); + }); +} + +static void repack_tiled_q4_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid<block_q4_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_K, + [](block_q4_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + float sub_scales[8]; + float sub_mins[8]; + + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile_src = sb_tiles + kt_local * tile_size; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + + uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; + const int shift = (kt_local & 1) ? 4 : 0; + + for (int cp = 0; cp < 16; cp++) { + const uint8_t val = tile_src[cp * 32 + row]; + qs_sub[2 * cp + 0] |= (uint8_t) ((val & 0x0F) << shift); + qs_sub[2 * cp + 1] |= (uint8_t) ((val >> 4) << shift); + } + + const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]); + const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]); + sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f; + sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f; + } + + pack_k4_scales_mins(b->scales, &b->d, &b->dmin, sub_scales, sub_mins); + }); +} + +static void repack_q5_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + const int64_t sb_per_row = t->ne[0] / QK_K; + const bool use_neon = (t->ne[1] % 8 == 0); + repack_kquant_to_tiled_grid<block_q5_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q5_K, + [sb_per_row, use_neon](uint8_t * tile_dst, const block_q5_K * b, int row, int kt_local) { + const float d = GGML_FP16_TO_FP32(b->d); + const float dmin = GGML_FP16_TO_FP32(b->dmin); + uint8_t * plane = tile_dst + 640; + + uint8_t sc, m; + get_scale_min_k4(kt_local, b->scales, &sc, &m); + + repack_k4_quants(tile_dst, b, row, kt_local, sb_per_row, use_neon); + + const uint8_t hbit = (uint8_t) (1 << kt_local); +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q5_k_qh(b->qh, hbit, plane, row); +#else + for (int cp = 0; cp < 16; cp++) { + const int i = cp / 4; + const int lane = (cp % 4) * 32 + row; + if (b->qh[2 * cp + 0] & hbit) { + plane[lane] |= (uint8_t) (1 << (2 * i)); + } + if (b->qh[2 * cp + 1] & hbit) { + plane[lane] |= (uint8_t) (1 << (2 * i + 1)); + } + } +#endif + + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(d * (float) sc); + scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(-dmin * (float) m); + }); +} + +static void repack_tiled_q5_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid<block_q5_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q5_K, + [](block_q5_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + float sub_scales[8]; + float sub_mins[8]; + + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile_src = sb_tiles + kt_local * tile_size; + const uint8_t * plane = tile_src + 640; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + + uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; + const int shift = (kt_local & 1) ? 4 : 0; + const uint8_t hbit = (uint8_t) (1 << kt_local); + + for (int cp = 0; cp < 16; cp++) { + const uint8_t val = tile_src[cp * 32 + row]; + qs_sub[2 * cp + 0] |= (uint8_t) ((val & 0x0F) << shift); + qs_sub[2 * cp + 1] |= (uint8_t) ((val >> 4) << shift); + + const int i = cp / 4; + const int lane = (cp % 4) * 32 + row; + if (plane[lane] & (1 << (2 * i))) { + b->qh[2 * cp + 0] |= hbit; + } + if (plane[lane] & (1 << (2 * i + 1))) { + b->qh[2 * cp + 1] |= hbit; + } + } + + const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]); + const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]); + sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f; + sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f; + } + + pack_k4_scales_mins(b->scales, &b->d, &b->dmin, sub_scales, sub_mins); + }); +} + +static inline void q1_0_tiled_bit_location(int k, int row, int & byte_offset, int & bit) { + const int quarter = k / 32; + const int kq = k % 32; + + byte_offset = quarter * 128 + 4 * row + (kq % 4); + bit = kq / 4; +} + +static void repack_q1_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + const block_q1_0 * src_matrix = (const block_q1_0 *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = ne0 / QK1_0; + const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + const int64_t start_slice = offset / slice_size; + const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3); + + GGML_ASSERT(ne0 % QK1_0 == 0); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const block_q1_0 * src_slice = src_matrix + (slice_idx - start_slice) * ne1 * n_k_tiles; + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; + + repack_parallel_for_ct(matrix_size, n_col_tiles, [&](int ct0, int ct1) { + for (int ct = ct0; ct < ct1; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + memset(tile_dst, 0, tile_size); + + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= ne1) { + continue; + } + + const block_q1_0 & block = src_slice[r * n_k_tiles + kt]; + scale_dst[row] = block.d; +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q1_0_block(block.qs, tile_dst, row); +#else + for (int k = 0; k < QK1_0; k++) { + int byte_offset; + int bit; + q1_0_tiled_bit_location(k, row, byte_offset, bit); + if (block.qs[k / 8] & (1u << (k % 8))) { + tile_dst[byte_offset] |= (uint8_t) (1u << bit); + } + } +#endif + } + } + } + }); + } +} + +static void repack_tiled_q1_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + block_q1_0 * dst_matrix = (block_q1_0 *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = ne0 / QK1_0; + const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const size_t row_size_bytes = ggml_row_size(t->type, ne0); + const size_t slice_size = ne1 * row_size_bytes; + const int64_t start_slice = offset / slice_size; + const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3); + + GGML_ASSERT(ne0 % QK1_0 == 0); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + const int64_t start_row = (cur_start_byte - (size_t) slice_idx * slice_size) / row_size_bytes; + const int64_t end_row = (std::min)((int64_t) ((cur_end_byte - (size_t) slice_idx * slice_size + row_size_bytes - 1) / row_size_bytes), ne1); + block_q1_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q1_0); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_row / 32; ct < (end_row + 31) / 32; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < start_row || r >= end_row) { + continue; + } + + block_q1_0 & block = dst_slice[(r - start_row) * n_k_tiles + kt]; + memset(block.qs, 0, sizeof(block.qs)); + block.d = scale_src[row]; + for (int k = 0; k < QK1_0; k++) { + int byte_offset; + int bit; + q1_0_tiled_bit_location(k, row, byte_offset, bit); + if (tile_src[byte_offset] & (1u << bit)) { + block.qs[k / 8] |= (uint8_t) (1u << (k % 8)); + } + } + } + } + } + } +} + +// --- Top-level Repack / Unpack Dispatchers --- + +void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) { + switch (tensor->type) { + case GGML_TYPE_Q1_0: + repack_q1_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q4_0: + case GGML_TYPE_IQ4_NL: + repack_q4_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q4_1: + repack_q4_1_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q5_0: + repack_q5_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q5_1: + repack_q5_1_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q8_0: + repack_q8_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_MXFP4: + repack_mxfp4_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q2_0: + repack_q2_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q2_K: + repack_q2_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q3_K: + repack_q3_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q4_K: + repack_q4_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q5_K: + repack_q5_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q6_K: + repack_q6_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_F16: + repack_float_to_f16_tiled<uint16_t>(tensor, data, 0, size, + [](uint16_t v) { return v; }); + break; + case GGML_TYPE_F32: + repack_float_to_f16_tiled<float>(tensor, data, 0, size, + [](float v) { return (uint16_t) GGML_FP32_TO_FP16(v); }); + break; + case GGML_TYPE_BF16: + repack_float_to_f16_tiled<ggml_bf16_t>(tensor, data, 0, size, + [](ggml_bf16_t v) { return (uint16_t) GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(v)); }); + break; + default: + break; + } +} + +void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t offset, size_t size) { + GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); + + switch (tensor->type) { + case GGML_TYPE_Q1_0: + repack_tiled_q1_0(data, tensor, offset, size); + break; + case GGML_TYPE_Q4_0: + case GGML_TYPE_IQ4_NL: + repack_tiled_q4_0(data, tensor, offset, size); + break; + case GGML_TYPE_Q4_1: + repack_tiled_q4_1(data, tensor, offset, size); + break; + case GGML_TYPE_Q5_0: + repack_tiled_q5_0(data, tensor, offset, size); + break; + case GGML_TYPE_Q5_1: + repack_tiled_q5_1(data, tensor, offset, size); + break; + case GGML_TYPE_Q8_0: + repack_tiled_q8_0(data, tensor, offset, size); + break; + case GGML_TYPE_MXFP4: + repack_tiled_mxfp4(data, tensor, offset, size); + break; + case GGML_TYPE_Q2_0: + repack_tiled_q2_0(data, tensor, offset, size); + break; + case GGML_TYPE_Q2_K: + repack_tiled_q2_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q3_K: + repack_tiled_q3_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q4_K: + repack_tiled_q4_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q5_K: + repack_tiled_q5_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q6_K: + repack_tiled_q6_K(data, tensor, offset, size); + break; + case GGML_TYPE_F16: + unpack_f16_tiled_to_float<uint16_t>(data, tensor, offset, size, + [](uint16_t v) { return v; }); + break; + case GGML_TYPE_F32: + unpack_f16_tiled_to_float<float>(data, tensor, offset, size, + [](uint16_t v) { return ggml_fp16_to_fp32((ggml_fp16_t) v); }); + break; + case GGML_TYPE_BF16: + unpack_f16_tiled_to_float<ggml_bf16_t>(data, tensor, offset, size, + [](uint16_t v) { return ggml_fp32_to_bf16(ggml_fp16_to_fp32((ggml_fp16_t) v)); }); + break; + default: + memcpy(data, (const char *) tensor->data + offset, size); + break; + } +} diff --git a/ggml/src/ggml-hexagon/htp-repack.h b/ggml/src/ggml-hexagon/htp-repack.h new file mode 100644 index 000000000..5b71f9c5a --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-repack.h @@ -0,0 +1,65 @@ +#ifndef HTP_REPACK_H +#define HTP_REPACK_H + +#include <stddef.h> +#include <stdint.h> + +#include "ggml.h" +#include "htp/htp-tensor.h" + +static inline bool htp_is_tiled_weight_type(enum ggml_type type) { + return type == GGML_TYPE_Q1_0 || type == GGML_TYPE_Q2_0 || + type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || + type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1 || + type == GGML_TYPE_MXFP4 || type == GGML_TYPE_IQ4_NL || + type == GGML_TYPE_Q8_0 || type == GGML_TYPE_Q6_K || + type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K || + type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || + type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || + type == GGML_TYPE_BF16; +} + +static inline bool htp_is_tiled_weight_tensor(const struct ggml_tensor * t) { + if (!htp_is_tiled_weight_type(t->type)) { + return false; + } + if (!ggml_is_contiguous(t) || ggml_is_view(t)) { + return false; + } + if (t->ne[0] < 32 || (t->ne[0] % 32 != 0) || t->ne[3] != 1) { + return false; + } + if (!ggml_is_quantized(t->type)) { + return t->ne[1] > 1; + } + return true; +} + +// Size of one tiled row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins, +// so they are larger than the ggml blocks. For the other tiled types (Q2_K included) the tile has the same size. +static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) { + if (type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || type == GGML_TYPE_BF16) { + return (size_t) ne0 * sizeof(ggml_fp16_t); + } + if (type == GGML_TYPE_Q6_K) { + return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q6_K / 32); + } + if (type == GGML_TYPE_Q2_0) { + return (size_t) (ne0 / 64) * 18; + } + if (type == GGML_TYPE_Q4_K) { + return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q4_K / 32); + } + if (type == GGML_TYPE_Q5_K) { + return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q5_K / 32); + } + if (type == GGML_TYPE_Q3_K) { + return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q3_K / 32); + } + return ggml_row_size(type, ne0); +} + +void htp_repack_tensor_tiled(struct ggml_tensor * t, const void * data, size_t size); +void htp_unpack_tensor_tiled(void * data, const struct ggml_tensor * t, size_t offset, size_t size); + +#endif // HTP_REPACK_H diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index 821f08c0b..cf7c2f297 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -20,7 +20,7 @@ add_library(${HTP_LIB} SHARED work-queue.c dma-queue.c hmx-queue.c - htp-tensor.c + htp-dcache.c matmul-ops.c flash-attn-ops.c gated-delta-net-ops.c @@ -28,14 +28,16 @@ add_library(${HTP_LIB} SHARED unary-ops.c sum-rows-ops.c softmax-ops.c - act-ops.c + glu-ops.c rope-ops.c set-rows-ops.c get-rows-ops.c cpy-ops.c + gather-rows.c repeat-ops.c ssm-conv.c cumsum-ops.c + top-k-mask-ops.c fill-ops.c concat-ops.c diag-ops.c @@ -43,17 +45,30 @@ add_library(${HTP_LIB} SHARED pad-ops.c argsort-ops.c im2col-ops.c + conv2d-ops.c roll-ops.c + pool-ops.c + upscale-ops.c allreduce-ops.c + dsv4-hc-ops.c + lightning-indexer-ops.c ) target_compile_definitions(${HTP_LIB} PRIVATE $<IF:$<BOOL:${HEXAGON_HTP_DEBUG}>,HTP_DEBUG=1,NDEBUG=1> $<IF:$<BOOL:${HEXAGON_HTP_DEBUG}>,FARF_HIGH=1,>) - build_idl(htp_iface.idl ${HTP_LIB}) set_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON) +if (NOT Python3_EXECUTABLE) + find_package(Python3 REQUIRED COMPONENTS Interpreter) +endif() + +add_custom_command(TARGET ${HTP_LIB} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo "Adding SWIV section $<TARGET_FILE_NAME:${HTP_LIB}>" + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/htp-swiv.py -i $<TARGET_FILE:${HTP_LIB}> -o $<TARGET_FILE:${HTP_LIB}> + VERBATIM) + install(TARGETS ${HTP_LIB}) diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c index 7fcc4e866..1b95917ae 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.c +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c @@ -15,7 +15,7 @@ #include "hvx-utils.h" #include "htp-tensor.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "allreduce-ops.h" #include "htp-fence.h" @@ -29,9 +29,9 @@ struct htp_allreduce_context { uint32_t row_size_aligned; uint32_t rank_elem_start; uint32_t rank_nelem; - uint32_t elems_per_thread; + uint32_t elems_per_hvx; uint32_t block_elems; - uint32_t vtcm_size_per_thread; + uint32_t vtcm_size_per_hvx; bool is_row_bcast; uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; uint8_t * dst_spad_base; @@ -39,7 +39,8 @@ struct htp_allreduce_context { }; #define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \ -static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ +static void allreduce_thread_dma_1d_##SUFFIX(const htp_thread_t * th, unsigned int nth, unsigned int ith, \ + void * data) { \ struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ struct htp_ops_context * octx = actx->octx; \ \ @@ -47,19 +48,19 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, const uint32_t n_dsts = actx->n_dsts; \ const uint32_t block_elems = actx->block_elems; \ \ - const uint32_t dr = actx->elems_per_thread; \ + const uint32_t dr = actx->elems_per_hvx; \ const uint32_t ir0 = actx->rank_elem_start + dr * ith; \ const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \ if (ir0 >= ir1) return; \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + struct htp_thread_trace * tr = th->trace; \ + dma_queue * dma_q = th->dma_q; \ \ - const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \ - uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \ - uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_thread_offset) : NULL; \ + const size_t vtcm_hvx_offset = ith * actx->vtcm_size_per_hvx; \ + uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_hvx_offset; \ + uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_hvx_offset) : NULL; \ \ - const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + const size_t spad_half = actx->vtcm_size_per_hvx / 2; \ uint32_t ir_prefetch = ir0; \ int spad_idx = 0; \ \ @@ -72,7 +73,7 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \ } \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + spad_idx * spad_half; \ const dma_addr_t s_ddr = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ } \ @@ -99,13 +100,13 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_pop(dma_q); \ } \ uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \ - uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \ - uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ + uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ uint8_t * r_spad = HAS_ADD ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ HVX_ADD_FN(d_spad, s0_spad, s1_spad, cur_elems); \ for (uint32_t s = 2; s < n_ranks; s++) { \ - uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ HVX_ADD_FN(d_spad, d_spad, ss_spad, cur_elems); \ } \ if (HAS_ADD) { \ @@ -120,7 +121,7 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \ size_t next_bytes = next_elems * sizeof(TYPE); \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ const dma_addr_t s_next = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), next_bytes, next_bytes, next_bytes, 1); \ } \ @@ -143,7 +144,7 @@ DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1) DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1) #define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \ -static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ +static void allreduce_thread_dma_2d_##SUFFIX(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ struct htp_ops_context * octx = actx->octx; \ \ @@ -154,19 +155,19 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, const uint32_t row_size_aligned = actx->row_size_aligned; \ const uint32_t row_bytes = ne0 * sizeof(TYPE); \ \ - const uint32_t dr = actx->elems_per_thread; \ + const uint32_t dr = actx->elems_per_hvx; \ const uint32_t r0 = actx->rank_elem_start + dr * ith; \ const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \ if (r0 >= r1) return; \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + struct htp_thread_trace * tr = th->trace; \ + dma_queue * dma_q = th->dma_q; \ \ - const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \ - uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \ - uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_thread_offset)) : NULL; \ + const size_t vtcm_hvx_offset = ith * actx->vtcm_size_per_hvx; \ + uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_hvx_offset; \ + uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_hvx_offset)) : NULL; \ \ - const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + const size_t spad_half = actx->vtcm_size_per_hvx / 2; \ uint32_t r_prefetch = r0; \ int spad_idx = 0; \ \ @@ -178,7 +179,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \ } \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + spad_idx * spad_half; \ const dma_addr_t s_ddr = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \ } \ @@ -204,8 +205,8 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_pop(dma_q); \ } \ uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \ - uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \ - uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ + uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ for (uint32_t row = 0; row < cur_rows; row++) { \ @@ -214,7 +215,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, const uint8_t * s1_row = s1_spad + row * row_size_aligned; \ HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \ for (uint32_t s = 2; s < n_ranks; s++) { \ - const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half + row * row_size_aligned; \ + const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half + row * row_size_aligned; \ HVX_ADD_FN(d_row, d_row, ss_row, ne0); \ } \ if (HAS_ADD) { \ @@ -230,7 +231,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, if (r_prefetch < r1) { \ uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ const dma_addr_t s_next = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \ } \ @@ -259,19 +260,19 @@ static int validate_allreduce( const struct htp_allreduce_kernel_params * kparams, uint32_t n_ranks ) { - if (kparams->n_threads == 0 || (uint32_t) kparams->n_threads > octx->ctx->n_threads) { + if (kparams->n_hvx == 0 || (uint32_t) kparams->n_hvx > octx->ctx->n_hvx) { return HTP_STATUS_INVAL_PARAMS; } - octx->n_threads = (uint32_t) kparams->n_threads; + octx->n_hvx = (uint32_t) kparams->n_hvx; - if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) { + if (kparams->vtcm_size_per_hvx <= 0 || kparams->vtcm_size <= 0) { return HTP_STATUS_INVAL_PARAMS; } const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count( - n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0); - const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread; + n_ranks, octx->n_hvx, has_add, kparams->is_row_bcast != 0); + const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_hvx; if (vtcm_size != (size_t) kparams->vtcm_size) { return HTP_STATUS_INVAL_PARAMS; } @@ -360,10 +361,10 @@ int op_allreduce(struct htp_ops_context * octx) { // 2. Multi-threaded Reduction across assigned rank chunk if (nelem > 0) { - const uint32_t n_threads = (uint32_t) kparams->n_threads; + const uint32_t n_hvx = (uint32_t) kparams->n_hvx; const uint32_t block_elems = (uint32_t) kparams->block_elems; - const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread; - const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread; + const uint32_t elems_per_hvx = (uint32_t) kparams->elems_per_hvx; + const uint32_t vtcm_size_per_hvx = (uint32_t) kparams->vtcm_size_per_hvx; struct htp_allreduce_context actx; actx.octx = octx; @@ -375,9 +376,9 @@ int op_allreduce(struct htp_ops_context * octx) { actx.row_size_aligned = (uint32_t) kparams->row_size_aligned; actx.rank_elem_start = (uint32_t) kparams->rank_elem_start; actx.rank_nelem = (uint32_t) kparams->rank_nelem; - actx.elems_per_thread = elems_per_thread; + actx.elems_per_hvx = elems_per_hvx; actx.block_elems = block_elems; - actx.vtcm_size_per_thread = vtcm_size_per_thread; + actx.vtcm_size_per_hvx = vtcm_size_per_hvx; actx.is_row_bcast = (kparams->is_row_bcast != 0); work_queue_func_t reduce_fun = NULL; @@ -409,13 +410,13 @@ int op_allreduce(struct htp_ops_context * octx) { uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; for (uint32_t s = 0; s < n_ranks; s++) { actx.src_spad_base[s] = vtcm_ptr; - vtcm_ptr += n_threads * vtcm_size_per_thread; + vtcm_ptr += n_hvx * vtcm_size_per_hvx; } actx.dst_spad_base = vtcm_ptr; - vtcm_ptr += n_threads * vtcm_size_per_thread; + vtcm_ptr += n_hvx * vtcm_size_per_hvx; if (has_add) { actx.res_spad_base = vtcm_ptr; - vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread; + vtcm_ptr += (actx.is_row_bcast ? 1 : n_hvx) * vtcm_size_per_hvx; } if (has_add && actx.is_row_bcast) { @@ -426,7 +427,7 @@ int op_allreduce(struct htp_ops_context * octx) { dma_queue_pop(dma_q); } - work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, reduce_fun, &actx, n_hvx); } // 4. Exit Barrier: Synchronize all ranks after writing diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h index 54ef60378..1fb486cb6 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h @@ -24,22 +24,22 @@ enum htp_allreduce_mode { static inline size_t htp_allreduce_vtcm_buffer_count( uint32_t n_ranks, - uint32_t n_threads, + uint32_t n_hvx, bool has_add, bool is_row_bcast ) { - return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0); + return (size_t) (n_ranks + 1) * n_hvx + (has_add ? (is_row_bcast ? 1 : n_hvx) : 0); } struct htp_allreduce_kernel_params { + uint8_t kernel_type; int32_t rank; int32_t n_ranks; - int32_t n_threads; + int32_t n_hvx; int32_t block_elems; // 1D: block_elems, 2D: block_rows - int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread - int32_t vtcm_size_per_thread; + int32_t elems_per_hvx; // 1D: nelem_per_hvx, 2D: nrows_per_hvx + int32_t vtcm_size_per_hvx; int32_t vtcm_size; - int32_t kernel_type; int32_t ne0; int32_t ne1; int32_t row_size_aligned; @@ -50,6 +50,12 @@ struct htp_allreduce_kernel_params { int32_t mode; // enum htp_allreduce_mode: FULL or SHARDED_FANOUT }; +#if defined(__cplusplus) +static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob"); +#endif + #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index 6f24d95f9..af2605954 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -315,55 +315,61 @@ static void bitonic_sort_vtcm_desc(uint8_t * values, uint8_t * indices, uint32_t } } -static void top_k_select_tiled(dma_queue * dma_q, struct htp_thread_trace * tr, - dma_addr_t src_addr, uint32_t n, uint32_t k, - float * values_buf, int32_t * indices_buf, - float * out_values, int32_t * out_indices, - uint32_t chunk_base) { - const uint32_t tile_elems = 1024; - uint32_t n_tiles = (n + tile_elems - 1) / tile_elems; - uint32_t candidate_count = n_tiles * k; - uint32_t merge_n_vec = hmx_ceil_div(candidate_count, 32); - uint32_t merge_n_vec_pow2 = 1; - while (merge_n_vec_pow2 < merge_n_vec) merge_n_vec_pow2 <<= 1; - uint32_t merge_elems = merge_n_vec_pow2 * 32; - float * candidate_values = values_buf + tile_elems; - int32_t * candidate_indices = indices_buf + tile_elems; - uint32_t candidate_pos = 0; - - for (uint32_t offset = 0; offset < n; offset += tile_elems) { - uint32_t tile_count = MIN(tile_elems, n - offset); - dma_sync_read(dma_q, values_buf, src_addr + (size_t) offset * sizeof(float), tile_count * sizeof(float)); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) offset); - if (tile_count < tile_elems) { - hvx_splat_f32_u((uint8_t *) (values_buf + tile_count), -INFINITY, tile_elems - tile_count); +// top-k of one row held in VTCM, k <= 64. Keeps a sorted list of the k best, rejects 256-float blocks with one max-reduce. +// Writes k values and k indices (ties: first seen wins). Slots with no finite value get -INF and index 0. +static void top_k_scan(const float * vals, uint32_t n, uint32_t k, float * out_v, int32_t * out_i) { + float best_v[64]; + int32_t best_i[64]; + uint32_t cnt = 0; + float thr = -INFINITY; + + const uint32_t blk = 256; + for (uint32_t off = 0; off < n; off += blk) { + const uint32_t bn = MIN(blk, n - off); + if (hvx_reduce_max_f32((const uint8_t *) (vals + off), bn) <= thr) { + continue; } - - bitonic_sort_vtcm_desc((uint8_t *) values_buf, (uint8_t *) indices_buf, tile_elems / 32, true); - uint32_t tile_k = MIN(k, tile_count); - for (uint32_t j = 0; j < tile_k; j++) { - candidate_values[candidate_pos] = values_buf[j]; - candidate_indices[candidate_pos] = indices_buf[j] + (int32_t) offset; - candidate_pos++; + for (uint32_t vi = 0; vi < bn; vi += 32) { + const uint32_t vn = MIN(32, bn - vi); + HVX_VectorAlias v_local; + v_local.v = hvx_vmem(vals + off + vi); + for (uint32_t i = 0; i < vn; i++) { + const float v = v_local.fp32[i]; + if (!(v > thr)) continue; + uint32_t pos = cnt < k ? cnt++ : k - 1; + while (pos > 0 && best_v[pos - 1] < v) { + best_v[pos] = best_v[pos - 1]; + best_i[pos] = best_i[pos - 1]; + pos--; + } + best_v[pos] = v; + best_i[pos] = (int32_t) (off + vi + i); + if (cnt == k) thr = best_v[k - 1]; + } } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) offset); } - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); - if (merge_elems > candidate_pos) { - hvx_splat_f32_u((uint8_t *) (candidate_values + candidate_pos), -INFINITY, merge_elems - candidate_pos); - for (uint32_t j = candidate_pos; j < merge_elems; j++) { - candidate_indices[j] = 0; + // fewer than k values above -INF: take -INF or NaN elements, then pad with index 0 + for (uint32_t off = 0; off < n && cnt < k; off += 32) { + const uint32_t vn = MIN(32, n - off); + HVX_VectorAlias v_local; + v_local.v = hvx_vmem(vals + off); + for (uint32_t j = 0; j < vn && cnt < k; j++) { + if (!(v_local.fp32[j] > -INFINITY)) { + best_v[cnt] = -INFINITY; + best_i[cnt] = (int32_t) (off + j); + cnt++; + } } } - - bitonic_sort_vtcm_desc((uint8_t *) candidate_values, (uint8_t *) candidate_indices, merge_n_vec_pow2, false); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); + for (; cnt < k; cnt++) { + best_v[cnt] = -INFINITY; + best_i[cnt] = 0; + } for (uint32_t j = 0; j < k; j++) { - out_values[j] = candidate_values[j]; - out_indices[j] = candidate_indices[j] + (int32_t) chunk_base; + out_v[j] = best_v[j]; + out_i[j] = best_i[j]; } } @@ -453,35 +459,39 @@ static void merge_runs( struct htp_sort_chunk_ctx { struct htp_ops_context * octx; const struct htp_sort_kernel_params * kparams; - uint8_t * vtcm_base; - uint32_t real_count[HTP_MAX_NTHREADS]; + uint8_t * phase1_base; + size_t phase1_slot_size; + float * merge_values; + int32_t * merge_indices; + uint32_t real_count[HTP_OP_MAX_NHVX]; }; -static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { +static void htp_sort_chunk_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_sort_chunk_ctx * cctx = (struct htp_sort_chunk_ctx *) data; struct htp_ops_context * octx = cctx->octx; const struct htp_sort_kernel_params * kparams = cctx->kparams; const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - uint32_t ne00 = (uint32_t) kparams->ne00; + uint32_t ne00 = src0->ne[0]; uint32_t chunk_base = i * chunk_elems; uint32_t real_count = (chunk_base < ne00) ? MIN(chunk_elems, ne00 - chunk_base) : 0; cctx->real_count[i] = real_count; - uint8_t * spad = cctx->vtcm_base + (size_t) kparams->phase1_slot_size * i; + uint8_t * spad = cctx->phase1_base + cctx->phase1_slot_size * i; float * values_buf = (float *) spad; int32_t * indices_buf = (int32_t *) (spad + chunk_elems * sizeof(float)); - float * merge_values = (float *) (cctx->vtcm_base + kparams->merge_values_off); - int32_t * merge_indices = (int32_t *) (cctx->vtcm_base + kparams->merge_indices_off); + float * merge_values = cctx->merge_values; + int32_t * merge_indices = cctx->merge_indices; - struct htp_thread_trace * tr = &octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const dma_addr_t src_addr = src0->data + (size_t) chunk_base * sizeof(float); if (kparams->is_top_k) { - uint32_t k = (uint32_t) kparams->k; + uint32_t k = dst->ne[0]; uint32_t local_k = MIN(k, chunk_elems); if (real_count == 0) { @@ -493,7 +503,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { } if (local_k == 1 && ne00 >= 128*1024) { - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); float max_value; @@ -505,14 +515,18 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { } if (local_k > 1 && local_k <= 64 && chunk_elems > 1024) { - top_k_select_tiled(octx->ctx->dma[i], tr, src_addr, real_count, local_k, - values_buf, indices_buf, - &merge_values[i * local_k], &merge_indices[i * local_k], - chunk_base); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + top_k_scan(values_buf, real_count, local_k, &merge_values[i * local_k], &merge_indices[i * local_k]); + for (uint32_t j = 0; j < local_k; j++) { + merge_indices[i * local_k + j] += (int32_t) chunk_base; + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); return; } - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); if (chunk_elems > real_count) { @@ -528,7 +542,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); } else { if (real_count > 0) { - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); init_indices_ramp(indices_buf, real_count, chunk_base); @@ -548,7 +562,7 @@ struct htp_argsort_merge_l1_ctx { bool asc; }; -static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data) { +static void htp_argsort_merge_l1_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_argsort_merge_l1_ctx * mctx = (struct htp_argsort_merge_l1_ctx *) data; struct htp_sort_chunk_ctx * cctx = mctx->cctx; const struct htp_sort_kernel_params * kparams = cctx->kparams; @@ -559,19 +573,19 @@ static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data uint32_t count0 = cctx->real_count[c0]; uint32_t count1 = cctx->real_count[c1]; - size_t slot_size = (size_t) kparams->phase1_slot_size; + size_t slot_size = cctx->phase1_slot_size; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - const float * val0 = (const float *) (cctx->vtcm_base + slot_size * c0); - const int32_t * idx0 = (const int32_t *) (cctx->vtcm_base + slot_size * c0 + chunk_elems * sizeof(float)); - const float * val1 = (const float *) (cctx->vtcm_base + slot_size * c1); - const int32_t * idx1 = (const int32_t *) (cctx->vtcm_base + slot_size * c1 + chunk_elems * sizeof(float)); + const float * val0 = (const float *) (cctx->phase1_base + slot_size * c0); + const int32_t * idx0 = (const int32_t *) (cctx->phase1_base + slot_size * c0 + chunk_elems * sizeof(float)); + const float * val1 = (const float *) (cctx->phase1_base + slot_size * c1); + const int32_t * idx1 = (const int32_t *) (cctx->phase1_base + slot_size * c1 + chunk_elems * sizeof(float)); uint32_t out_offset = (2 * i) * chunk_elems; - float * out_val = (float *) (cctx->vtcm_base + kparams->merge_values_off) + out_offset; - int32_t * out_idx = (int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + out_offset; + float * out_val = cctx->merge_values + out_offset; + int32_t * out_idx = cctx->merge_indices + out_offset; - struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); @@ -582,30 +596,30 @@ struct htp_argsort_merge_l2_ctx { bool asc; }; -static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data) { +static void htp_argsort_merge_l2_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_argsort_merge_l2_ctx * mctx = (struct htp_argsort_merge_l2_ctx *) data; struct htp_sort_chunk_ctx * cctx = mctx->cctx; const struct htp_sort_kernel_params * kparams = cctx->kparams; - size_t slot_size = (size_t) kparams->phase1_slot_size; + size_t slot_size = cctx->phase1_slot_size; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; uint32_t off0 = (4 * i) * chunk_elems; uint32_t off1 = (4 * i + 2) * chunk_elems; - const float * val0 = (const float *) (cctx->vtcm_base + kparams->merge_values_off) + off0; - const int32_t * idx0 = (const int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + off0; - const float * val1 = (const float *) (cctx->vtcm_base + kparams->merge_values_off) + off1; - const int32_t * idx1 = (const int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + off1; + const float * val0 = cctx->merge_values + off0; + const int32_t * idx0 = cctx->merge_indices + off0; + const float * val1 = cctx->merge_values + off1; + const int32_t * idx1 = cctx->merge_indices + off1; uint32_t count0 = cctx->real_count[4*i + 0] + cctx->real_count[4*i + 1]; uint32_t count1 = cctx->real_count[4*i + 2] + cctx->real_count[4*i + 3]; - uint8_t * slot_base = cctx->vtcm_base + (4 * i) * slot_size; + uint8_t * slot_base = cctx->phase1_base + (4 * i) * slot_size; float * out_val = (float *) slot_base; int32_t * out_idx = (int32_t *) (slot_base + 4 * chunk_elems * sizeof(float)); - struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); @@ -614,8 +628,11 @@ static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data struct htp_sort_multi_row_ctx { struct htp_ops_context * octx; const struct htp_sort_kernel_params * kparams; - uint8_t * vtcm_base; - uint32_t nrows_per_thread; + uint8_t * phase1_base; + size_t phase1_slot_size; + uint32_t row_start; + uint32_t row_end; + uint32_t nrows_per_hvx; }; struct htp_tensor_row_coord { @@ -624,12 +641,18 @@ struct htp_tensor_row_coord { uint32_t i03; }; -static inline struct htp_tensor_row_coord init_row_coord(uint32_t r, uint32_t ne01, uint32_t ne02) { +static inline struct htp_tensor_row_coord init_row_coord( + uint32_t r, + uint32_t ne01, + uint32_t ne02, + const struct fastdiv_values * div_ne01, + const struct fastdiv_values * div_ne01_ne02 +) { uint32_t ne01_ne02 = ne01 * ne02; - uint32_t i03 = (ne01_ne02 > 0) ? (r / ne01_ne02) : 0; - uint32_t rem = (ne01_ne02 > 0) ? (r % ne01_ne02) : 0; - uint32_t i02 = (ne01 > 0) ? (rem / ne01) : 0; - uint32_t i01 = (ne01 > 0) ? (rem % ne01) : 0; + uint32_t i03 = (ne01_ne02 > 0) ? fastdiv(r, div_ne01_ne02) : 0; + uint32_t rem = (ne01_ne02 > 0) ? (r - i03 * ne01_ne02) : 0; + uint32_t i02 = (ne01 > 0) ? fastdiv(rem, div_ne01) : 0; + uint32_t i01 = (ne01 > 0) ? (rem - i02 * ne01) : 0; return (struct htp_tensor_row_coord){ i01, i02, i03 }; } @@ -655,10 +678,14 @@ static inline void compute_row_sort( uint32_t ne00, uint32_t chunk_elems, bool is_top_k, - enum ggml_sort_order order + enum ggml_sort_order order, + uint32_t k ) { if (is_top_k) { - if (ne00 == 32 || ne00 == 64 || ne00 == 128 || ne00 == 256 || ne00 == 512 || ne00 == 1024) { + if (k > 1 && k <= 64 && ne00 > 1024) { + float out_v[64]; + top_k_scan(cur_values, ne00, k, out_v, cur_indices); + } else if (ne00 == 32 || ne00 == 64 || ne00 == 128 || ne00 == 256 || ne00 == 512 || ne00 == 1024) { switch (ne00) { case 32: sort32_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; case 64: sort64_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; @@ -694,14 +721,14 @@ static inline void compute_row_sort( } } -static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data) { +static void htp_sort_multi_row_job(const htp_thread_t * th, unsigned int n, unsigned int ith, void * data) { struct htp_sort_multi_row_ctx * mctx = (struct htp_sort_multi_row_ctx *) data; struct htp_ops_context * octx = mctx->octx; const struct htp_sort_kernel_params * kparams = mctx->kparams; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - uint32_t ne00 = (uint32_t) kparams->ne00; + uint32_t ne00 = src0->ne[0]; uint32_t ne01 = src0->ne[1]; uint32_t ne02 = src0->ne[2]; @@ -713,35 +740,35 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data uint32_t nb2 = dst->nb[2]; uint32_t nb3 = dst->nb[3]; - uint32_t row_start = (uint32_t) kparams->row_start; - uint32_t row_end = (uint32_t) kparams->row_end; - uint32_t r_start = row_start + ith * mctx->nrows_per_thread; - uint32_t r_end = MIN(r_start + mctx->nrows_per_thread, row_end); + uint32_t row_start = mctx->row_start; + uint32_t row_end = mctx->row_end; + uint32_t r_start = row_start + ith * mctx->nrows_per_hvx; + uint32_t r_end = MIN(r_start + mctx->nrows_per_hvx, row_end); if (r_start >= r_end) return; - size_t spad_offset = ith * (size_t) (kparams->phase1_slot_size * kparams->n_slots); - uint8_t * thread_spad = mctx->vtcm_base + spad_offset; - size_t slot_size = (size_t) kparams->phase1_slot_size; + size_t slot_size = mctx->phase1_slot_size; + size_t spad_offset = ith * slot_size * (size_t) kparams->n_slots; + uint8_t * hvx_spad = mctx->phase1_base + spad_offset; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - float * values_buf = (float *) thread_spad; - int32_t * indices_buf = (int32_t *) (thread_spad + chunk_elems * sizeof(float)); + float * values_buf = (float *) hvx_spad; + int32_t * indices_buf = (int32_t *) (hvx_spad + chunk_elems * sizeof(float)); - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; bool is_top_k = kparams->is_top_k != 0; - uint32_t k = (uint32_t) kparams->k; + uint32_t k = dst->ne[0]; enum ggml_sort_order order = (enum ggml_sort_order) kparams->order; const uint32_t src_bytes = ne00 * sizeof(float); const uint32_t dst_bytes = (is_top_k ? k : ne00) * sizeof(int32_t); if (kparams->n_slots == 2) { - struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02); + struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); for (uint32_t r = r_start, spad_idx = 0; r < r_end && spad_idx < 2; r++, spad_idx++) { - uint8_t * cur_spad = thread_spad + spad_idx * slot_size; + uint8_t * cur_spad = hvx_spad + spad_idx * slot_size; float * cur_values = (float *) cur_spad; int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float)); @@ -759,12 +786,12 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data step_row_coord(&prime_coord, ne01, ne02); } - struct htp_tensor_row_coord dst_coord = init_row_coord(r_start, ne01, ne02); - struct htp_tensor_row_coord next_src_coord = init_row_coord(r_start + 2, ne01, ne02); + struct htp_tensor_row_coord dst_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); + struct htp_tensor_row_coord next_src_coord = init_row_coord(r_start + 2, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); for (uint32_t r = r_start; r < r_end; r++) { uint32_t cur_slot = (r - r_start) & 1; - uint8_t * cur_spad = thread_spad + cur_slot * slot_size; + uint8_t * cur_spad = hvx_spad + cur_slot * slot_size; float * cur_values = (float *) cur_spad; int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float)); @@ -775,7 +802,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data dma_queue_pop(dma_q); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); - compute_row_sort(cur_values, cur_indices, ne00, chunk_elems, is_top_k, order); + compute_row_sort(cur_values, cur_indices, ne00, chunk_elems, is_top_k, order, k); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); // Push writeback of current slot @@ -796,7 +823,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data dma_queue_flush(dma_q); } else { - struct htp_tensor_row_coord coord = init_row_coord(r_start, ne01, ne02); + struct htp_tensor_row_coord coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); for (uint32_t r = r_start; r < r_end; r++) { dma_addr_t src_addr = src0->data + coord_offset(coord, nb01, nb02, nb03); dma_addr_t dst_addr = dst->data + coord_offset(coord, nb1, nb2, nb3); @@ -804,7 +831,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data dma_sync_read(dma_q, values_buf, src_addr, src_bytes); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); - compute_row_sort(values_buf, indices_buf, ne00, chunk_elems, is_top_k, order); + compute_row_sort(values_buf, indices_buf, ne00, chunk_elems, is_top_k, order, k); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); dma_sync_write(dma_q, dst_addr, indices_buf, dst_bytes); @@ -847,63 +874,50 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { uint32_t ne00 = src0->ne[0]; uint32_t k = dst->ne[0]; - struct htp_sort_kernel_params kparams_local; const struct htp_sort_kernel_params * kparams = (const struct htp_sort_kernel_params *) octx->kernel_params; - if (kparams->n_threads == 0) { - struct htp_sort_vtcm_layout layout; - if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, octx->n_threads, octx->ctx->vtcm_size, is_top_k)) { - return HTP_STATUS_VTCM_TOO_SMALL; - } - memset(&kparams_local, 0, sizeof(kparams_local)); - kparams_local.n_threads = (int32_t) layout.n_threads; - kparams_local.total_rows = (int32_t) total_rows; - kparams_local.row_start = (int32_t) row_start; - kparams_local.row_end = (int32_t) row_end; - kparams_local.ne00 = (int32_t) ne00; - kparams_local.k = (int32_t) k; - kparams_local.order = is_top_k ? GGML_SORT_ORDER_DESC : (int32_t) octx->op_params[0]; - kparams_local.is_top_k = is_top_k ? 1 : 0; - kparams_local.use_dma = 1; - kparams_local.chunk_elems = (int32_t) layout.chunk_elems; - kparams_local.n_chunks = (int32_t) layout.n_chunks; - kparams_local.vtcm_size = (int32_t) layout.total_bytes; - kparams_local.phase1_slot_size = (int32_t) layout.phase1_slot_size; - kparams_local.merge_values_off = (int32_t) layout.merge_values_off; - kparams_local.merge_indices_off = (int32_t) layout.merge_indices_off; - kparams_local.merge_elems = (int32_t) layout.merge_elems; - kparams_local.n_slots = (int32_t) layout.n_slots; - kparams = &kparams_local; - } else if (octx->ctx->mdev.count > 1) { - memcpy(&kparams_local, kparams, sizeof(kparams_local)); - kparams_local.row_start = (int32_t) row_start; - kparams_local.row_end = (int32_t) row_end; - kparams = &kparams_local; + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; } - if (octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) { - FARF(ERROR, "sort: VTCM size too small. Needed %d, have %zu", kparams->vtcm_size, octx->ctx->vtcm_size); + struct htp_sort_vtcm_layout layout; + htp_sort_vtcm_layout_build(&layout, + total_rows, + (uint32_t) kparams->chunk_elems, + (uint32_t) kparams->n_chunks, + (uint32_t) kparams->merge_elems, + (uint32_t) kparams->n_hvx, + (uint32_t) kparams->n_slots); + + if (octx->ctx->vtcm_size < layout.total_bytes) { + FARF(ERROR, "sort: VTCM size too small. Needed %zu, have %zu", layout.total_bytes, octx->ctx->vtcm_size); return HTP_STATUS_VTCM_TOO_SMALL; } - uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + uint8_t * phase1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_phase1); + float * merge_values = VTCM_LAYOUT_PTR(float, vtcm_base, layout.off_merge_values); + int32_t * merge_indices = VTCM_LAYOUT_PTR(int32_t, vtcm_base, layout.off_merge_indices); if (total_rows == 1 && kparams->n_chunks > 1) { struct htp_sort_chunk_ctx cctx; - cctx.octx = octx; - cctx.kparams = kparams; - cctx.vtcm_base = vtcm_base; + cctx.octx = octx; + cctx.kparams = kparams; + cctx.phase1_base = phase1_base; + cctx.phase1_slot_size = layout.phase1_slot_size; + cctx.merge_values = merge_values; + cctx.merge_indices = merge_indices; memset(cctx.real_count, 0, sizeof(cctx.real_count)); - work_queue_run(octx->ctx->work_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks); + work_queue_run(octx->ctx->hvx_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks); if (is_top_k) { uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; uint32_t local_k = MIN(k, chunk_elems); if (k == 1 && ne00 >= 128*1024) { - float * cand_vals = (float *) (vtcm_base + kparams->merge_values_off); - int32_t * cand_idxs = (int32_t *) (vtcm_base + kparams->merge_indices_off); + float * cand_vals = merge_values; + int32_t * cand_idxs = merge_indices; float best_val = cand_vals[0]; int32_t best_idx = cand_idxs[0]; for (int32_t c = 1; c < kparams->n_chunks; c++) { @@ -918,8 +932,8 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { } uint32_t total_candidates = (uint32_t) kparams->n_chunks * local_k; - float * cand_vals = (float *) (vtcm_base + kparams->merge_values_off); - int32_t * cand_idxs = (int32_t *) (vtcm_base + kparams->merge_indices_off); + float * cand_vals = merge_values; + int32_t * cand_idxs = merge_indices; uint32_t merge_elems = (uint32_t) kparams->merge_elems; if (merge_elems > total_candidates) { @@ -938,23 +952,23 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { bool asc = kparams->order == GGML_SORT_ORDER_ASC; if (kparams->n_chunks == 8) { struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 4); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l1_job, &l1ctx, 4); struct htp_argsort_merge_l2_ctx l2ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l2_job, &l2ctx, 2); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l2_job, &l2ctx, 2); - size_t slot_size = (size_t) kparams->phase1_slot_size; + size_t slot_size = layout.phase1_slot_size; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - const float * val0 = (const float *) vtcm_base; - const int32_t * idx0 = (const int32_t *) (vtcm_base + 4 * chunk_elems * sizeof(float)); - const float * val1 = (const float *) (vtcm_base + 4 * slot_size); - const int32_t * idx1 = (const int32_t *) (vtcm_base + 4 * slot_size + 4 * chunk_elems * sizeof(float)); + const float * val0 = (const float *) phase1_base; + const int32_t * idx0 = (const int32_t *) (phase1_base + 4 * chunk_elems * sizeof(float)); + const float * val1 = (const float *) (phase1_base + 4 * slot_size); + const int32_t * idx1 = (const int32_t *) (phase1_base + 4 * slot_size + 4 * chunk_elems * sizeof(float)); uint32_t n01 = cctx.real_count[0] + cctx.real_count[1] + cctx.real_count[2] + cctx.real_count[3]; uint32_t n23 = cctx.real_count[4] + cctx.real_count[5] + cctx.real_count[6] + cctx.real_count[7]; - int32_t * final_idx = (int32_t *) (vtcm_base + kparams->merge_indices_off); + int32_t * final_idx = merge_indices; struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); merge_runs(val0, idx0, n01, val1, idx1, n23, NULL, final_idx, asc); @@ -962,33 +976,33 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); } else if (kparams->n_chunks == 4) { struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 2); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l1_job, &l1ctx, 2); uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; uint32_t n01 = cctx.real_count[0] + cctx.real_count[1]; uint32_t n23 = cctx.real_count[2] + cctx.real_count[3]; - const float * val01 = (const float *) (vtcm_base + kparams->merge_values_off); - const int32_t * idx01 = (const int32_t *) (vtcm_base + kparams->merge_indices_off); - const float * val23 = (const float *) (vtcm_base + kparams->merge_values_off) + 2 * chunk_elems; - const int32_t * idx23 = (const int32_t *) (vtcm_base + kparams->merge_indices_off) + 2 * chunk_elems; + const float * val01 = merge_values; + const int32_t * idx01 = merge_indices; + const float * val23 = merge_values + 2 * chunk_elems; + const int32_t * idx23 = merge_indices + 2 * chunk_elems; - int32_t * final_idx = (int32_t *) vtcm_base; + int32_t * final_idx = (int32_t *) phase1_base; struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); merge_runs(val01, idx01, n01, val23, idx23, n23, NULL, final_idx, asc); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); } else if (kparams->n_chunks == 2) { - size_t slot_size = (size_t) kparams->phase1_slot_size; + size_t slot_size = layout.phase1_slot_size; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - const float * val0 = (const float *) (vtcm_base); - const int32_t * idx0 = (const int32_t *) (vtcm_base + chunk_elems * sizeof(float)); - const float * val1 = (const float *) (vtcm_base + slot_size); - const int32_t * idx1 = (const int32_t *) (vtcm_base + slot_size + chunk_elems * sizeof(float)); + const float * val0 = (const float *) (phase1_base); + const int32_t * idx0 = (const int32_t *) (phase1_base + chunk_elems * sizeof(float)); + const float * val1 = (const float *) (phase1_base + slot_size); + const int32_t * idx1 = (const int32_t *) (phase1_base + slot_size + chunk_elems * sizeof(float)); - int32_t * final_idx = (int32_t *) (vtcm_base + kparams->merge_indices_off); + int32_t * final_idx = merge_indices; struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); merge_runs(val0, idx0, cctx.real_count[0], val1, idx1, cctx.real_count[1], NULL, final_idx, asc); @@ -996,7 +1010,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); } else { uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - int32_t * final_idx = (int32_t *) (vtcm_base + chunk_elems * sizeof(float)); + int32_t * final_idx = (int32_t *) (phase1_base + chunk_elems * sizeof(float)); dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); } } @@ -1006,7 +1020,10 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { struct htp_sort_multi_row_ctx mctx; mctx.octx = octx; mctx.kparams = kparams; - mctx.vtcm_base = vtcm_base; + mctx.phase1_base = phase1_base; + mctx.phase1_slot_size = layout.phase1_slot_size; + mctx.row_start = row_start; + mctx.row_end = row_end; uint32_t dst_row_bytes = (uint32_t) dst_row_size; uint32_t rows_per_chunk = 0; @@ -1014,13 +1031,13 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { if (rows_per_chunk == 0) { rows_per_chunk = 1; } - uint32_t dr = (nrows + (uint32_t) kparams->n_threads - 1) / (uint32_t) kparams->n_threads; + uint32_t dr = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); if (rows_per_chunk > 1) { dr = hex_round_up(dr, rows_per_chunk); } - mctx.nrows_per_thread = dr; + mctx.nrows_per_hvx = dr; - work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, (uint32_t) kparams->n_threads); + work_queue_run(octx->ctx->hvx_queue, htp_sort_multi_row_job, &mctx, octx->n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.h b/ggml/src/ggml-hexagon/htp/argsort-ops.h index 05320a2e9..d1cb8087b 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.h +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.h @@ -7,53 +7,80 @@ #include <string.h> #include "hex-fastdiv.h" +#include "htp-vtcm.h" struct htp_sort_kernel_params { - int32_t n_threads; - int32_t total_rows; - int32_t row_start; - int32_t row_end; - int32_t ne00; - int32_t k; - int32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1) - int32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT - int32_t use_dma; // 1 if DMA enabled - int32_t chunk_elems; - int32_t n_chunks; - int32_t vtcm_size; - int32_t phase1_slot_size; - int32_t merge_values_off; - int32_t merge_indices_off; - int32_t merge_elems; - int32_t n_slots; - int32_t pad[15]; + uint32_t n_hvx; + uint32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1) + uint32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT + uint32_t chunk_elems; + uint32_t n_chunks; + uint32_t vtcm_size; + uint32_t merge_elems; + uint32_t n_slots; + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne01_ne02; }; struct htp_sort_vtcm_layout { size_t total_bytes; size_t phase1_slot_size; - size_t merge_values_off; - size_t merge_indices_off; + size_t off_phase1; + size_t off_merge_values; + size_t off_merge_indices; uint32_t chunk_elems; uint32_t n_chunks; uint32_t merge_elems; - uint32_t n_threads; + uint32_t n_hvx; uint32_t n_slots; }; +static inline void htp_sort_vtcm_layout_build( + struct htp_sort_vtcm_layout * L, + uint32_t total_rows, + uint32_t chunk_elems, + uint32_t n_chunks, + uint32_t merge_elems, + uint32_t n_hvx, + uint32_t n_slots) { + + memset(L, 0, sizeof(*L)); + L->chunk_elems = chunk_elems; + L->n_chunks = n_chunks; + L->merge_elems = merge_elems; + L->n_hvx = n_hvx; + L->n_slots = n_slots; + + const size_t values_size = ((chunk_elems * sizeof(float)) + 127) & ~127; + const size_t indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127; + L->phase1_slot_size = ((values_size + indices_size) + 255) & ~255; + + size_t off = 0; + if (total_rows > 1) { + VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_hvx); + } else { + VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_chunks * n_slots); + const size_t merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127; + const size_t merge_indices_size = ((merge_elems * sizeof(int32_t)) + 127) & ~127; + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_values, merge_values_size, n_chunks > 1); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_indices, merge_indices_size, n_chunks > 1); + } + L->total_bytes = off; +} + static inline bool htp_sort_solve_layout( struct htp_sort_vtcm_layout * layout, uint32_t ne00, uint32_t total_rows, uint32_t k, - uint32_t n_threads_max, + uint32_t n_hvx_max, size_t vtcm_budget, bool is_top_k) { memset(layout, 0, sizeof(*layout)); if (total_rows > 1) { - uint32_t n_threads = total_rows < n_threads_max ? total_rows : n_threads_max; + uint32_t n_hvx = total_rows < n_hvx_max ? total_rows : n_hvx_max; uint32_t n_vec = (ne00 + 31) / 32; uint32_t n_vec_pow2 = 1; @@ -69,24 +96,14 @@ static inline bool htp_sort_solve_layout( n_slots = 1; } - size_t spad_per_thread = spad_per_slot * n_slots; - - while (n_threads > 1 && (spad_per_thread * n_threads) > vtcm_budget) { - n_threads--; - } + size_t spad_per_hvx = spad_per_slot * n_slots; - size_t total_bytes = spad_per_thread * n_threads; - if (total_bytes > vtcm_budget) { - return false; + while (n_hvx > 1 && (spad_per_hvx * n_hvx) > vtcm_budget) { + n_hvx--; } - layout->total_bytes = total_bytes; - layout->phase1_slot_size = spad_per_slot; - layout->chunk_elems = ne00_padded; - layout->n_chunks = 1; - layout->n_threads = n_threads; - layout->n_slots = n_slots; - return true; + htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_hvx, n_slots); + return layout->total_bytes <= vtcm_budget; } uint32_t n_vec = (ne00 + 31) / 32; @@ -95,24 +112,14 @@ static inline bool htp_sort_solve_layout( uint32_t n_chunks = 1; if (ne00 > 1024) { - while (n_chunks * 2 <= n_threads_max && n_chunks * 2 <= n_vec_pow2) { + while (n_chunks * 2 <= n_hvx_max && n_chunks * 2 <= n_vec_pow2) { n_chunks *= 2; } } uint32_t chunk_n_vec = n_vec_pow2 / n_chunks; uint32_t chunk_elems = chunk_n_vec * 32; - - size_t phase1_values_size = ((chunk_elems * sizeof(float)) + 127) & ~127; - size_t phase1_indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127; - size_t phase1_slot_size = ((phase1_values_size + phase1_indices_size) + 255) & ~255; - size_t phase1_total_size = phase1_slot_size * n_chunks; - - size_t merge_values_size = 0; - size_t merge_indices_size = 0; - size_t merge_values_off = phase1_total_size; - size_t merge_indices_off = 0; - uint32_t merge_elems = 0; + uint32_t merge_elems = 0; if (n_chunks > 1) { if (is_top_k) { @@ -125,47 +132,25 @@ static inline bool htp_sort_solve_layout( } else { merge_elems = n_vec_pow2 * 32; } - merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127; - merge_indices_size = ((merge_elems * sizeof(int32_t)) + 127) & ~127; - merge_indices_off = merge_values_off + merge_values_size; } - size_t total_bytes = phase1_total_size + merge_values_size + merge_indices_size; + htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, n_chunks, 1); - if (total_bytes > vtcm_budget && n_chunks > 1) { - n_chunks = 1; + if (layout->total_bytes > vtcm_budget && n_chunks > 1) { + n_chunks = 1; chunk_elems = n_vec_pow2 * 32; - phase1_values_size = ((chunk_elems * sizeof(float)) + 127) & ~127; - phase1_indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127; - phase1_slot_size = ((phase1_values_size + phase1_indices_size) + 255) & ~255; - phase1_total_size = phase1_slot_size; - merge_values_size = 0; - merge_indices_size = 0; - merge_values_off = phase1_total_size; - merge_indices_off = 0; - merge_elems = 0; - total_bytes = phase1_total_size; + merge_elems = 0; + htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, 1, 1); } - if (total_bytes > vtcm_budget) { + if (layout->total_bytes > vtcm_budget) { return false; } - uint32_t n_slots = 1; - if (n_chunks == 1 && phase1_slot_size * 2 <= vtcm_budget) { - n_slots = 2; - total_bytes = phase1_slot_size * 2; + if (n_chunks == 1 && layout->phase1_slot_size * 2 <= vtcm_budget) { + htp_sort_vtcm_layout_build(layout, 1, chunk_elems, 1, 0, 1, 2); } - layout->total_bytes = total_bytes; - layout->phase1_slot_size = phase1_slot_size; - layout->merge_values_off = merge_values_off; - layout->merge_indices_off = merge_indices_off; - layout->chunk_elems = chunk_elems; - layout->n_chunks = n_chunks; - layout->merge_elems = merge_elems; - layout->n_threads = n_chunks; - layout->n_slots = n_slots; return true; } diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index 1286176af..a5951556b 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -14,7 +14,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "binary-ops.h" #include "htp-ctx.h" #include "htp-ops.h" @@ -39,9 +39,10 @@ struct htp_binary_context { struct fastdiv_values src1_dim3_div; // ne13 uint32_t block_max; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_rows; uint32_t row_start; + uint32_t n_srcs; size_t src0_row_size_aligned; size_t src1_row_size_aligned; size_t dst_row_size_aligned; @@ -103,13 +104,13 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_ // Out-of-line compute micro-kernels -typedef void (*compute_scalar_dma_t)( +typedef void (*compute_scalar_t)( uint8_t * dst, const uint8_t * src0, const void * s1_table, uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); -#define DEFINE_COMPUTE_SCALAR_DMA(NAME, TYPE, HVX_STMT) \ -static void compute_scalar_dma_##NAME( \ +#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \ +static void compute_scalar_##NAME( \ uint8_t * dst, const uint8_t * src0, const void * s1_table, \ uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, \ size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ @@ -125,31 +126,6 @@ static void compute_scalar_dma_##NAME( \ } \ } -DEFINE_COMPUTE_SCALAR_DMA(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(sub_f16, _Float16, hvx_sub_scalar_f16_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(mul_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(mul_f16, _Float16, hvx_mul_scalar_f16_aa(r_dst, r_src0, val, ne00)) -DEFINE_COMPUTE_SCALAR_DMA(div_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, 1.0f / (val), ne00)) -DEFINE_COMPUTE_SCALAR_DMA(div_f16, _Float16, hvx_div_scalar_f16_aa(r_dst, r_src0, val, ne00)) - -typedef void (*compute_scalar_t)( - uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride, - uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); - -#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \ -static void compute_scalar_##NAME( \ - uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride, \ - uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ - for (uint32_t r = 0; r < n_rows; r++) { \ - uint8_t * r_dst = dst + r * dst_stride; \ - const uint8_t * r_src0 = src0 + r * src0_stride; \ - TYPE val = *(const TYPE *)(src1_ptr + r * s1_stride); \ - HVX_STMT; \ - } \ -} - DEFINE_COMPUTE_SCALAR(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00)) DEFINE_COMPUTE_SCALAR(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00)) DEFINE_COMPUTE_SCALAR(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00)) @@ -205,17 +181,17 @@ DEFINE_COMPUTE_ROW_BCAST(div_f16, hvx_div_f16_aaa) typedef void (*compute_complex_t)( uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, - uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); #define DEFINE_COMPUTE_COMPLEX(NAME, HVX_FN) \ static void compute_complex_##NAME( \ uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \ - uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \ + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \ uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ for (uint32_t r = 0; r < n_rows; r++) { \ uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \ - const uint8_t * r_src1 = src1_plane + i11 * nb11; \ + const uint8_t * r_src1 = src1_plane + i11 * src1_stride; \ HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, r_src1, ne00); \ } \ } @@ -231,17 +207,17 @@ DEFINE_COMPUTE_COMPLEX(div_f16, hvx_div_f16_aau) typedef void (*compute_repeat_t)( uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, - uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10); #define DEFINE_COMPUTE_REPEAT(NAME, TYPE, HVX_FN) \ static void compute_repeat_##NAME( \ uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \ - uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \ + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \ uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10) { \ for (uint32_t r = 0; r < n_rows; r++) { \ uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \ - const uint8_t * r_src1_row = src1_plane + i11 * nb11; \ + const uint8_t * r_src1_row = src1_plane + i11 * src1_stride; \ uint8_t * r_dst = dst + r * dst_stride; \ const uint8_t * r_src0 = src0 + r * src0_stride; \ for (uint32_t c = 0; c < ne00; c += ne10) { \ @@ -261,17 +237,17 @@ DEFINE_COMPUTE_REPEAT(div_f32, float, hvx_div_f32_uuu) DEFINE_COMPUTE_REPEAT(div_f16, _Float16, hvx_div_f16_uuu) typedef void (*compute_add_id_t)( - uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data, - uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride, + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids, + uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); static void compute_add_id_f32( - uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data, - uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride, + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids, + uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride, uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { for (uint32_t r = 0; r < n_rows; r++) { uint32_t r_i01 = i01 + r; - const int32_t idx = *(const int32_t *)(src2_data + r_i01 * nb20 + i02 * nb21); + const int32_t idx = src2_ids[i02 * ne01 + r_i01]; if (idx < 0) { memcpy(dst + r * dst_stride, src0 + r * src0_stride, ne00 * sizeof(float)); continue; @@ -283,28 +259,32 @@ static void compute_add_id_f32( } } -// 1a. Scalar src1 in VTCM via DMA (ne10 == 1, ne12 == 1, ne13 == 1) -static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void * data) { +// 1. Scalar src1 in VTCM via DMA (ne10 == 1) +static void binary_thread_scalar(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t plane_stride = (size_t) ne12 * ne11 * elem_size; + const size_t row_stride = (size_t) ne11 * elem_size; + const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-scalar-dma: %d/%d (%u:%u) row-size %u (%u)", + FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; const void * s1_table = VTCM_LAYOUT_PTR(const void, bctx->vtcm_base, layout->off_src1); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -329,95 +309,7 @@ static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - compute_scalar_dma_t compute = (compute_scalar_dma_t) bctx->compute; - - for (uint32_t ir = start_row; ir < end_row; ) { - uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - - uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; - - uint32_t i03, i02, i01, rem; - i03 = fastdiv(ir, &bctx->src0_dim12_div); - rem = ir - i03 * (ne02 * ne01); - i02 = fastdiv(rem, &bctx->src0_dim1_div); - i01 = rem - i02 * ne01; - - uint32_t cur_i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - compute(d_spad, s0_spad, s1_table, cur_i11, ne11, current_block_size, - bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - - dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, - current_block_size); - - if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03, p02, p01, prem; - p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - prem = ir_prefetch - p03 * (ne02 * ne01); - p02 = fastdiv(prem, &bctx->src0_dim1_div); - p01 = prem - p02 * ne01; - dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, - next_block_size); - ir_prefetch += next_block_size; - } - ir += current_block_size; - } - - dma_queue_flush(dma_q); -} - -// 1b. Scalar src1 dynamic / pointer (ne10 == 1) -static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) { - struct htp_binary_context * bctx = (struct htp_binary_context *) data; - struct htp_ops_context * octx = bctx->octx; - htp_binary_preamble; - - const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); - if (start_row >= end_row) return; - - FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", - ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - - const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); - size_t src0_spad_half = layout->src0_spad_half_size; - size_t dst_spad_half = layout->dst_spad_half_size; - - dma_queue * dma_q = octx->ctx->dma[ith]; - uint32_t ir_prefetch = start_row; - int spad_idx = 0; - - for (int k = 0; k < 2 && ir_prefetch < end_row; k++) { - uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t i03, i02, i01, rem; - i03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - rem = ir_prefetch - i03 * (ne02 * ne01); - i02 = fastdiv(rem, &bctx->src0_dim1_div); - i01 = rem - i02 * ne01; - - dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - - uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; - uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - - dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); - ir_prefetch += current_block_size; - spad_idx ^= 1; - } - - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_scalar_t compute = (compute_scalar_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -435,17 +327,17 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); uint32_t i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div); - - const uint8_t * src1_ptr = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint32_t s1_stride = (ne11 == 1) ? 0 : nb11; + const uint8_t * scalar_table = (const uint8_t *) s1_table + + i13 * plane_stride + i12 * row_stride; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - compute(d_spad, s0_spad, src1_ptr, s1_stride, current_block_size, + compute(d_spad, s0_spad, scalar_table, i11, ne11, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, + current_block_size); if (ir_prefetch < end_row) { uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); @@ -455,7 +347,8 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data p02 = fastdiv(prem, &bctx->src0_dim1_div); p01 = prem - p02 * ne01; dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, + next_block_size); ir_prefetch += next_block_size; } ir += current_block_size; @@ -465,29 +358,30 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data } // 2. Vector Same Shape (ne1x == ne0x) or Simple Broadcast -static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_same_shape(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t src1_spad_half = layout->src1_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; + const uint32_t n_srcs = bctx->n_srcs; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -499,12 +393,7 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, i02 = fastdiv(rem, &bctx->src0_dim1_div); i01 = rem - i02 * ne01; - uint32_t i13 = (ne13 == 1) ? 0 : i03; - uint32_t i12 = (ne12 == 1) ? 0 : i02; - uint32_t i11 = (ne11 == 1) ? 0 : i01; - dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - dma_addr_t src1_curr = src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; @@ -513,12 +402,20 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); - dma_queue_push(dma_q, dma_make_data(s1_spad, src1_curr), bctx->src1_row_size_aligned, nb11, row_size_bytes, current_block_size); + for (uint32_t s = 1; s < n_srcs; s++) { + const struct htp_tensor * src_s = octx->src[s]; + uint32_t is3 = (src_s->ne[3] == 1) ? 0 : i03; + uint32_t is2 = (src_s->ne[2] == 1) ? 0 : i02; + uint32_t is1 = (src_s->ne[1] == 1) ? 0 : i01; + dma_addr_t s_curr = src_s->data + is3 * src_s->nb[3] + is2 * src_s->nb[2] + is1 * src_s->nb[1]; + uint8_t * s_spad = s1_spad + (s - 1) * 2 * src1_spad_half; + dma_queue_push(dma_q, dma_make_data(s_spad, s_curr), bctx->src1_row_size_aligned, src_s->nb[1], row_size_bytes, current_block_size); + } ir_prefetch += current_block_size; spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_same_shape_t compute = (compute_same_shape_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -530,6 +427,11 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); compute(d_spad, s0_spad, s1_spad, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, bctx->src1_row_size_aligned, ne00); + for (uint32_t s = 2; s < n_srcs; s++) { + uint8_t * ss_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + compute(d_spad, d_spad, ss_spad, current_block_size, + bctx->dst_row_size_aligned, bctx->dst_row_size_aligned, bctx->src1_row_size_aligned, ne00); + } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03, i02, i01, rem; @@ -548,15 +450,18 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, p02 = fastdiv(prem, &bctx->src0_dim1_div); p01 = prem - p02 * ne01; - uint32_t p13 = (ne13 == 1) ? 0 : p03; - uint32_t p12 = (ne12 == 1) ? 0 : p02; - uint32_t p11 = (ne11 == 1) ? 0 : p01; - dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_addr_t s1_next = src1->data + p13 * nb13 + p12 * nb12 + p11 * nb11; - dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - dma_queue_push(dma_q, dma_make_data(s1_spad, s1_next), bctx->src1_row_size_aligned, nb11, row_size_bytes, next_block_size); + + for (uint32_t s = 1; s < n_srcs; s++) { + const struct htp_tensor * src_s = octx->src[s]; + uint32_t ps3 = (src_s->ne[3] == 1) ? 0 : p03; + uint32_t ps2 = (src_s->ne[2] == 1) ? 0 : p02; + uint32_t ps1 = (src_s->ne[1] == 1) ? 0 : p01; + dma_addr_t s_next = src_s->data + ps3 * src_s->nb[3] + ps2 * src_s->nb[2] + ps1 * src_s->nb[1]; + uint8_t * s_spad = s1_spad + (s - 1) * 2 * src1_spad_half; + dma_queue_push(dma_q, dma_make_data(s_spad, s_next), bctx->src1_row_size_aligned, src_s->nb[1], row_size_bytes, next_block_size); + } ir_prefetch += next_block_size; } @@ -567,27 +472,27 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, } // 3. Row Broadcast (ne11 == 1, ne12 == 1, single row src1) -static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_row_broadcast(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -612,7 +517,7 @@ static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int it spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_row_bcast_t compute = (compute_row_bcast_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -650,26 +555,32 @@ static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int it } // 4. Vector Complex (ne10 == ne00, complex broadcast) -static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_complex(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; + const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); + + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t vtcm_row_stride = (size_t) ne10 * elem_size; + const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride; + const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -692,7 +603,7 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_complex_t compute = (compute_complex_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -707,10 +618,10 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); - const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12; + const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11, + compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); @@ -735,26 +646,32 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi } // 5. Element Repeat (ne10 != ne00) -static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_element_repeat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; + const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); + + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t vtcm_row_stride = (size_t) ne10 * elem_size; + const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride; + const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride; FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -777,7 +694,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_repeat_t compute = (compute_repeat_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -792,10 +709,10 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); - const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12; + const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11, + compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00, ne10); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); @@ -820,13 +737,12 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi } // 6. ADD_ID (src1 gathered via src2 indices) -static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_add_id_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; - const struct htp_tensor * src2 = octx->src[2]; const struct htp_tensor * dst = octx->dst; const uint32_t ne00 = src0->ne[0]; @@ -844,18 +760,19 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); + const int32_t * vtcm_src2 = VTCM_LAYOUT_PTR(const int32_t, bctx->vtcm_base, layout->off_src2); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -878,7 +795,7 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); @@ -892,8 +809,8 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); compute_add_id_t compute = (compute_add_id_t) bctx->compute; - compute(d_spad, s0_spad, vtcm_src1, (const char *)(uintptr_t)src2->data, - i01, i02, src2->nb[0], src2->nb[1], src1_stride, + compute(d_spad, s0_spad, vtcm_src1, vtcm_src2, + i01, i02, ne01, src1_stride, current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); @@ -953,8 +870,9 @@ struct binary_chunked_context { uint32_t elem_start; uint32_t nelem; uint32_t chunk_size; - uint32_t chunks_per_thread; + uint32_t chunks_per_hvx; uint32_t total_chunks; + uint32_t n_srcs; bool is_scalar; float scalar_f32; _Float16 scalar_f16; @@ -963,16 +881,15 @@ struct binary_chunked_context { compute_binary_scalar_chunked_f16_t compute_scalar_f16; }; -static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_chunked(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct binary_chunked_context * ctx = (struct binary_chunked_context *) data; struct htp_ops_context * octx = ctx->octx; const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - const uint32_t start_chunk = ctx->chunks_per_thread * ith; - const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_thread, ctx->total_chunks); + const uint32_t start_chunk = ctx->chunks_per_hvx * ith; + const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_hvx, ctx->total_chunks); if (start_chunk >= end_chunk) { return; } @@ -986,11 +903,12 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat ith, nth, start_chunk, end_chunk, ctx->total_chunks, ctx->nelem); const struct htp_binary_vtcm_layout * layout = &ctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread)); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_hvx)); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; + const uint32_t n_srcs = ctx->n_srcs; uint32_t prefetch_chunk = start_chunk; int spad_idx = 0; @@ -1009,16 +927,19 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat dma_queue_push(dma_q, dma_make_data(d_curr, d_spad), chunk_bytes, chunk_bytes, cur_bytes, 0); dma_queue_push(dma_q, dma_make_data(s0_spad, s0_curr), chunk_bytes, chunk_bytes, cur_bytes, 1); if (!ctx->is_scalar) { - dma_addr_t s1_curr = src1->data + (size_t) c_start * elem_size; uint8_t * s1_spad = src1_spad_base + spad_idx * chunk_bytes; - dma_queue_push(dma_q, dma_make_data(s1_spad, s1_curr), chunk_bytes, chunk_bytes, cur_bytes, 1); + for (uint32_t s = 1; s < n_srcs; s++) { + dma_addr_t s_curr = octx->src[s]->data + (size_t) c_start * elem_size; + uint8_t * s_spad = s1_spad + (s - 1) * 2 * chunk_bytes; + dma_queue_push(dma_q, dma_make_data(s_spad, s_curr), chunk_bytes, chunk_bytes, cur_bytes, 1); + } } prefetch_chunk++; spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t c = start_chunk; c < end_chunk; c++) { const uint32_t c_start = ctx->elem_start + c * chunk_size; @@ -1039,6 +960,10 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat } } else { ctx->compute(d_spad, s0_spad, s1_spad, cur_elems); + for (uint32_t s = 2; s < n_srcs; s++) { + uint8_t * s_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + ctx->compute(d_spad, d_spad, s_spad, cur_elems); + } } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) c); @@ -1054,8 +979,11 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat dma_addr_t s0_next = src0->data + (size_t) pc_start * elem_size; dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), chunk_bytes, chunk_bytes, p_bytes, 1); if (!ctx->is_scalar) { - dma_addr_t s1_next = src1->data + (size_t) pc_start * elem_size; - dma_queue_push(dma_q, dma_make_data(s1_spad, s1_next), chunk_bytes, chunk_bytes, p_bytes, 1); + for (uint32_t s = 1; s < n_srcs; s++) { + dma_addr_t s_next = octx->src[s]->data + (size_t) pc_start * elem_size; + uint8_t * s_spad = s1_spad + (s - 1) * 2 * chunk_bytes; + dma_queue_push(dma_q, dma_make_data(s_spad, s_next), chunk_bytes, chunk_bytes, p_bytes, 1); + } } prefetch_chunk++; @@ -1070,6 +998,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; const struct htp_binary_kernel_params * kparams = (const struct htp_binary_kernel_params *) octx->kernel_params; + const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; @@ -1081,7 +1010,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { const size_t dst_row_size = dst->ne[0] * elem_size; if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) { - const bool is_scalar = kparams->is_scalar || (src1->ne[0] == 1 && src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1); + const bool is_scalar = kparams->is_scalar; compute_binary_chunked_t compute = NULL; compute_binary_scalar_chunked_f32_t compute_scalar_f32 = NULL; @@ -1141,12 +1070,16 @@ static int execute_op_binary(struct htp_ops_context * octx) { const uint32_t elems_per_line = (src0_type == HTP_TYPE_F32) ? 32 : 64; if (octx->ctx->mdev.count > 1) { - const bool can_split = htp_tensor_mdev_data_aligned(dst) && - htp_tensor_mdev_data_aligned(src0) && - (is_scalar || htp_tensor_mdev_data_aligned(src1)) && - htp_tensor_is_contiguous(dst, elem_size) && - htp_tensor_is_contiguous(src0, elem_size) && - (is_scalar || htp_tensor_is_contiguous(src1, elem_size)); + bool can_split = htp_tensor_mdev_data_aligned(dst) && + htp_tensor_mdev_data_aligned(src0) && + (is_scalar || htp_tensor_mdev_data_aligned(src1)) && + htp_tensor_is_contiguous(dst, elem_size) && + htp_tensor_is_contiguous(src0, elem_size) && + (is_scalar || htp_tensor_is_contiguous(src1, elem_size)); + for (uint32_t s = 2; can_split && s < n_srcs; s++) { + can_split = htp_tensor_mdev_data_aligned(octx->src[s]) && + htp_tensor_is_contiguous(octx->src[s], elem_size); + } const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( total_elems, can_split ? elems_per_line : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); @@ -1158,7 +1091,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -1168,10 +1101,10 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t chunk_size = kparams->chunk_size > 0 ? kparams->chunk_size : (32768 / elem_size); + const uint32_t chunk_size = kparams->chunk_size; const uint32_t total_chunks = (nelem + chunk_size - 1) / chunk_size; - const uint32_t n_threads = (total_chunks >= 2) ? MIN(octx->n_threads, total_chunks) : 1; - const uint32_t chunks_per_thread = (total_chunks + n_threads - 1) / n_threads; + const uint32_t n_hvx = (total_chunks >= 2) ? MIN(octx->n_hvx, total_chunks) : 1; + const uint32_t chunks_per_hvx = (total_chunks + n_hvx - 1) / n_hvx; float scalar_f32 = 0.0f; _Float16 scalar_f16 = 0; @@ -1195,8 +1128,9 @@ static int execute_op_binary(struct htp_ops_context * octx) { .elem_start = elem_start, .nelem = nelem, .chunk_size = chunk_size, - .chunks_per_thread = chunks_per_thread, + .chunks_per_hvx = chunks_per_hvx, .total_chunks = total_chunks, + .n_srcs = n_srcs, .is_scalar = is_scalar, .scalar_f32 = scalar_f32, .scalar_f16 = scalar_f16, @@ -1205,7 +1139,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { .compute_scalar_f16 = compute_scalar_f16, }; - work_queue_run(octx->ctx->work_queue, binary_thread_chunked, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, binary_thread_chunked, &cctx, n_hvx); return HTP_STATUS_OK; } @@ -1225,28 +1159,15 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; const size_t src1_row_size_aligned = kparams->src1_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; - if (htp_tensor_is_extended(src1)) { - if (kparams->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE && - kparams->kernel_type != HTP_BINARY_KERNEL_ROW_BCAST && - kparams->kernel_type != HTP_BINARY_KERNEL_SCALAR_DMA && - kparams->kernel_type != HTP_BINARY_KERNEL_ADD_ID) { - return HTP_STATUS_NO_SUPPORT; - } - } - - if (octx->op == HTP_OP_ADD_ID && htp_tensor_is_extended(octx->src[2])) { - return HTP_STATUS_NO_SUPPORT; - } - struct htp_binary_context bctx; bctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; htp_binary_vtcm_layout_build(&bctx.vtcm_layout, kparams, octx->ctx->vtcm_size); @@ -1255,47 +1176,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - dma_queue * dma_q = octx->ctx->dma[0]; - uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1); - if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { - dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1); - } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA) { - dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[1] * elem_size, 1); - } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { - dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), - kparams->src1_row_size_aligned, src1->nb[1], - src1->ne[0] * elem_size, src1->ne[1]); - } - - bctx.octx = octx; - bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); - bctx.total_rows = nrows; - bctx.row_start = row_start; - bctx.block_max = bctx.vtcm_layout.rows_per_buffer; - bctx.src0_row_size_aligned = src0_row_size_aligned; - bctx.src1_row_size_aligned = src1_row_size_aligned; - bctx.dst_row_size_aligned = dst_row_size_aligned; - bctx.row_size_bytes = src0_row_size; - - bctx.src0_dim1_div = init_fastdiv_values(src0->ne[1]); - bctx.src0_dim2_div = init_fastdiv_values(src0->ne[2]); - bctx.src0_dim12_div = init_fastdiv_values(src0->ne[1] * src0->ne[2]); - - bctx.src1_dim1_div = init_fastdiv_values(src1->ne[1]); - bctx.src1_dim2_div = init_fastdiv_values(src1->ne[2]); - bctx.src1_dim3_div = init_fastdiv_values(src1->ne[3]); - - bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]); - bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]); - - bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]); - bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]); - - bctx.split_at_ne01 = (octx->op == HTP_OP_ADD_ID) || - ((src0->ne[2] > 1) && ((src1->ne[1] > 1) || (src1->ne[2] > 1) || !src0_contig_dim1 || !dst_contig_dim1)); - bctx.split_at_ne02 = (src0->ne[3] > 1) && ((src1->ne[2] > 1) || (src1->ne[3] > 1) || !src0_contig_dim2 || !dst_contig_dim2); - - worker_callback_t worker_func = NULL; + work_queue_func_t worker_func = NULL; void * compute_func = NULL; switch (kparams->kernel_type) { @@ -1339,26 +1220,6 @@ static int execute_op_binary(struct htp_ops_context * octx) { } } break; - case HTP_BINARY_KERNEL_SCALAR_DMA: - worker_func = binary_thread_scalar_dma; - if (src0_type == HTP_TYPE_F32) { - switch (octx->op) { - case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f32; break; - case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f32; break; - case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f32; break; - case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f32; break; - default: break; - } - } else if (src0_type == HTP_TYPE_F16) { - switch (octx->op) { - case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f16; break; - case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f16; break; - case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f16; break; - case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f16; break; - default: break; - } - } - break; case HTP_BINARY_KERNEL_SCALAR: worker_func = binary_thread_scalar; if (src0_type == HTP_TYPE_F32) { @@ -1432,27 +1293,94 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - bctx.compute = compute_func; + dma_queue * dma_q = octx->ctx->dma[0]; + uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1); + if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1); + } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR || + kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT || + kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) { + if (!kparams->src1_strided) { + const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size; + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1); + if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR && octx->src[2] != NULL) { + uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2); + dma_queue_push(dma_q, dma_make_data(vtcm_src2, octx->src[2]->data), bctx.vtcm_layout.src2_size, 0, src1_bytes, 1); + } + } else { + const uint32_t ne11 = src1->ne[1]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t ne13 = src1->ne[3]; + const size_t row_bytes = src1->ne[0] * elem_size; + for (uint32_t i13 = 0; i13 < ne13; i13++) { + for (uint32_t i12 = 0; i12 < ne12; i12++) { + dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2]; + uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes; + while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr), + row_bytes, src1->nb[1], row_bytes, ne11)) { + dma_queue_pop(dma_q); + } + } + } + } + } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { + const struct htp_tensor * src2 = octx->src[2]; + uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2); + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), + kparams->src1_row_size_aligned, src1->nb[1], + src1->ne[0] * elem_size, src1->ne[1]); + const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data), + src2_row_bytes, src2->nb[1], + src2_row_bytes, src2->ne[1]); + } + + bctx.octx = octx; + bctx.nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); + bctx.total_rows = nrows; + bctx.row_start = row_start; + bctx.n_srcs = n_srcs; + bctx.block_max = bctx.vtcm_layout.rows_per_buffer; + bctx.src0_row_size_aligned = src0_row_size_aligned; + bctx.src1_row_size_aligned = src1_row_size_aligned; + bctx.dst_row_size_aligned = dst_row_size_aligned; + bctx.row_size_bytes = src0_row_size; + + bctx.src0_dim1_div = kparams->div_ne01; + bctx.src0_dim2_div = kparams->div_ne02; + bctx.src0_dim12_div = kparams->div_ne012; + + bctx.src1_dim1_div = kparams->div_ne11; + bctx.src1_dim2_div = kparams->div_ne12; + bctx.src1_dim3_div = kparams->div_ne13; + + bctx.split_at_ne01 = kparams->split_at_ne01; + bctx.split_at_ne02 = kparams->split_at_ne02; + bctx.compute = compute_func; if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST || - kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA || + kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR || + kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT || + kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX || kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { - dma_queue_pop(dma_q); + dma_queue_flush(dma_q); + if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR && octx->src[2] != NULL && octx->op == HTP_OP_MUL) { + uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2); + const uint32_t n_scalars = src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3]; + if (src0_type == HTP_TYPE_F32) { + hvx_mul_f32_aaa(vtcm_src1, vtcm_src1, vtcm_src2, n_scalars); + } else if (src0_type == HTP_TYPE_F16) { + hvx_mul_f16_aaa(vtcm_src1, vtcm_src1, vtcm_src2, n_scalars); + } + } } - work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker_func, &bctx, n_hvx); return HTP_STATUS_OK; } int op_binary(struct htp_ops_context * octx) { - - // Does not support permutations of src1 - const struct htp_tensor * src1 = octx->src[1]; - if (src1->nb[1] < src1->nb[0]) { - return HTP_STATUS_NO_SUPPORT; - } - const uint32_t src0_type = octx->src[0]->type; if ((src0_type == HTP_TYPE_F32) || (src0_type == HTP_TYPE_F16)) { return execute_op_binary(octx); diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h index aa8ff4959..6f849396c 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.h +++ b/ggml/src/ggml-hexagon/htp/binary-ops.h @@ -6,13 +6,13 @@ #include <string.h> #include "hex-common.h" +#include "hex-fastdiv.h" #include "htp-ops.h" #include "htp-vtcm.h" enum htp_binary_kernel_type { HTP_BINARY_KERNEL_SAME_SHAPE = 0, HTP_BINARY_KERNEL_ROW_BCAST, - HTP_BINARY_KERNEL_SCALAR_DMA, HTP_BINARY_KERNEL_SCALAR, HTP_BINARY_KERNEL_ADD_ID, HTP_BINARY_KERNEL_COMPLEX, @@ -21,8 +21,16 @@ enum htp_binary_kernel_type { }; struct htp_binary_kernel_params { - uint32_t kernel_type; - uint32_t n_threads; + uint8_t kernel_type; + uint8_t split_at_ne01; + uint8_t split_at_ne02; + uint8_t src1_strided; + + uint8_t n_srcs; + uint8_t is_scalar; + uint8_t pad[2]; + + uint32_t n_hvx; uint32_t rows_per_buffer; uint32_t src0_row_size_aligned; @@ -30,15 +38,43 @@ struct htp_binary_kernel_params { uint32_t dst_row_size_aligned; uint32_t src1_size; + uint32_t src2_size; uint32_t vtcm_size; uint32_t chunk_size; uint32_t chunk_bytes; - uint32_t is_scalar; + + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne012; + + struct fastdiv_values div_ne11; + struct fastdiv_values div_ne12; + struct fastdiv_values div_ne13; }; #if defined(__cplusplus) static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob"); + +struct ggml_tensor; +static inline bool htp_binary_strides_safe( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + size_t elem_size +) { + if ((src0->ne[0] != 1 && src0->nb[0] != elem_size) || + (src1->ne[0] != 1 && src1->nb[0] != elem_size) || + (dst->ne[0] != 1 && dst->nb[0] != elem_size)) { + return false; + } + if ((src0->ne[1] > 1 && src0->nb[1] < src0->ne[0] * elem_size) || + (src1->ne[1] > 1 && src1->nb[1] < src1->ne[0] * elem_size) || + (dst->ne[1] > 1 && dst->nb[1] < dst->ne[0] * elem_size)) { + return false; + } + return true; +} #else _Static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob"); #endif @@ -47,17 +83,19 @@ struct htp_binary_vtcm_layout { size_t total_bytes; size_t off_src0; size_t off_src1; + size_t off_src2; size_t off_dst; - size_t src0_bytes_per_thread; - size_t src1_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t src1_spad_half_size; size_t dst_spad_half_size; size_t src1_size; + size_t src2_size; uint32_t rows_per_buffer; }; @@ -68,20 +106,22 @@ static inline void htp_binary_vtcm_layout_build( ) { memset(L, 0, sizeof(*L)); - const uint32_t n_threads = kparams->n_threads; - if (n_threads == 0) { + const uint32_t n_hvx = kparams->n_hvx; + if (n_hvx == 0) { return; } + const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2; + if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) { const size_t chunk_bytes = kparams->chunk_bytes; if (chunk_bytes == 0) { return; } - L->src0_bytes_per_thread = 2 * chunk_bytes; - L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * chunk_bytes); - L->dst_bytes_per_thread = 2 * chunk_bytes; + L->src0_bytes_per_hvx = 2 * chunk_bytes; + L->src1_bytes_per_hvx = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes); + L->dst_bytes_per_hvx = 2 * chunk_bytes; L->src0_spad_half_size = chunk_bytes; L->src1_spad_half_size = kparams->is_scalar ? 0 : chunk_bytes; @@ -90,9 +130,9 @@ static inline void htp_binary_vtcm_layout_build( L->rows_per_buffer = 1; L->src1_size = 0; - const size_t src0_total = n_threads * L->src0_bytes_per_thread; - const size_t src1_total = kparams->is_scalar ? 128 : (n_threads * L->src1_bytes_per_thread); - const size_t dst_total = n_threads * L->dst_bytes_per_thread; + const size_t src0_total = n_hvx * L->src0_bytes_per_hvx; + const size_t src1_total = kparams->is_scalar ? 128 : (n_hvx * L->src1_bytes_per_hvx); + const size_t dst_total = n_hvx * L->dst_bytes_per_hvx; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); @@ -108,43 +148,100 @@ static inline void htp_binary_vtcm_layout_build( } const size_t spad_row_total = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) - ? 2 * (kparams->src0_row_size_aligned + kparams->src1_row_size_aligned + kparams->dst_row_size_aligned) + ? 2 * (kparams->src0_row_size_aligned + (n_srcs - 1) * kparams->src1_row_size_aligned + kparams->dst_row_size_aligned) : 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned); - if (spad_row_total == 0 || vtcm_size < kparams->src1_size) { + const size_t static_vtcm_size = kparams->src1_size + kparams->src2_size; + if (spad_row_total == 0 || vtcm_size < static_vtcm_size) { return; } - const size_t rows_per_buffer = (vtcm_size - kparams->src1_size) / (n_threads * spad_row_total); + const size_t rows_per_buffer = (kparams->rows_per_buffer > 0) + ? kparams->rows_per_buffer + : (vtcm_size - static_vtcm_size) / (n_hvx * spad_row_total); if (rows_per_buffer == 0) { return; } L->rows_per_buffer = (uint32_t) rows_per_buffer; L->src1_size = kparams->src1_size; + L->src2_size = kparams->src2_size; - L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned; - L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned; - L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) - ? rows_per_buffer * 2 * kparams->src1_row_size_aligned + L->src0_bytes_per_hvx = rows_per_buffer * 2 * kparams->src0_row_size_aligned; + L->dst_bytes_per_hvx = rows_per_buffer * 2 * kparams->dst_row_size_aligned; + L->src1_bytes_per_hvx = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + ? rows_per_buffer * 2 * (n_srcs - 1) * kparams->src1_row_size_aligned : 0; - L->src0_spad_half_size = L->src0_bytes_per_thread / 2; - L->src1_spad_half_size = L->src1_bytes_per_thread / 2; - L->dst_spad_half_size = L->dst_bytes_per_thread / 2; + L->src0_spad_half_size = L->src0_bytes_per_hvx / 2; + L->src1_spad_half_size = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + ? rows_per_buffer * kparams->src1_row_size_aligned + : 0; + L->dst_spad_half_size = L->dst_bytes_per_hvx / 2; - const size_t src0_total = n_threads * L->src0_bytes_per_thread; + const size_t src0_total = n_hvx * L->src0_bytes_per_hvx; const size_t src1_total = (kparams->src1_size > 0) ? kparams->src1_size - : n_threads * L->src1_bytes_per_thread; - const size_t dst_total = n_threads * L->dst_bytes_per_thread; + : n_hvx * L->src1_bytes_per_hvx; + const size_t src2_total = kparams->src2_size; + const size_t dst_total = n_hvx * L->dst_bytes_per_hvx; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); VTCM_LAYOUT_ALLOC(off, off_src1, src1_total); + VTCM_LAYOUT_ALLOC(off, off_src2, src2_total); VTCM_LAYOUT_ALLOC(off, off_dst, dst_total); L->total_bytes = off; } +static inline bool htp_binary_solve_chunked( + struct htp_binary_vtcm_layout * L, + struct htp_binary_kernel_params * kparams, + uint32_t total_elems, + uint32_t elem_size, + uint32_t max_hvx, + uint32_t n_srcs, + bool is_scalar, + size_t vtcm_size +) { + const size_t n_bufs = is_scalar ? 2 : (size_t) n_srcs + 1; + const size_t scalar_bytes = is_scalar ? 128 : 0; + const size_t budget_per_buf = (max_hvx > 0 && vtcm_size > scalar_bytes) + ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_hvx * n_bufs), 128) + : 0; + const uint32_t max_chunk_bytes = (budget_per_buf > 0 && budget_per_buf < 32768) ? (uint32_t) budget_per_buf : 32768u; + const uint32_t max_chunk_elems = max_chunk_bytes / elem_size; + if (max_chunk_elems == 0) { + return false; + } + + const uint32_t min_chunk_elems = 256 < max_chunk_elems ? 256 : max_chunk_elems; + const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_hvx) - 1) / (2 * max_hvx), 32); + const uint32_t chunk_size = (target_chunk_elems < min_chunk_elems) ? min_chunk_elems + : (target_chunk_elems > max_chunk_elems) ? max_chunk_elems + : target_chunk_elems; + const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); + const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size; + const uint32_t n_hvx = (total_chunks >= 2) ? (max_hvx < total_chunks ? max_hvx : total_chunks) : 1; + + kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; + kparams->n_hvx = n_hvx; + kparams->n_srcs = (uint8_t) n_srcs; + kparams->rows_per_buffer = 1; + kparams->src1_row_size_aligned = is_scalar ? 0 : kparams->src0_row_size_aligned; + kparams->src1_size = 0; + kparams->chunk_size = chunk_size; + kparams->chunk_bytes = chunk_bytes; + kparams->is_scalar = is_scalar ? 1 : 0; + + htp_binary_vtcm_layout_build(L, kparams, vtcm_size); + if (L->total_bytes == 0 || L->total_bytes > vtcm_size) { + return false; + } + + kparams->vtcm_size = (uint32_t) L->total_bytes; + return true; +} + #endif diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index 4dc146393..a374a8fd6 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -1,30 +1,32 @@ +#include "concat-ops.h" #include "dma-queue.h" #include "hex-common.h" -#include "hex-cpy-dma.h" +#include "dma-copy.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hexagon_protos.h" #include "hexagon_types.h" #include "htp-ctx.h" +#include "htp-fence.h" #include "htp-ops.h" #include "htp-tensor.h" #include "htp-vtcm.h" #include "hvx-utils.h" #include "hvx_hexagon_protos.h" +#include "gather-rows.h" #include <string.h> struct htp_concat_context { struct htp_ops_context * octx; - uint32_t dim; - uint32_t nrows_per_thread; - uint32_t row_start; - uint32_t nrows; - uint32_t elem_start; - uint32_t nelems; - uint32_t nplanes; - struct fastdiv_values div_ne0; - struct fastdiv_values div_ne1; + uint8_t * spad0_base; + uint8_t * spad1_base; + uint32_t spad0_size_per_hvx; + uint32_t spad1_size_per_hvx; + uint32_t row_start; + uint32_t nrows; + uint32_t nrows_per_hvx; + uint32_t nplanes; struct fastdiv_values div_ne2; }; @@ -34,7 +36,7 @@ static inline dma_addr_t concat_plane_addr(const struct htp_tensor * t, uint32_t return t->data + i2 * t->nb[2] + i3 * t->nb[3]; } -static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * data) { +static void concat_2d_f32_transposed(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_concat_context * cctx = (struct htp_concat_context *) data; struct htp_ops_context * octx = cctx->octx; @@ -45,91 +47,113 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src0_ne0 = src0->ne[0]; const uint32_t src1_ne0 = src1->ne[0]; - const uint32_t row_end = cctx->row_start + cctx->nrows; - const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; - const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; - if (start_i >= end_i) return; + // dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column + const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_hvx; + const uint32_t row_end = MIN(row_begin + cctx->nrows_per_hvx, cctx->row_start + cctx->nrows); + if (row_begin >= row_end) return; - dma_queue * dma_q = octx->ctx->dma[ith]; - - uint8_t * spad0_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * spad1_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; - - const uint32_t block_i = 32; - const uint32_t spad1_stride = block_i * sizeof(float); - - const HVX_Vector offsets = hvx_vec_gather_offsets_w(spad1_stride); - const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 32); - const uint32_t src0_row_bytes = src0_ne0 * sizeof(float); - const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN); - const uint32_t src0_pre = src0_row_padded - src0_row_bytes; - const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(float); - - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct fastdiv_values * div_ne2 = &cctx->div_ne2; const uint32_t ne2 = dst->ne[2]; - uint32_t p = 0; - uint32_t i = start_i; - - const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1]; - dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(float), src1_ne0); - - const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1]; - dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i)); + // tile: block_rows dst rows x block_cols src1 columns + const uint32_t block_rows = 128; + const uint32_t block_cols = 256; + // +1 element spreads the gather lanes across VTCM, so they do not serialize on one path + const uint32_t spad1_stride = block_rows * sizeof(float) + sizeof(float); - dma_queue_pop(dma_q); // src1 - - while (p < cctx->nplanes) { - const uint32_t current_block_i = MIN(end_i - i, block_i); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); - for (uint32_t j = 0; j < src1_ne0; j += 32) { - const uint8_t * src_ptr = spad1_base + j * spad1_stride; - uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(float); - hvx_transpose_32x32_w_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 32)); + const HVX_Vector offsets = hvx_vec_gather_offsets_w(spad1_stride); + const uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, 32), block_cols); + const uint32_t src0_row_bytes = src0_ne0 * sizeof(float); + const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN); + const uint32_t src0_pre = src0_row_padded - src0_row_bytes; + const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(float); + + // two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other + uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_hvx; + uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_hvx; + const uint32_t spad0_slot = block_rows * spad0_stride; + const uint32_t spad1_slot = block_cols_padded * spad1_stride; + + // tiles walk (plane, row, col) with col fastest; src0 goes out with the col == 0 tile only + uint32_t plane = 0; + uint32_t row = row_begin; + uint32_t col = 0; + uint32_t slot = 0; + bool dst_pending = false; + + const dma_addr_t src1_addr = concat_plane_addr(src1, plane, div_ne2, ne2) + row * src1->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(row_end - row, block_rows) * sizeof(float), MIN(src1_ne0, block_cols)); + + const dma_addr_t src0_addr = concat_plane_addr(src0, plane, div_ne2, ne2) + row * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_stride, src0->nb[1], src0_row_bytes, MIN(row_end - row, block_rows)); + + while (plane < cctx->nplanes) { + const uint32_t nrows = MIN(row_end - row, block_rows); + const uint32_t ncols = MIN(src1_ne0 - col, block_cols); + + uint8_t * spad0 = spad0_base + slot * spad0_slot; + uint8_t * spad1 = spad1_base + slot * spad1_slot; + + dma_queue_pop(dma_q); // src1 + if (col == 0) { + dma_queue_pop(dma_q); // src0 } - hvx_gather_sync(spad0_base + src0_row_padded); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); - - uint32_t np = p; - uint32_t ni = i + block_i; - if (ni >= end_i) { - ni = start_i; - np++; + + uint32_t next_plane = plane; + uint32_t next_row = row; + uint32_t next_col = col + block_cols; + if (next_col >= src1_ne0) { + next_col = 0; + next_row = row + block_rows; + if (next_row >= row_end) { + next_row = row_begin; + next_plane++; + } } - const bool has_next = np < cctx->nplanes; - const uint32_t next_block_i = MIN(end_i - ni, block_i); + const bool has_next = next_plane < cctx->nplanes; + const uint32_t next_nrows = MIN(row_end - next_row, block_rows); + const uint32_t next_ncols = MIN(src1_ne0 - next_col, block_cols); - // spad1 is free after the gather sync, prefetch next src1 ahead of the dst write + // other spad1 slot is free since the previous gather sync if (has_next) { - const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1]; - dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(float), src1_ne0); + const dma_addr_t addr = concat_plane_addr(src1, next_plane, div_ne2, ne2) + next_row * src1->nb[1] + next_col * src1->nb[0]; + dma_queue_push(dma_q, dma_make_data(spad1_base + (slot ^ 1) * spad1_slot, addr), spad1_stride, src1->nb[0], next_nrows * sizeof(float), next_ncols); } - dma_queue_pop(dma_q); // src0 - - const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1]; - dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i); - - if (has_next) { - dma_queue_pop(dma_q); // next src1 + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row); + for (uint32_t c = 0; c < ncols; c += 32) { + hvx_transpose_32x32_w_gather(spad0 + src0_row_padded + c * sizeof(float), spad0_stride, spad1 + c * spad1_stride, spad1_stride, offsets, nrows, MIN(ncols - c, 32)); } - dma_queue_pop(dma_q); // dst + hvx_gather_sync(spad0 + src0_row_padded); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row); - // spad0 is free after the dst write - if (has_next) { - const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1]; - dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i); + // other spad0 slot is free once dst[b-1] is written + if (dst_pending) { + dma_queue_pop(dma_q); // dst + } + if (has_next && next_col == 0) { + const dma_addr_t addr = concat_plane_addr(src0, next_plane, div_ne2, ne2) + next_row * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base + (slot ^ 1) * spad0_slot + src0_pre, addr), spad0_stride, src0->nb[1], src0_row_bytes, next_nrows); } - p = np; - i = ni; + // the col == 0 tile also carries the src0 columns in front + const uint32_t lead = (col == 0) ? src0_row_bytes : 0; + const dma_addr_t dst_addr = concat_plane_addr(dst, plane, div_ne2, ne2) + row * dst->nb[1] + src0_row_bytes + col * sizeof(float) - lead; + dma_queue_push(dma_q, dma_make_data(dst_addr, spad0 + src0_row_padded - lead), dst->nb[1], spad0_stride, lead + ncols * sizeof(float), nrows); + dst_pending = true; + + plane = next_plane; + row = next_row; + col = next_col; + slot ^= 1; } + dma_queue_flush(dma_q); } -static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * data) { +static void concat_2d_f16_transposed(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_concat_context * cctx = (struct htp_concat_context *) data; struct htp_ops_context * octx = cctx->octx; @@ -140,301 +164,228 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src0_ne0 = src0->ne[0]; const uint32_t src1_ne0 = src1->ne[0]; - const uint32_t row_end = cctx->row_start + cctx->nrows; - const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; - const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; - if (start_i >= end_i) return; - - dma_queue * dma_q = octx->ctx->dma[ith]; - - uint8_t * spad0_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * spad1_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; + // dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column + const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_hvx; + const uint32_t row_end = MIN(row_begin + cctx->nrows_per_hvx, cctx->row_start + cctx->nrows); + if (row_begin >= row_end) return; - const uint32_t block_i = 64; - const uint32_t spad1_stride = block_i * sizeof(__fp16); - - const HVX_Vector offsets = hvx_vec_gather_offsets_h(spad1_stride); - const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 64); - const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16); - const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN); - const uint32_t src0_pre = src0_row_padded - src0_row_bytes; - const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(__fp16); - - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct fastdiv_values * div_ne2 = &cctx->div_ne2; const uint32_t ne2 = dst->ne[2]; - uint32_t p = 0; - uint32_t i = start_i; - - const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1]; - dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(__fp16), src1_ne0); - - const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1]; - dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i)); - - dma_queue_pop(dma_q); // src1 + // tile: block_rows dst rows x block_cols src1 columns + const uint32_t block_rows = 128; + const uint32_t block_cols = 512; + // +1 element spreads the gather lanes across VTCM, so they do not serialize on one path + const uint32_t spad1_stride = block_rows * sizeof(__fp16) + sizeof(__fp16); - while (p < cctx->nplanes) { - const uint32_t current_block_i = MIN(end_i - i, block_i); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); - for (uint32_t j = 0; j < src1_ne0; j += 64) { - const uint8_t * src_ptr = spad1_base + j * spad1_stride; - uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(__fp16); - hvx_transpose_64x64_h_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 64)); + const HVX_Vector offsets = hvx_vec_gather_offsets_h(spad1_stride); + const uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, 64), block_cols); + const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16); + const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN); + const uint32_t src0_pre = src0_row_padded - src0_row_bytes; + const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(__fp16); + + // two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other + uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_hvx; + uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_hvx; + const uint32_t spad0_slot = block_rows * spad0_stride; + const uint32_t spad1_slot = block_cols_padded * spad1_stride; + + // tiles walk (plane, row, col) with col fastest; src0 goes out with the col == 0 tile only + uint32_t plane = 0; + uint32_t row = row_begin; + uint32_t col = 0; + uint32_t slot = 0; + bool dst_pending = false; + + const dma_addr_t src1_addr = concat_plane_addr(src1, plane, div_ne2, ne2) + row * src1->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(row_end - row, block_rows) * sizeof(__fp16), MIN(src1_ne0, block_cols)); + + const dma_addr_t src0_addr = concat_plane_addr(src0, plane, div_ne2, ne2) + row * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_stride, src0->nb[1], src0_row_bytes, MIN(row_end - row, block_rows)); + + while (plane < cctx->nplanes) { + const uint32_t nrows = MIN(row_end - row, block_rows); + const uint32_t ncols = MIN(src1_ne0 - col, block_cols); + + uint8_t * spad0 = spad0_base + slot * spad0_slot; + uint8_t * spad1 = spad1_base + slot * spad1_slot; + + dma_queue_pop(dma_q); // src1 + if (col == 0) { + dma_queue_pop(dma_q); // src0 } - hvx_gather_sync(spad0_base + src0_row_padded); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); - - uint32_t np = p; - uint32_t ni = i + block_i; - if (ni >= end_i) { - ni = start_i; - np++; + + uint32_t next_plane = plane; + uint32_t next_row = row; + uint32_t next_col = col + block_cols; + if (next_col >= src1_ne0) { + next_col = 0; + next_row = row + block_rows; + if (next_row >= row_end) { + next_row = row_begin; + next_plane++; + } } - const bool has_next = np < cctx->nplanes; - const uint32_t next_block_i = MIN(end_i - ni, block_i); + const bool has_next = next_plane < cctx->nplanes; + const uint32_t next_nrows = MIN(row_end - next_row, block_rows); + const uint32_t next_ncols = MIN(src1_ne0 - next_col, block_cols); - // spad1 is free after the gather sync, prefetch next src1 ahead of the dst write + // other spad1 slot is free since the previous gather sync if (has_next) { - const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1]; - dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(__fp16), src1_ne0); + const dma_addr_t addr = concat_plane_addr(src1, next_plane, div_ne2, ne2) + next_row * src1->nb[1] + next_col * src1->nb[0]; + dma_queue_push(dma_q, dma_make_data(spad1_base + (slot ^ 1) * spad1_slot, addr), spad1_stride, src1->nb[0], next_nrows * sizeof(__fp16), next_ncols); } - dma_queue_pop(dma_q); // src0 - - const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1]; - dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i); - - if (has_next) { - dma_queue_pop(dma_q); // next src1 + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row); + for (uint32_t c = 0; c < ncols; c += 64) { + hvx_transpose_64x64_h_gather(spad0 + src0_row_padded + c * sizeof(__fp16), spad0_stride, spad1 + c * spad1_stride, spad1_stride, offsets, nrows, MIN(ncols - c, 64)); } - dma_queue_pop(dma_q); // dst + hvx_gather_sync(spad0 + src0_row_padded); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row); - // spad0 is free after the dst write - if (has_next) { - const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1]; - dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i); + // other spad0 slot is free once dst[b-1] is written + if (dst_pending) { + dma_queue_pop(dma_q); // dst } + if (has_next && next_col == 0) { + const dma_addr_t addr = concat_plane_addr(src0, next_plane, div_ne2, ne2) + next_row * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base + (slot ^ 1) * spad0_slot + src0_pre, addr), spad0_stride, src0->nb[1], src0_row_bytes, next_nrows); + } + + // the col == 0 tile also carries the src0 columns in front + const uint32_t lead = (col == 0) ? src0_row_bytes : 0; + const dma_addr_t dst_addr = concat_plane_addr(dst, plane, div_ne2, ne2) + row * dst->nb[1] + src0_row_bytes + col * sizeof(__fp16) - lead; + dma_queue_push(dma_q, dma_make_data(dst_addr, spad0 + src0_row_padded - lead), dst->nb[1], spad0_stride, lead + ncols * sizeof(__fp16), nrows); + dst_pending = true; - p = np; - i = ni; + plane = next_plane; + row = next_row; + col = next_col; + slot ^= 1; } + dma_queue_flush(dma_q); } -static void concat_generic(unsigned int nth, unsigned int ith, void * data) { - struct htp_concat_context * cctx = (struct htp_concat_context *) data; - struct htp_ops_context * octx = cctx->octx; - +static int concat_regular(struct htp_ops_context * octx, int dim, uint32_t type_size) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - const int dim = cctx->dim; - const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; - - const uint32_t ne[4] = {dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]}; - - // Per-device element range aligned to prevent false sharing - const uint32_t elem_start = cctx->elem_start; - const uint32_t nelems = cctx->nelems; - const uint32_t chunk_size = fastdiv(nelems + nth - 1, &octx->n_threads_div); - - const uint32_t start_idx = MIN(elem_start + ith * chunk_size, elem_start + nelems); - const uint32_t end_idx = MIN(start_idx + chunk_size, elem_start + nelems); - - // Naive scalar element-wise copy - for (uint32_t idx = start_idx; idx < end_idx; idx++) { - uint32_t idx_div_ne0 = fastdiv(idx, &cctx->div_ne0); - uint32_t i0 = idx - idx_div_ne0 * ne[0]; - - uint32_t idx_div_ne01 = fastdiv(idx_div_ne0, &cctx->div_ne1); - uint32_t i1 = idx_div_ne0 - idx_div_ne01 * ne[1]; - - uint32_t idx_div_ne012 = fastdiv(idx_div_ne01, &cctx->div_ne2); - uint32_t i2 = idx_div_ne01 - idx_div_ne012 * ne[2]; - uint32_t i3 = idx_div_ne012; + struct htp_tensor view0 = *dst; + struct htp_tensor view1 = *dst; + for (int d = 0; d < HTP_OP_MAX_DIMS; d++) { + view0.ne[d] = src0->ne[d]; + view1.ne[d] = src1->ne[d]; + } + view1.data += src0->ne[dim] * dst->nb[dim]; - uint8_t * dst_ptr = (uint8_t *)dst->data + i3 * dst->nb[3] + i2 * dst->nb[2] + i1 * dst->nb[1] + i0 * dst->nb[0]; + const uint32_t total_rows_0 = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t total_rows_1 = src1->ne[1] * src1->ne[2] * src1->ne[3]; - uint32_t idx_dim = 0; - if (dim == 0) idx_dim = i0; - else if (dim == 1) idx_dim = i1; - else if (dim == 2) idx_dim = i2; - else if (dim == 3) idx_dim = i3; + uint32_t rstart0 = 0, nrows0 = total_rows_0; + uint32_t rstart1 = 0, nrows1 = total_rows_1; - const struct htp_tensor * src = (idx_dim < src0->ne[dim]) ? src0 : src1; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range0 = htp_tensor_mdev_partition( + total_rows_0, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + rstart0 = range0.start; + nrows0 = range0.count; - uint32_t s0 = i0; - uint32_t s1 = i1; - uint32_t s2 = i2; - uint32_t s3 = i3; + const struct htp_tensor_mdev_range range1 = htp_tensor_mdev_partition( + total_rows_1, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + rstart1 = range1.start; + nrows1 = range1.count; + } - if (dim == 0 && src == src1) s0 -= src0->ne[0]; - if (dim == 1 && src == src1) s1 -= src0->ne[1]; - if (dim == 2 && src == src1) s2 -= src0->ne[2]; - if (dim == 3 && src == src1) s3 -= src0->ne[3]; + dma_queue * q = octx->ctx->dma[0]; - uint8_t * src_ptr = (uint8_t *)src->data + s3 * src->nb[3] + s2 * src->nb[2] + s1 * src->nb[1] + s0 * src->nb[0]; + dma_cpy_sametype_sameshape_range(q, &view0, src0, type_size, rstart0, nrows0); + dma_cpy_sametype_sameshape_range(q, &view1, src1, type_size, rstart1, nrows1); + dma_queue_flush(q); - if (type_size == 4) { - *(float*)dst_ptr = *(float*)src_ptr; - } else { - *(__fp16*)dst_ptr = *(__fp16*)src_ptr; - } - } + return HTP_STATUS_OK; } -static bool concat_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) { - if (dim < 0 || dim >= HTP_OP_MAX_DIMS) { - return false; +static int concat_transposed(struct htp_ops_context * octx, const struct htp_concat_kernel_params * kparams, uint32_t type_size) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; } - const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; - const struct htp_tensor * dst = octx->dst; + const struct htp_tensor * dst = octx->dst; - // Not partitioned across devices: the row/element-split paths handle that. - if (octx->ctx->mdev.count > 1 || - (dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_I32) || - src0->type != dst->type || src1->type != dst->type || src0->nb[0] != type_size || src1->nb[0] != type_size || - dst->nb[0] != type_size || (size_t) dst->ne[0] * type_size > DMA_MAX_SIZE_24B || - dst->nb[1] > DMA_MAX_STRIDE_24B || src0->nb[1] > DMA_MAX_STRIDE_24B || src1->nb[1] > DMA_MAX_STRIDE_24B) { - return false; + const uint32_t total_rows = dst->ne[1]; + uint32_t row_start = 0; + uint32_t nrows = total_rows; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; } - for (int d = 0; d < HTP_OP_MAX_DIMS; d++) { - const uint32_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d]; - if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) { - return false; - } + if (nrows == 0 || dst->ne[2] == 0 || dst->ne[3] == 0) { + return HTP_STATUS_OK; } - // The two views of dst, shaped like the sources. - struct htp_tensor view0 = *dst; - struct htp_tensor view1 = *dst; - for (int d = 0; d < HTP_OP_MAX_DIMS; d++) { - view0.ne[d] = src0->ne[d]; - view1.ne[d] = src1->ne[d]; + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; } - view1.data += (uint64_t) src0->ne[dim] * dst->nb[dim]; - dma_queue * q = octx->ctx->dma[0]; + const uint32_t n_hvx = octx->n_hvx; - cpy_dma_sametype_sameshape(q, &view0, src0, type_size); - cpy_dma_sametype_sameshape(q, &view1, src1, type_size); - dma_queue_flush(q); - return true; + struct htp_concat_transposed_vtcm_layout layout; + htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_hvx); + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + + struct htp_concat_context cctx; + cctx.octx = octx; + cctx.spad0_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad0); + cctx.spad1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad1); + cctx.spad0_size_per_hvx = layout.src0_spad_size_per_hvx; + cctx.spad1_size_per_hvx = layout.src1_spad_size_per_hvx; + cctx.row_start = row_start; + cctx.nrows = nrows; + cctx.nplanes = dst->ne[2] * dst->ne[3]; + cctx.div_ne2 = kparams->u.transposed.div_ne2; + cctx.nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); + + work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed; + work_queue_run(octx->ctx->hvx_queue, worker_func, &cctx, n_hvx); + return HTP_STATUS_OK; } int op_concat(struct htp_ops_context * octx) { - int dim = octx->op_params[0]; - if (dim < 0 || dim >= HTP_OP_MAX_DIMS) { - return HTP_STATUS_NO_SUPPORT; - } - + const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; + int status = HTP_STATUS_OK; const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; - bool is_src1_transposed = (src1->nb[0] > src1->nb[1]); - bool is_src0_transposed = (src0->nb[0] > src0->nb[1]); - if (concat_dma(octx, dim, type_size)) { - return HTP_STATUS_OK; - } - - uint32_t n_threads = octx->n_threads; - struct htp_concat_context cctx; - cctx.octx = octx; - cctx.dim = dim; - cctx.div_ne0 = init_fastdiv_values(dst->ne[0]); - cctx.div_ne1 = init_fastdiv_values(dst->ne[1]); - cctx.div_ne2 = init_fastdiv_values(dst->ne[2]); - - void (*worker_func)(unsigned int, unsigned int, void *) = concat_generic; - - const bool rows_ok = src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size; - - if (dim == 0 && is_src1_transposed && !is_src0_transposed && rows_ok) { - const uint32_t total_rows = dst->ne[1]; - const size_t dst_data_row_size = dst->ne[0] * type_size; - uint32_t row_start = 0; - uint32_t nrows = total_rows; - if (octx->ctx->mdev.count > 1) { - uint32_t rows_per_chunk = 0; - htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_data_row_size, &rows_per_chunk); - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); - row_start = range.start; - nrows = range.count; - } - - if (nrows == 0) { - return HTP_STATUS_OK; - } - - cctx.row_start = row_start; - cctx.nrows = nrows; - cctx.nplanes = dst->ne[2] * dst->ne[3]; - - uint32_t block_i = (type_size == 4) ? 32 : 64; - - cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); - - // Allocate VTCM - uint32_t spad1_stride = block_i * type_size; - - uint32_t src1_ne0_padded = hex_round_up(src1->ne[0], block_i); - // src0 row is right-aligned to VLEN so the gathered src1 part starts aligned - uint32_t spad0_row_bytes = hex_round_up(src0->ne[0] * type_size, VLEN) + src1_ne0_padded * type_size; + switch (kparams->kernel_type) { + case HTP_CONCAT_KERNEL_REGULAR: + status = concat_regular(octx, kparams->dim, type_size); + break; - octx->src0_spad.size_per_thread = block_i * spad0_row_bytes; - octx->src1_spad.size_per_thread = src1_ne0_padded * spad1_stride; + case HTP_CONCAT_KERNEL_TRANSPOSED: + status = concat_transposed(octx, kparams, type_size); + break; - octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread; - octx->src1_spad.size = n_threads * octx->src1_spad.size_per_thread; + case HTP_CONCAT_KERNEL_GATHER_ROWS: + status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_hvx); + break; - if (octx->src0_spad.size + octx->src1_spad.size > octx->ctx->vtcm_size) { - return HTP_STATUS_VTCM_TOO_SMALL; - } - - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; - octx->src0_spad.src = NULL; - octx->src1_spad.src = NULL; - - if (type_size == 4) { - worker_func = concat_2d_f32_transposed; - } else { - worker_func = concat_2d_f16_transposed; - } - } else { - if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { - return HTP_STATUS_NO_SUPPORT; - } - - const uint32_t total_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; - uint32_t elem_start = 0; - uint32_t nelems = total_elements; - if (octx->ctx->mdev.count > 1) { - const uint32_t elems_per_chunk = HEX_L2_LINE_SIZE / type_size; - const bool can_split = htp_tensor_mdev_data_aligned(dst) && htp_tensor_is_contiguous(dst, type_size) && !htp_tensor_is_permuted(dst); - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elements, can_split ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); - elem_start = range.start; - nelems = range.count; - } - - if (nelems == 0) { - return HTP_STATUS_OK; - } - - cctx.elem_start = elem_start; - cctx.nelems = nelems; + default: + status = HTP_STATUS_NO_SUPPORT; + break; } - work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads); - return HTP_STATUS_OK; + htp_ops_context_set_status(octx, status); + + return octx->status; } diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h new file mode 100644 index 000000000..580fe0e15 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/concat-ops.h @@ -0,0 +1,74 @@ +#ifndef HTP_CONCAT_OPS_H +#define HTP_CONCAT_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include "gather-rows.h" +#include <stdint.h> + +enum htp_concat_kernel_type { + HTP_CONCAT_KERNEL_UNSUPPORTED = 0, + HTP_CONCAT_KERNEL_REGULAR = 1, + HTP_CONCAT_KERNEL_TRANSPOSED = 2, + HTP_CONCAT_KERNEL_GATHER_ROWS = 3, +}; + +struct htp_concat_transposed_params { + struct fastdiv_values div_ne2; +}; + +struct htp_concat_kernel_params { + uint8_t kernel_type; + uint8_t dim; + uint8_t n_hvx; + uint8_t pad; + + uint32_t vtcm_size; + + union { + struct htp_concat_transposed_params transposed; + struct htp_gather_rows_params gather; + } u; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_concat_transposed_vtcm_layout { + uint32_t src0_spad_size_per_hvx; + uint32_t src1_spad_size_per_hvx; + size_t off_spad0; + size_t off_spad1; + uint32_t total_bytes; +}; + +static inline void htp_concat_transposed_vtcm_layout_build( + struct htp_concat_transposed_vtcm_layout * L, + uint32_t src0_ne0, + uint32_t src1_ne0, + uint32_t type_size, + uint32_t n_hvx) { + + // src1_ne0 is processed in chunks of block_cols tokens, so VTCM does not grow with the batch + uint32_t block_rows = 128; + uint32_t block_cols = (type_size == 4) ? 256 : 512; + // +1 element spreads the gather lanes across VTCM, so they do not serialize on one path + uint32_t spad1_stride = block_rows * type_size + type_size; + uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, (type_size == 4) ? 32 : 64), block_cols); + uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + block_cols_padded * type_size; + + // double-buffered: two slots per thread + L->src0_spad_size_per_hvx = 2 * block_rows * spad0_row_bytes; + L->src1_spad_size_per_hvx = 2 * block_cols_padded * spad1_stride; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_hvx * L->src0_spad_size_per_hvx); + VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_hvx * L->src1_spad_size_per_hvx); + L->total_bytes = (uint32_t) off; +} + +#endif // HTP_CONCAT_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.c b/ggml/src/ggml-hexagon/htp/conv2d-ops.c new file mode 100644 index 000000000..67e8ec67b --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.c @@ -0,0 +1,3259 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" + +#include <HAP_farf.h> +#include <stdint.h> +#include <string.h> + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "conv2d-ops.h" +#include "dma-copy.h" +#include "dma-queue.h" +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-trace.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hvx-base.h" +#include "hvx-copy.h" +#include "hvx-reduce.h" +#include "hmx-queue.h" +#include "hmx-utils.h" + +// fp16 elements between the HMX scales of two row tiles +#define CONV2D_HMX_SCALES_N (HTP_CONV2D_HMX_SCALES_SIZE / sizeof(__fp16)) + +struct htp_conv2d_hmx_task { + __fp16 * output; + const __fp16 * kernel_tiles; + const __fp16 * patch_tiles; + const __fp16 * scales; + uint32_t n_row_tiles; + uint32_t n_col_tiles; + uint32_t n_dot_tiles; +}; + +// hvx_vec_f32_to_f16_shuff() on v81 turns an exact 0 into 2^-24, which shows up as a nonzero conv output where the result is 0 +static inline HVX_Vector conv2d_f32_to_f16_shuff_z(HVX_Vector v0, HVX_Vector v1, HVX_Vector zero) { + const HVX_Vector q0 = Q6_Vqf32_vadd_VsfVsf(v0, zero); + const HVX_Vector q1 = Q6_Vqf32_vadd_VsfVsf(v1, zero); + return Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(q1, q0)); +} + +static inline HVX_Vector conv2d_f32_to_f16_shuff(HVX_Vector v0, HVX_Vector v1) { + return conv2d_f32_to_f16_shuff_z(v0, v1, Q6_V_vzero()); +} + +static inline HVX_Vector conv2d_f32_to_f16(HVX_Vector v0, HVX_Vector v1) { + return Q6_Vh_vdeal_Vh(conv2d_f32_to_f16_shuff(v0, v1)); +} + +// v if keep, else 0. Loops with transposes use this for padding rows: a hoisted zero vector would spill. +static inline HVX_Vector conv2d_keep(HVX_Vector v, bool keep) { + return Q6_V_vand_VV(v, Q6_V_vsplat_R(keep ? ~0u : 0)); +} + +// HMX output scale of one row tile: 2^-e with 2^e >= K * max|w|, so the fp16 result |y| * 2^-e <= max|x| does not overflow +static void conv2d_hmx_rt_scale(__fp16 * scales, const __fp16 * w, uint32_t n_tiles, uint32_t K) { + const HVX_Vector * v = (const HVX_Vector *) w; + const uint32_t n = n_tiles * (HMX_FP16_TILE_N_ELMS / 64); + + // |w| fp16 bits compare like unsigned ints + HVX_Vector m = Q6_V_vzero(); + for (uint32_t i = 0; i < n; i++) { + m = Q6_Vuh_vmax_VuhVuh(m, hvx_vec_abs_f16(v[i])); + } + m = hvx_vec_reduce_max_f16(m); + + // every lane holds the max, convert on HVX to avoid a scalar fp16 promotion + union { uint32_t u; float f; } wmax = { .u = Q6_R_vextract_VR(Q6_V_lo_W(hvx_vec_f16_to_f32(m)), 0) }; + union { float f; uint32_t u; } bound = { .f = (float) K * wmax.f }; + + int32_t e = (int32_t) (bound.u >> 23) - 127 + ((bound.u & 0x7fffff) != 0); + e = MAX(0, MIN(e, 24)); + + // 2^-e in fp16, subnormal below 2^-14 + const uint32_t s16 = e <= 14 ? (uint32_t) (15 - e) << 10 : 1u << (24 - e); + hmx_init_column_scales(scales, Q6_V_vsplat_R(s16)); +} + +// f32 splat of 1 / scale of row tile rt. The scale is 2^-e, so decode e from its fp16 bits (subnormal for e > 14). +static inline HVX_Vector conv2d_hmx_rt_restore(const __fp16 * scales, uint32_t rt) { + const uint32_t s16 = Q6_R_vextract_VR(*(const HVX_Vector *) (scales + (size_t) rt * CONV2D_HMX_SCALES_N), 0) & 0xffff; + const uint32_t e = s16 >= 0x400 ? 15 - (s16 >> 10) : 24 - (31 - __builtin_clz(s16)); + return Q6_V_vsplat_R((127 + e) << 23); +} + +// Output rows [*row_base, *row_base + *n_rows) of total for this device. +// The output is written by DMA only, so device ranges need no cache line alignment. +static void conv2d_mdev_rows(const struct htp_ops_context * octx, uint32_t total, uint32_t * row_base, uint32_t * n_rows) { + *row_base = 0; + *n_rows = total; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + *n_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + *row_base = range.start; + *n_rows = range.count; + } +} + +static void conv2d_hmx_worker(const htp_thread_t * th, void * data) { + struct htp_conv2d_hmx_task * task = (struct htp_conv2d_hmx_task *) data; + + const size_t dot_stride = task->n_dot_tiles * HMX_FP16_TILE_N_ELMS; + + for (uint32_t r = 0; r < task->n_row_tiles; ++r) { + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int) (task->scales + r * CONV2D_HMX_SCALES_N))); + + const __fp16 * row_base = task->kernel_tiles + r * dot_stride; + const __fp16 * col_base = task->patch_tiles; + __fp16 * out_tile = task->output + r * task->n_col_tiles * HMX_FP16_TILE_N_ELMS; + + for (uint32_t c = 0; c < task->n_col_tiles; ++c) { + const __fp16 * row_tiles = row_base; + const __fp16 * col_tiles = col_base; + asm volatile(HMX_CLRACC_F16()); + + uint32_t rem = task->n_dot_tiles; + while (rem > 0) { + const uint32_t n_tiles = MIN(rem, 32); + const uint32_t range = 2048u * n_tiles - 1; + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_tiles), "r"(col_tiles)); + row_tiles += n_tiles * HMX_FP16_TILE_N_ELMS; + col_tiles += n_tiles * HMX_FP16_TILE_N_ELMS; + rem -= n_tiles; + } + + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(out_tile), "r"(0) : "memory"); + col_base += dot_stride; + out_tile += HMX_FP16_TILE_N_ELMS; + } + } +} + +// Tiled path values whose source differs between conv_2d and conv_3d. +// conv_3d runs as conv_2d with virtual KH = KD * KHr and OH = OD * OHr, only the input stage unfolds them. +struct htp_conv_geom { + uint32_t KH, KHr, Cin, Cout; + uint32_t OH, OHr, N, ID; + int32_t s0, s1, s2, p0, p1, p2, d0, d1, d2; + size_t x_nb_d, x_nb_c, x_nb_n; // input strides for d, cin, n + size_t y_nb_c, y_nb_n; // dst strides for cout, n + size_t w_nb_co; // kernel cout stride +}; + +// conv_3d tensors are [W, H, D, C * N] with the channel as the inner index of ne[3] +static void conv_geom_init(struct htp_conv_geom * g, const struct htp_ops_context * octx) { + const struct htp_tensor * kernel = octx->src[0]; + const struct htp_tensor * input = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + const int32_t * op = octx->op_params; + + if (octx->op == HTP_OP_CONV_3D) { + const uint32_t IC = op[9]; + const uint32_t OC = op[11]; + *g = (struct htp_conv_geom) { + .KH = kernel->ne[2] * kernel->ne[1], .KHr = kernel->ne[1], .Cin = IC, .Cout = OC, + .OH = dst->ne[2] * dst->ne[1], .OHr = dst->ne[1], .N = op[10], .ID = input->ne[2], + .s0 = op[0], .s1 = op[1], .s2 = op[2], .p0 = op[3], .p1 = op[4], .p2 = op[5], .d0 = op[6], .d1 = op[7], .d2 = op[8], + .x_nb_d = input->nb[2], .x_nb_c = input->nb[3], .x_nb_n = (size_t) IC * input->nb[3], + .y_nb_c = dst->nb[3], .y_nb_n = (size_t) OC * dst->nb[3], + .w_nb_co = (size_t) IC * kernel->nb[3], + }; + } else { + *g = (struct htp_conv_geom) { + .KH = kernel->ne[1], .KHr = kernel->ne[1], .Cin = kernel->ne[2], .Cout = kernel->ne[3], + .OH = dst->ne[1], .OHr = dst->ne[1], .N = dst->ne[3], .ID = 1, + .s0 = op[0], .s1 = op[1], .s2 = 1, .p0 = op[2], .p1 = op[3], .p2 = 0, .d0 = op[4], .d1 = op[5], .d2 = 1, + .x_nb_d = 0, .x_nb_c = input->nb[2], .x_nb_n = input->nb[3], + .y_nb_c = dst->nb[2], .y_nb_n = dst->nb[3], + .w_nb_co = kernel->nb[3], + }; + } +} + +struct htp_conv2d_hmx_context { + struct htp_ops_context * octx; + const struct htp_conv2d_kernel_params * kp; + const struct htp_conv_geom * g; + struct htp_conv2d_vtcm_layout layout; + __fp16 * scales; // [rt] HMX scales + __fp16 * w_tiles; + __fp16 * act[2]; + __fp16 * out[2]; + uint8_t * thr_base; + uint32_t t_base; + uint32_t t_end; + uint32_t rt0; + uint32_t n_rt; + int32_t pack_chunk; + int32_t unpack_chunk; + uint32_t pack_tpt; + uint32_t unpack_tpt; +}; + +// col tiles [t0, t1) of chunk c for thread ith, returns first col tile of the chunk +static inline uint32_t conv2d_hmx_chunk_range(const struct htp_conv2d_hmx_context * hc, uint32_t c, uint32_t tpt, + uint32_t ith, uint32_t * t0, uint32_t * t1) { + const uint32_t cs = hc->t_base + c * hc->kp->ct_chunk; + const uint32_t ce = MIN(cs + hc->kp->ct_chunk, hc->t_end); + *t0 = MIN(cs + ith * tpt, ce); + *t1 = MIN(*t0 + tpt, ce); + return cs; +} + +static void conv2d_hmx_wpack_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; + struct htp_ops_context * octx = hc->octx; + const struct htp_conv2d_kernel_params * kp = hc->kp; + const struct htp_tensor * kernel = octx->src[0]; + + const uint32_t Cout = hc->g->Cout; + const uint32_t K = kp->tiled.k_size; + const uint32_t k_pad = kp->tiled.n_k_tiles * HMX_FP16_TILE_N_COLS; + const uint32_t WK = kp->tiled.wk_chunk; + const bool is_f16 = kernel->type == HTP_TYPE_F16; + const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); + + dma_queue * dmaq = th->dma_q; + uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size; + const size_t w_buf_size = (size_t) WK * 32 * sizeof(__fp16); + const size_t w_raw_size = (size_t) WK * 32 * sizeof(float); + const size_t w_step = w_buf_size + w_raw_size; + + __fp16 * wbuf[2] = { (__fp16 *) thr, (__fp16 *) (thr + w_step) }; + float * wraw[2] = { (float *) ((uint8_t *) wbuf[0] + w_buf_size), + (float *) ((uint8_t *) wbuf[1] + w_buf_size) }; + uint8_t * wdma[2] = { is_f16 ? (uint8_t *) wbuf[0] : (uint8_t *) wraw[0], + is_f16 ? (uint8_t *) wbuf[1] : (uint8_t *) wraw[1] }; + + const HVX_Vector zero = Q6_V_vzero(); + + struct htp_thread_trace * tr = th->trace; + + for (uint32_t rt = ith; rt < hc->n_rt; rt += nth) { + const uint32_t co0 = (hc->rt0 + rt) * 32; + const uint32_t nco = MIN(32, Cout - co0); + + uint32_t bi = 0; + for (uint32_t i = 0; i < 2 && i * WK < k_pad; i++) { + const uint32_t kc = i * WK; + const uint32_t kv = MIN(WK, K - kc); + dma_queue_push(dmaq, dma_make_data(wdma[i], kernel->data + (size_t) co0 * hc->g->w_nb_co + (size_t) kc * es), + WK * es, hc->g->w_nb_co, kv * es, nco); + } + + for (uint32_t kc = 0; kc < k_pad; kc += WK) { + const uint32_t kv = MIN(WK, K - kc); + const uint32_t hd = MIN(WK, k_pad - kc); + + dma_queue_pop(dmaq); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) kc); + + for (uint32_t r = 0; r < 32; r++) { + HVX_Vector * dv = (HVX_Vector *) (wbuf[bi] + (size_t) r * WK); + const HVX_Vector * sv = (const HVX_Vector *) (wraw[bi] + (size_t) r * WK); + for (uint32_t j = 0; j < hd; j += 64) { + HVX_Vector v = zero; + if (r < nco && j < kv) { + v = is_f16 ? dv[j / 64] : conv2d_f32_to_f16(sv[j / 32], sv[j / 32 + 1]); + if (kv - j < 64) { + v = Q6_V_vmux_QVV(Q6_Q_vsetq_R((kv - j) * sizeof(__fp16)), v, zero); + } + } + dv[j / 64] = v; + } + } + + __fp16 * tiles = hc->w_tiles + ((size_t) rt * kp->tiled.n_k_tiles + kc / 32) * HMX_FP16_TILE_N_ELMS; + hmx_interleave_cols_to_tiles(tiles, wbuf[bi], 32, hd, WK, 1, 0, 32); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) kc); + + const uint32_t next_kc = kc + 2 * WK; + if (next_kc < k_pad) { + const uint32_t next_kv = MIN(WK, K - next_kc); + dma_queue_push(dmaq, dma_make_data(wdma[bi], kernel->data + (size_t) co0 * hc->g->w_nb_co + (size_t) next_kc * es), + WK * es, hc->g->w_nb_co, next_kv * es, nco); + } + bi ^= 1; + } + } +} + +// Stage Cin*KH input rows for G col tiles: stage[(cin*KH + kh)*SR + j] = x[n][cin][ih(kh)][ow0*s0 - p0 + j], zero outside +// For conv_3d, kh = kd * KHr + kh' and oh = od * OHr + oh' are unfolded here +static void conv2d_hmx_stage(const struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp, + const struct htp_conv_geom * g, dma_queue * dmaq, float * stage, uint32_t n, uint32_t oh, + uint32_t ow0, uint32_t G) { + const struct htp_tensor * kernel = octx->src[0]; + const struct htp_tensor * input = octx->src[1]; + + const uint32_t KW = kernel->ne[0]; + const uint32_t KH = g->KH; + const uint32_t Cin = g->Cin; + const int32_t IW = input->ne[0]; + const int32_t IH = input->ne[1]; + const int32_t s0 = g->s0; + const int32_t s1 = g->s1; + const int32_t p0 = g->p0; + const int32_t p1 = g->p1; + const int32_t d0 = g->d0; + const int32_t d1 = g->d1; + const uint32_t SR = kp->tiled.stage_row; + + const int32_t Wb = 32 * G * s0 + (KW - 1) * d0; + const int32_t iw0 = (int32_t) ow0 * s0 - p0; + const int32_t lo = MAX(0, -iw0); + const int32_t hi = MIN(Wb, IW - iw0); + + const uint32_t od = fastdiv(oh, &kp->tiled.div_ohr); + oh -= od * g->OHr; + + for (uint32_t kh = 0; kh < KH; kh++) { + const int32_t kd = fastdiv(kh, &kp->tiled.div_khr); + const int32_t id = (int32_t) od * g->s2 + kd * g->d2 - g->p2; + const int32_t ih = (int32_t) oh * s1 + (int32_t) (kh - kd * g->KHr) * d1 - p1; + float * rows = stage + (size_t) kh * SR; + + if (id < 0 || id >= (int32_t) g->ID || ih < 0 || ih >= IH || hi <= lo) { + for (uint32_t c = 0; c < Cin; c++) { + hvx_splat_f32_u(rows + (size_t) c * KH * SR, 0.0f, Wb); + } + dma_queue_push(dmaq, dma_make_data(rows, 0), 0, 0, 0, 0); + continue; + } + + const dma_addr_t src = input->data + (size_t) n * g->x_nb_n + (size_t) id * g->x_nb_d + (size_t) ih * input->nb[1] + (size_t) (iw0 + lo) * sizeof(float); + dma_queue_push(dmaq, dma_make_data(rows + lo, src), (size_t) KH * SR * sizeof(float), g->x_nb_c, + (size_t) (hi - lo) * sizeof(float), Cin); + + if (lo > 0 || hi < Wb) { + for (uint32_t c = 0; c < Cin; c++) { + float * row = rows + (size_t) c * KH * SR; + if (lo > 0) { + hvx_splat_f32_u(row, 0.0f, lo); + } + if (hi < Wb) { + hvx_splat_f32_u(row + hi, 0.0f, Wb - hi); + } + } + } + } +} + +static inline HVX_Vector conv2d_hmx_load_strided_f32(const float * src, uint32_t s0, HVX_Vector offsets, HVX_Vector * scratch) { + if (s0 == 1) { + return hvx_vmemu(src); + } + if (s0 == 2) { + const HVX_Vector lo = hvx_vmemu(src); + const HVX_Vector hi = hvx_vmemu(src + 32); + return Q6_V_lo_W(Q6_W_vdeal_VVR(hi, lo, -4)); + } + Q6_vgather_ARMVw(scratch, (size_t) src, 32 * s0 * sizeof(float) - 1, offsets); + hvx_gather_sync(scratch); + return *scratch; +} + +// One weight-side tile column: vector i holds k rows 2i and 2i+1 interleaved over 32 pixels +static inline void conv2d_hmx_pack_tile(__fp16 * restrict dst, const float * restrict src, uint32_t K, uint32_t KW, + uint32_t s0, uint32_t d0, uint32_t SR, uint32_t n_vec, HVX_Vector offsets, + HVX_Vector * scratch) { + HVX_Vector * restrict out = (HVX_Vector *) dst; + const HVX_Vector zero = Q6_V_vzero(); + const uint32_t wrap = SR - KW * d0; + + const float * p = src; + uint32_t kw = 0; + uint32_t i = 0; + + for (; i < K / 2; i++) { + const HVX_Vector a = conv2d_hmx_load_strided_f32(p, s0, offsets, scratch); + p += d0; + if (++kw == KW) { + kw = 0; + p += wrap; + } + const HVX_Vector b = conv2d_hmx_load_strided_f32(p, s0, offsets, scratch); + p += d0; + if (++kw == KW) { + kw = 0; + p += wrap; + } + out[i] = conv2d_f32_to_f16_shuff(a, b); + } + if (K & 1) { + out[i++] = conv2d_f32_to_f16_shuff(conv2d_hmx_load_strided_f32(p, s0, offsets, scratch), zero); + } + for (; i < n_vec; i++) { + out[i] = zero; + } +} + +// CWHN: stage[kh*SR + j*Cin + cin] = x[n][ih(kh)][ow0*s0 - p0 + j][cin], zero outside +static void conv2d_hmx_stage_cwhn(const struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp, + dma_queue * dmaq, float * stage, uint32_t n, uint32_t oh, uint32_t ow0, uint32_t G) { + const struct htp_tensor * kernel = octx->src[0]; + const struct htp_tensor * input = octx->src[1]; + + const uint32_t KW = kernel->ne[0]; + const uint32_t KH = kernel->ne[1]; + const uint32_t Cin = kernel->ne[2]; + const int32_t IW = input->ne[0]; + const int32_t IH = input->ne[1]; + const int32_t s0 = octx->op_params[0]; + const int32_t s1 = octx->op_params[1]; + const int32_t p0 = octx->op_params[2]; + const int32_t p1 = octx->op_params[3]; + const int32_t d0 = octx->op_params[4]; + const int32_t d1 = octx->op_params[5]; + const uint32_t SR = kp->tiled.stage_row; + + const int32_t Wb = 32 * G * s0 + (KW - 1) * d0; + const int32_t iw0 = (int32_t) ow0 * s0 - p0; + const int32_t lo = MAX(0, -iw0); + const int32_t hi = MIN(Wb, IW - iw0); + + for (uint32_t kh = 0; kh < KH; kh++) { + const int32_t ih = (int32_t) oh * s1 + (int32_t) kh * d1 - p1; + float * row = stage + (size_t) kh * SR; + + if (ih < 0 || ih >= IH || hi <= lo) { + hvx_splat_f32_u(row, 0.0f, Wb * Cin); + dma_queue_push(dmaq, dma_make_data(row, 0), 0, 0, 0, 0); + continue; + } + + const dma_addr_t src = input->data + (size_t) n * input->nb[3] + (size_t) ih * input->nb[1] + (size_t) (iw0 + lo) * input->nb[0]; + const size_t size = (size_t) (hi - lo) * Cin * sizeof(float); + dma_queue_push(dmaq, dma_make_data(row + lo * Cin, src), size, size, size, 1); + + if (lo > 0) { + hvx_splat_f32_u(row, 0.0f, lo * Cin); + } + if (hi < Wb) { + hvx_splat_f32_u(row + hi * Cin, 0.0f, (Wb - hi) * Cin); + } + } +} + +static void conv2d_hmx_cwhn_f16(__fp16 * restrict dst, const float * restrict src, uint32_t KH, uint32_t SR, uint32_t n) { + for (uint32_t kh = 0; kh < KH; kh++) { + hvx_copy_f16_f32_aa((uint8_t *) (dst + (size_t) kh * SR), (const uint8_t *) (src + (size_t) kh * SR), n); + } +} + +// Stage the segment of col tiles starting at t, returns its number of col tiles +static uint32_t conv2d_hmx_issue_seg(const struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp, + const struct htp_conv_geom * g, dma_queue * dmaq, float * stage, uint32_t t, uint32_t t1) { + const uint32_t OH = g->OH; + const uint32_t n_owt = kp->tiled.n_ow_tiles; + const uint32_t row = fastdiv(t, &kp->tiled.div_ow_tiles); + const uint32_t owt = t - row * n_owt; + const uint32_t G = MIN(MIN(kp->tiled.seg_tiles, t1 - t), n_owt - owt); + const uint32_t n = fastdiv(row, &kp->tiled.div_oh); + const uint32_t oh = row - n * OH; + + if (kp->cwhn) { + conv2d_hmx_stage_cwhn(octx, kp, dmaq, stage, n, oh, owt * 32, G); + } else { + conv2d_hmx_stage(octx, kp, g, dmaq, stage, n, oh, owt * 32, G); + } + return G; +} + +static inline void conv2d_vshuff_w(HVX_Vector * a, HVX_Vector * b) { + const HVX_VectorPair p = Q6_W_vshuff_VVR(*b, *a, -4); + *a = Q6_V_lo_W(p); + *b = Q6_V_hi_W(p); +} + +static inline void conv2d_transpose_finish(HVX_Vector * restrict scratch, HVX_Vector * restrict lo, HVX_Vector * restrict hi); + +// 32x32 word transpose: out[i] word j = in[j * in_stride] word i; out 0..15 -> lo, 16..31 -> hi +static inline void conv2d_transpose_w32(const HVX_Vector * restrict in, uint32_t in_stride, HVX_Vector * restrict scratch, + HVX_Vector * restrict lo, HVX_Vector * restrict hi) { + for (uint32_t i = 0; i < 8; i++) { + HVX_Vector v0 = in[(i + 0) * in_stride]; + HVX_Vector v1 = in[(i + 8) * in_stride]; + HVX_Vector v2 = in[(i + 16) * in_stride]; + HVX_Vector v3 = in[(i + 24) * in_stride]; + conv2d_vshuff_w(&v0, &v2); + conv2d_vshuff_w(&v1, &v3); + conv2d_vshuff_w(&v0, &v1); + conv2d_vshuff_w(&v2, &v3); + scratch[i + 0] = v0; + scratch[i + 8] = v1; + scratch[i + 16] = v2; + scratch[i + 24] = v3; + } + conv2d_transpose_finish(scratch, lo, hi); +} + +static inline void conv2d_transpose_finish(HVX_Vector * restrict scratch, HVX_Vector * restrict lo, HVX_Vector * restrict hi) { + for (uint32_t g = 0; g < 4; g++) { + HVX_Vector w0 = scratch[8 * g + 0]; + HVX_Vector w1 = scratch[8 * g + 1]; + HVX_Vector w2 = scratch[8 * g + 2]; + HVX_Vector w3 = scratch[8 * g + 3]; + HVX_Vector w4 = scratch[8 * g + 4]; + HVX_Vector w5 = scratch[8 * g + 5]; + HVX_Vector w6 = scratch[8 * g + 6]; + HVX_Vector w7 = scratch[8 * g + 7]; + conv2d_vshuff_w(&w0, &w4); + conv2d_vshuff_w(&w1, &w5); + conv2d_vshuff_w(&w2, &w6); + conv2d_vshuff_w(&w3, &w7); + conv2d_vshuff_w(&w0, &w2); + conv2d_vshuff_w(&w1, &w3); + conv2d_vshuff_w(&w4, &w6); + conv2d_vshuff_w(&w5, &w7); + conv2d_vshuff_w(&w0, &w1); + conv2d_vshuff_w(&w2, &w3); + conv2d_vshuff_w(&w4, &w5); + conv2d_vshuff_w(&w6, &w7); + HVX_Vector * o = (g < 2 ? lo : hi) + 8 * (g & 1); + o[0] = w0; + o[1] = w1; + o[2] = w2; + o[3] = w3; + o[4] = w4; + o[5] = w5; + o[6] = w6; + o[7] = w7; + } +} + +// Same transpose, but pixel j is base + j * stride. Skips the vin store/reload. +static inline void conv2d_transpose_strided(const __fp16 * restrict base, uint32_t stride, HVX_Vector * restrict scratch, + HVX_Vector * restrict lo, HVX_Vector * restrict hi) { + for (uint32_t i = 0; i < 8; i++) { + HVX_Vector v0 = hvx_vmemu(base + (i + 0) * stride); + HVX_Vector v1 = hvx_vmemu(base + (i + 8) * stride); + HVX_Vector v2 = hvx_vmemu(base + (i + 16) * stride); + HVX_Vector v3 = hvx_vmemu(base + (i + 24) * stride); + conv2d_vshuff_w(&v0, &v2); + conv2d_vshuff_w(&v1, &v3); + conv2d_vshuff_w(&v0, &v1); + conv2d_vshuff_w(&v2, &v3); + scratch[i + 0] = v0; + scratch[i + 8] = v1; + scratch[i + 16] = v2; + scratch[i + 24] = v3; + } + conv2d_transpose_finish(scratch, lo, hi); +} + +// CWHN: k is n_groups groups of k_group contiguous values (per kh, or per (kh, kw) if d0 > 1), each padded to k_group_pad. +// A full 64-wide block is one vector per pixel and is transposed in place. A partial block is two masked loads, +// the second one rotated into the upper half. +static void conv2d_hmx_pack_tile_cwhn(__fp16 * restrict dst, const __fp16 * restrict src, HVX_Vector * restrict vin, + HVX_Vector * restrict scratch, const struct htp_conv2d_kernel_params * kp, + uint32_t gpr, uint32_t Cin, uint32_t s0, uint32_t d0) { + const uint32_t k_pad = kp->tiled.n_k_tiles * HMX_FP16_TILE_N_COLS; + const uint32_t KG = kp->tiled.k_group; + const uint32_t KGp = kp->tiled.k_group_pad; + const uint32_t NG = kp->tiled.n_groups; + const uint32_t SR = kp->tiled.stage_row; + + HVX_Vector * restrict out = (HVX_Vector *) dst; + + uint32_t g = 0; + uint32_t gw = 0; + uint32_t r = 0; + size_t roff = 0; + size_t goff = 0; + + for (uint32_t b = 0; b < k_pad / 64; b++) { + const __fp16 * pa[2]; + uint32_t va[2]; + for (uint32_t h = 0; h < 2; h++) { + va[h] = g < NG && r < KG ? MIN(32, KG - r) : 0; + pa[h] = g < NG ? src + goff + r : src; + r += 32; + if (r >= KGp) { + r = 0; + g++; + if (++gw == gpr) { + gw = 0; + roff += SR; + goff = roff; + } else { + goff += d0 * Cin; + } + } + } + + if (va[0] == 32 && va[1] == 32 && pa[1] == pa[0] + 32) { + conv2d_transpose_strided(pa[0], Cin * s0, scratch, out + 32 * b, out + 32 * b + 16); + continue; + } + + const HVX_VectorPred q0 = Q6_Q_vsetq_R(va[0] * sizeof(__fp16)); + const HVX_VectorPred q1 = Q6_Q_vsetq_R(va[1] * sizeof(__fp16)); + for (uint32_t j = 0; j < 32; j++) { + const HVX_Vector v0 = Q6_V_vand_QV(q0, hvx_vmemu(pa[0] + j * Cin * s0)); + const HVX_Vector v1 = Q6_V_vand_QV(q1, hvx_vmemu(pa[1] + j * Cin * s0)); + vin[j] = Q6_V_vor_VV(v0, Q6_V_vror_VR(v1, 64)); + } + + conv2d_transpose_w32(vin, 1, scratch, out + 32 * b, out + 32 * b + 16); + } +} + +struct wpack_kmajor_task { + uint32_t rt; + uint32_t g; + uint32_t kc; + uint32_t nco; + uint32_t nk; + uint32_t hk; +}; + +static inline void conv2d_hmx_wpack_kmajor_compute(const struct htp_conv2d_hmx_context * hc, + const uint8_t * wraw, + const struct wpack_kmajor_task * task, + HVX_Vector * vin, + HVX_Vector * scratch, + HVX_Vector * sink, + bool is_f16, + uint32_t es) { + const struct htp_conv2d_kernel_params * kp = hc->kp; + const uint32_t KGp = kp->tiled.k_group_pad; + const HVX_VectorPred qco = Q6_Q_vsetq2_R(task->nco * sizeof(__fp16)); + for (uint32_t kt = 0; kt < task->hk / 32; kt++) { + for (uint32_t j = 0; j < 32; j++) { + const uint32_t k = kt * 32 + j; + const HVX_Vector * sv = (const HVX_Vector *) (wraw + (size_t) k * 64 * es); + const HVX_Vector v = is_f16 ? sv[0] : conv2d_f32_to_f16(sv[0], sv[1]); + vin[j] = conv2d_keep(Q6_V_vand_QV(qco, v), k < task->nk); + } + + const uint32_t ktg = (task->g * KGp + task->kc) / 32 + kt; + HVX_Vector * lo = (HVX_Vector *) (hc->w_tiles + ((size_t) task->rt * kp->tiled.n_k_tiles + ktg) * HMX_FP16_TILE_N_ELMS); + HVX_Vector * hi = task->rt + 1 < hc->n_rt ? + (HVX_Vector *) (hc->w_tiles + ((size_t) (task->rt + 1) * kp->tiled.n_k_tiles + ktg) * HMX_FP16_TILE_N_ELMS) : sink; + conv2d_transpose_w32(vin, 1, scratch, lo, hi); + } +} + +static inline void conv2d_hmx_wpack_kmajor_dma(const struct htp_conv2d_hmx_context * hc, + dma_queue * dmaq, + uint8_t * wraw, + const struct wpack_kmajor_task * task, + uint32_t es, + size_t ks) { + const struct htp_tensor * kernel = hc->octx->src[0]; + const uint32_t KG = hc->kp->tiled.k_group; + if (task->nk > 0) { + const dma_addr_t src = kernel->data + ((size_t) task->g * KG + task->kc) * ks + (size_t) (hc->rt0 + task->rt) * 32 * es; + dma_queue_push(dmaq, dma_make_data(wraw, src), 64 * es, ks, task->nco * es, task->nk); + } else { + dma_queue_push(dmaq, dma_make_data(wraw, 0), 0, 0, 0, 0); + } +} + +struct conv2d_wpack_kmajor_iter { + uint32_t cb; + uint32_t g; + uint32_t kc; + uint32_t owner; + uint32_t n_cb; + uint32_t NG; + uint32_t KG; + uint32_t KGp; + uint32_t k_pad; + uint32_t WK; + uint32_t Cout; + uint32_t rt0; + uint32_t nth; + uint32_t ith; +}; + +static inline bool conv2d_wpack_kmajor_next_task(struct conv2d_wpack_kmajor_iter * it, struct wpack_kmajor_task * task) { + while (it->cb < it->n_cb) { + const uint32_t span = it->g + 1 == it->NG ? it->k_pad - it->g * it->KGp : it->KGp; + while (it->kc < span) { + const bool mine = it->owner == it->ith; + it->owner = it->owner + 1 == it->nth ? 0 : it->owner + 1; + const uint32_t kc = it->kc; + it->kc += it->WK; + if (mine) { + const uint32_t rt = 2 * it->cb; + const uint32_t co0 = (it->rt0 + rt) * 32; + task->rt = rt; + task->g = it->g; + task->kc = kc; + task->nco = MIN(64, it->Cout - co0); + task->nk = kc < it->KG ? MIN(it->WK, it->KG - kc) : 0; + task->hk = MIN(it->WK, span - kc); + return true; + } + } + it->kc = 0; + if (++it->g == it->NG) { + it->g = 0; + it->cb++; + } + } + return false; +} + +// Kernel stored as [k][cout] (cout contiguous): DMA k rows of 64 cout, transpose into two row tiles +static void conv2d_hmx_wpack_kmajor_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; + struct htp_ops_context * octx = hc->octx; + const struct htp_conv2d_kernel_params * kp = hc->kp; + const struct htp_tensor * kernel = octx->src[0]; + + const uint32_t Cout = kernel->ne[3]; + const uint32_t k_pad = kp->tiled.n_k_tiles * HMX_FP16_TILE_N_COLS; + const uint32_t WK = kp->tiled.wk_chunk; + const uint32_t KG = kp->tiled.k_group; + const uint32_t KGp = kp->tiled.k_group_pad; + const uint32_t NG = kp->tiled.n_groups; + const size_t ks = kp->tiled.w_kstride; + const bool is_f16 = kernel->type == HTP_TYPE_F16; + const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); + const uint32_t n_cb = (hc->n_rt + 1) / 2; + + dma_queue * dmaq = th->dma_q; + const size_t w_raw_size = (size_t) WK * 64 * sizeof(float); + uint8_t * wraws[2] = { hc->thr_base + (size_t) ith * kp->thr_size, + hc->thr_base + (size_t) ith * kp->thr_size + w_raw_size }; + HVX_Vector * vin = (HVX_Vector *) (wraws[1] + w_raw_size); + HVX_Vector * scratch = vin + 32; + HVX_Vector * sink = scratch + 32; + + struct htp_thread_trace * tr = th->trace; + + struct conv2d_wpack_kmajor_iter it = { + .cb = 0, + .g = 0, + .kc = 0, + .owner = 0, + .n_cb = n_cb, + .NG = NG, + .KG = KG, + .KGp = KGp, + .k_pad = k_pad, + .WK = WK, + .Cout = Cout, + .rt0 = hc->rt0, + .nth = nth, + .ith = ith, + }; + + struct wpack_kmajor_task tasks[2]; + uint32_t n_tasks = 0; + + // push(0), push(1) + for (uint32_t i = 0; i < 2; i++) { + if (conv2d_wpack_kmajor_next_task(&it, &tasks[i])) { + conv2d_hmx_wpack_kmajor_dma(hc, dmaq, wraws[i], &tasks[i], es, ks); + n_tasks++; + } + } + + // pop(N), compute(N), push(N + 2) + uint32_t bi = 0; + while (n_tasks > 0) { + dma_queue_pop(dmaq); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) tasks[bi].kc); + conv2d_hmx_wpack_kmajor_compute(hc, wraws[bi], &tasks[bi], vin, scratch, sink, is_f16, es); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) tasks[bi].kc); + n_tasks--; + + struct wpack_kmajor_task next_task; + if (conv2d_wpack_kmajor_next_task(&it, &next_task)) { + tasks[bi] = next_task; + conv2d_hmx_wpack_kmajor_dma(hc, dmaq, wraws[bi], &tasks[bi], es, ks); + n_tasks++; + } + bi ^= 1; + } +} + +static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, const htp_thread_t * th, uint32_t ith) { + struct htp_ops_context * octx = hc->octx; + const struct htp_conv2d_kernel_params * kp = hc->kp; + const struct htp_tensor * kernel = octx->src[0]; + + const uint32_t KW = kernel->ne[0]; + const uint32_t KH = hc->g->KH; + const uint32_t Cin = hc->g->Cin; + const uint32_t s0 = (uint32_t) hc->g->s0; + const uint32_t d0 = (uint32_t) hc->g->d0; + const uint32_t n_vec = kp->tiled.n_k_tiles * (HMX_FP16_TILE_N_ELMS / 64); + + dma_queue * dmaq = th->dma_q; + uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size; + float * stages[2] = { + (float *) thr, + (float *) (thr + kp->stage_size), + }; + __fp16 * act = hc->act[hc->pack_chunk & 1]; + + // CWHN: fp16 stage (+256 bytes of over-read slack), then 32 + 32 vectors for the transpose + const uint32_t n_stage = kp->tiled.n_stage > 1 ? 2 : 1; + __fp16 * stage16 = (__fp16 *) (thr + (size_t) n_stage * kp->stage_size + (size_t) 2 * 32 * kp->tiled.seg_tiles * 32 * sizeof(float)); + HVX_Vector * vin = (HVX_Vector *) ((uint8_t *) stage16 + kp->stage_size / 2 + 256); + HVX_Vector * scratch = vin + 32; + const uint32_t gpr = kp->tiled.n_groups == KH ? 1 : KW; + + uint32_t t0, t1; + const uint32_t cs = conv2d_hmx_chunk_range(hc, hc->pack_chunk, hc->pack_tpt, ith, &t0, &t1); + + struct htp_thread_trace * tr = th->trace; + + if (t0 >= t1) { + return; + } + + // Reuse output staging after unpack has drained its DMA. + HVX_Vector * gather_tmp = (HVX_Vector *) (thr + (size_t) n_stage * kp->stage_size); + const HVX_Vector offsets = kp->cwhn || s0 <= 2 ? Q6_V_vzero() : hvx_vec_gather_offsets_w(s0 * sizeof(float)); + + uint32_t ng[2] = { 0, 0 }; + uint32_t tq = t0; + for (uint32_t i = 0; i < n_stage && tq < t1; i++) { + ng[i] = conv2d_hmx_issue_seg(octx, kp, hc->g, dmaq, stages[i], tq, t1); + tq += ng[i]; + } + + uint32_t bi = 0; + for (uint32_t t = t0; t < t1;) { + const uint32_t G = ng[bi]; + float * stage = stages[bi]; + + for (uint32_t p = 0; p < KH; p++) { + dma_queue_pop(dmaq); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) t); + if (kp->cwhn) { + const uint32_t Wb = 32 * G * s0 + (KW - 1) * d0; + conv2d_hmx_cwhn_f16(stage16, stage, KH, kp->tiled.stage_row, Wb * Cin); + for (uint32_t g = 0; g < G; g++) { + __fp16 * tile = act + (size_t) (t - cs + g) * n_vec * 64; + conv2d_hmx_pack_tile_cwhn(tile, stage16 + (size_t) g * 32 * Cin * s0, vin, scratch, kp, gpr, Cin, s0, d0); + } + } else { + for (uint32_t g = 0; g < G; g++) { + __fp16 * tile = act + (size_t) (t - cs + g) * n_vec * 64; + const float * pixels = stage + (size_t) g * 32 * s0; + if (s0 == 1) { + conv2d_hmx_pack_tile(tile, pixels, kp->tiled.k_size, KW, 1, d0, kp->tiled.stage_row, n_vec, offsets, gather_tmp); + } else if (s0 == 2) { + conv2d_hmx_pack_tile(tile, pixels, kp->tiled.k_size, KW, 2, d0, kp->tiled.stage_row, n_vec, offsets, gather_tmp); + } else { + conv2d_hmx_pack_tile(tile, pixels, kp->tiled.k_size, KW, s0, d0, kp->tiled.stage_row, n_vec, offsets, gather_tmp); + } + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) t); + + if (tq < t1) { + ng[bi] = conv2d_hmx_issue_seg(octx, kp, hc->g, dmaq, stage, tq, t1); + tq += ng[bi]; + } + t += G; + bi = bi + 1 == n_stage ? 0 : bi + 1; + } +} + +static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, const htp_thread_t * th, uint32_t ith) { + struct htp_ops_context * octx = hc->octx; + const struct htp_conv2d_kernel_params * kp = hc->kp; + const struct htp_tensor * dst = octx->dst; + + const uint32_t OW = dst->ne[0]; + const uint32_t OH = hc->g->OH; + const uint32_t Cout = hc->g->Cout; + const uint32_t n_owt = kp->tiled.n_ow_tiles; + const uint32_t OS = kp->tiled.seg_tiles * 32; + + dma_queue * dmaq = th->dma_q; + uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size + (size_t) (kp->tiled.n_stage > 1 ? 2 : 1) * kp->stage_size; + float * ostage[2] = { + (float *) thr, + (float *) (thr + (size_t) 32 * OS * sizeof(float)), + }; + const __fp16 * out = hc->out[hc->unpack_chunk & 1]; + + uint32_t t0, t1; + const uint32_t cs = conv2d_hmx_chunk_range(hc, hc->unpack_chunk, hc->unpack_tpt, ith, &t0, &t1); + const uint32_t nct = MIN(cs + kp->ct_chunk, hc->t_end) - cs; + + struct htp_thread_trace * tr = th->trace; + + if (t0 >= t1) { + return; + } + + uint32_t b = 0; + uint32_t n_pushed = 0; + + for (uint32_t t = t0; t < t1;) { + const uint32_t row = fastdiv(t, &kp->tiled.div_ow_tiles); + const uint32_t owt = t - row * n_owt; + const uint32_t G = MIN(MIN(kp->tiled.seg_tiles, t1 - t), n_owt - owt); + const uint32_t n = fastdiv(row, &kp->tiled.div_oh); + const uint32_t oh = row - n * OH; + const uint32_t ow0 = owt * 32; + const uint32_t nv = MIN(G * 32, OW - ow0); + + for (uint32_t rt = 0; rt < hc->n_rt; rt++) { + const uint32_t co0 = (hc->rt0 + rt) * 32; + const uint32_t nco = MIN(32, Cout - co0); + const HVX_Vector restore = conv2d_hmx_rt_restore(hc->scales, rt); + + if (n_pushed >= 2) { + dma_queue_pop(dmaq); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) t); + float * ob = ostage[b]; + for (uint32_t g = 0; g < G; g++) { + const HVX_Vector * tv = (const HVX_Vector *) (out + ((size_t) rt * nct + (t - cs + g)) * HMX_FP16_TILE_N_ELMS); + for (uint32_t r = 0; r < 16; r++) { + const HVX_VectorPair pr = hvx_vec_f16_to_f32_shuff(tv[r]); + *(HVX_Vector *) (ob + (size_t) (2 * r + 0) * OS + g * 32) = hvx_vec_mul_f32_f32(Q6_V_lo_W(pr), restore); + *(HVX_Vector *) (ob + (size_t) (2 * r + 1) * OS + g * 32) = hvx_vec_mul_f32_f32(Q6_V_hi_W(pr), restore); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) t); + + const dma_addr_t ddr = dst->data + (size_t) n * hc->g->y_nb_n + (size_t) oh * dst->nb[1] + (size_t) co0 * hc->g->y_nb_c + (size_t) ow0 * sizeof(float); + dma_queue_push(dmaq, dma_make_data(ddr, ob), hc->g->y_nb_c, (size_t) OS * sizeof(float), (size_t) nv * sizeof(float), nco); + n_pushed++; + b ^= 1; + } + t += G; + } + + // Drain the pipeline + if (n_pushed >= 1) { + dma_queue_pop(dmaq); + } + if (n_pushed >= 2) { + dma_queue_pop(dmaq); + } +} + +static void conv2d_hmx_phase_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; + + if (hc->unpack_chunk >= 0) { + conv2d_hmx_unpack(hc, th, ith); + } + // weights of this rt block are packed before its first chunk + if (hc->pack_chunk == 0) { + const uint32_t n_kt = hc->kp->tiled.n_k_tiles; + for (uint32_t rt = ith; rt < hc->n_rt; rt += nth) { + conv2d_hmx_rt_scale(hc->scales + rt * CONV2D_HMX_SCALES_N, hc->w_tiles + (size_t) rt * n_kt * HMX_FP16_TILE_N_ELMS, n_kt, + hc->kp->tiled.k_size); + } + } + if (hc->pack_chunk >= 0) { + conv2d_hmx_pack(hc, th, ith); + } +} + +static inline uint32_t conv2d_hmx_tpt(const struct htp_conv2d_hmx_context * hc, int32_t c, uint32_t nth) { + if (c < 0) { + return 0; + } + const uint32_t cs = hc->t_base + (uint32_t) c * hc->kp->ct_chunk; + const uint32_t nct = MIN(hc->kp->ct_chunk, hc->t_end - cs); + return (nct + nth - 1) / nth; +} + +static void conv2d_hmx_run_phase(struct htp_conv2d_hmx_context * hc, int32_t pack, int32_t unpack, uint32_t nth) { + hc->pack_chunk = pack; + hc->unpack_chunk = unpack; + hc->pack_tpt = conv2d_hmx_tpt(hc, pack, nth); + hc->unpack_tpt = conv2d_hmx_tpt(hc, unpack, nth); + work_queue_run(hc->octx->ctx->hvx_queue, conv2d_hmx_phase_thread, hc, nth); +} + +static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp) { + struct htp_conv_geom g; + conv_geom_init(&g, octx); + + uint32_t row_base, n_rows; + conv2d_mdev_rows(octx, g.N * g.OH, &row_base, &n_rows); + if (n_rows == 0) { + return HTP_STATUS_OK; + } + + const size_t tile = 2048; + const size_t w_size = (size_t) kp->rt_block * kp->tiled.n_k_tiles * tile; + const size_t act_size = (size_t) kp->ct_chunk * kp->tiled.n_k_tiles * tile; + const size_t out_size = (size_t) kp->rt_block * kp->ct_chunk * tile; + + struct htp_conv2d_vtcm_layout layout; + htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_hvx); + + uint8_t * base = (uint8_t *) octx->ctx->vtcm_base; + + struct htp_conv2d_hmx_context hc = { + .octx = octx, + .kp = kp, + .g = &g, + .layout = layout, + .scales = VTCM_LAYOUT_PTR(__fp16, base, layout.off_scales), + .w_tiles = VTCM_LAYOUT_PTR(__fp16, base, layout.off_w), + .act = { VTCM_LAYOUT_PTR(__fp16, base, layout.off_act[0]), VTCM_LAYOUT_PTR(__fp16, base, layout.off_act[1]) }, + .out = { VTCM_LAYOUT_PTR(__fp16, base, layout.off_out[0]), VTCM_LAYOUT_PTR(__fp16, base, layout.off_out[1]) }, + .thr_base = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_thr), + .t_base = row_base * kp->tiled.n_ow_tiles, + .t_end = (row_base + n_rows) * kp->tiled.n_ow_tiles, + }; + + const uint32_t nth = kp->n_hvx; + const int32_t n_chunks = (int32_t) ((hc.t_end - hc.t_base + kp->ct_chunk - 1) / kp->ct_chunk); + + struct htp_conv2d_hmx_task task; + + for (uint32_t rt0 = 0; rt0 < kp->n_row_tiles; rt0 += kp->rt_block) { + hc.rt0 = rt0; + hc.n_rt = MIN(kp->rt_block, kp->n_row_tiles - rt0); + work_queue_run(octx->ctx->hvx_queue, kp->w_kmajor ? conv2d_hmx_wpack_kmajor_thread : conv2d_hmx_wpack_thread, &hc, nth); + + conv2d_hmx_run_phase(&hc, 0, -1, nth); + + for (int32_t c = 0; c < n_chunks; c++) { + const uint32_t cs = hc.t_base + (uint32_t) c * kp->ct_chunk; + + task.output = hc.out[c & 1]; + task.kernel_tiles = hc.w_tiles; + task.patch_tiles = hc.act[c & 1]; + task.scales = hc.scales; + task.n_row_tiles = hc.n_rt; + task.n_col_tiles = MIN(kp->ct_chunk, hc.t_end - cs); + task.n_dot_tiles = kp->tiled.n_k_tiles; + hmx_queue_push(octx->ctx->hmx_queue, hmx_queue_make_desc(conv2d_hmx_worker, &task)); + + const int32_t pack = c + 1 < n_chunks ? c + 1 : -1; + const int32_t unpack = c - 1; + if (pack >= 0 || unpack >= 0) { + conv2d_hmx_run_phase(&hc, pack, unpack, nth); + } + + hmx_queue_pop(octx->ctx->hmx_queue); + } + + conv2d_hmx_run_phase(&hc, -1, n_chunks - 1, nth); + } + + return HTP_STATUS_OK; +} + +// ---- HMX_WIN: implicit GEMM over sliding crouton windows ---- + +#define CONV2D_WIN_RT_FULL 0x7FE // spatial mask 11111, channel stop 31 +#define CONV2D_WIN_SCAT_MAX 4 // scatter croutons for Cin <= 2 * CONV2D_WIN_SCAT_MAX + +// whcn small Cin: crouton byte offset of word j of a [c | c + 1] pixel pair vector +static const int32_t conv2d_hmx_win_scat_off[32] __attribute__((aligned(VLEN))) = { + 0 * 128, 1 * 128, 2 * 128, 3 * 128, 4 * 128, 5 * 128, 6 * 128, 7 * 128, + 8 * 128, 9 * 128, 10 * 128, 11 * 128, 12 * 128, 13 * 128, 14 * 128, 15 * 128, + 0 * 128 + 4, 1 * 128 + 4, 2 * 128 + 4, 3 * 128 + 4, 4 * 128 + 4, 5 * 128 + 4, 6 * 128 + 4, 7 * 128 + 4, + 8 * 128 + 4, 9 * 128 + 4, 10 * 128 + 4, 11 * 128 + 4, 12 * 128 + 4, 13 * 128 + 4, 14 * 128 + 4, 15 * 128 + 4, +}; + +struct htp_conv2d_win_context { + struct htp_ops_context * octx; + const struct htp_conv2d_kernel_params * kp; + struct htp_conv2d_vtcm_layout layout; + __fp16 * scales; // [rt] HMX scales + __fp16 * w_tiles; // [rt][tap][cc] weight tiles + __fp16 * act[2]; // [cc][b] croutons of a chunk + __fp16 * out[2]; // [rt][tile] output tiles of a chunk + uint8_t * thr_base; + uint32_t t_base; // device col tile range + uint32_t t_end; + uint32_t vr0; // device virtual output row range + uint32_t vr1; + uint32_t rt0; + uint32_t n_rt; + int32_t pack_chunk; + int32_t unpack_chunk; + uint32_t n_scat; // small Cin: scatters per crouton into zeroed croutons, else 0 + uint8_t * live[2]; // per chunk: tiles with an output row to write +}; + +struct htp_conv2d_win_task { + __fp16 * out; + const __fp16 * w; + const __fp16 * act; + const __fp16 * scales; + const uint32_t * taps; // activation address offset of each tap (crouton + window offset fields) + uint32_t n_taps; + uint32_t n_rt; + uint32_t n_tiles; + uint32_t out_stride; // elements between row tiles in out + uint32_t n_cc; + uint32_t plane; // bytes between channel chunks in act + uint32_t rt_last; + uint32_t wr_last; + const uint8_t * live; // tiles with at least one output row to write +}; + +// Activation address offset of input pixel offset u: crouton u / 32, window offset u % 32 in Rs[10:7] and Rs[1] +static inline uint32_t conv2d_hmx_win_act_off(uint32_t u) { + return ((u >> 5) << 11) | ((u & 30) << 6) | ((u & 1) << 1); +} + +static void conv2d_hmx_win_worker(const htp_thread_t * th, void * data) { + const struct htp_conv2d_win_task * j = (const struct htp_conv2d_win_task *) data; + + const uint32_t * taps = j->taps; + const uint32_t T = j->n_taps; + const uint32_t n_cc = j->n_cc; + const uint32_t plane = j->plane; + const uint32_t rt_last = HMX_FP16_TILE_SIZE | j->rt_last; + const uint32_t wr_last = j->wr_last; + const uint32_t n_tiles = j->n_tiles; + const uint32_t n_rt = j->n_rt; + const uint32_t ostride = j->out_stride; + const uint32_t act = (uint32_t) j->act; + const size_t wstride = (size_t) T * n_cc * HMX_FP16_TILE_N_ELMS; + + for (uint32_t r = 0; r < n_rt; r++) { + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int) (j->scales + r * CONV2D_HMX_SCALES_N))); + + __fp16 * o = j->out + r * ostride; + const __fp16 * w0 = j->w + r * wstride; + + for (uint32_t i = 0; i < n_tiles; i++, o += HMX_FP16_TILE_N_ELMS) { + if (!j->live[i]) { + continue; + } + const uint32_t base = act + i * HMX_FP16_TILE_SIZE; + const __fp16 * w = w0; + + asm volatile(HMX_CLRACC_F16()); + if (n_cc == 1) { + for (uint32_t t = 0; t < T; t++) { + asm volatile("{ activation.hf = mxmem(%0, %1):single\n weight.hf = mxmem(%2, %3) }" + :: "r"(base + taps[t]), "r"(rt_last), "r"(w), "r"(wr_last)); + w += HMX_FP16_TILE_N_ELMS; + } + } else { + for (uint32_t t = 0; t < T; t++) { + uint32_t rs = base + taps[t]; + for (uint32_t cc = 1; cc < n_cc; cc++) { + asm volatile("{ activation.hf = mxmem(%0, %1):single\n weight.hf = mxmem(%2, %3) }" + :: "r"(rs), "r"(HMX_FP16_TILE_SIZE | CONV2D_WIN_RT_FULL), "r"(w), "r"(HMX_FP16_TILE_SIZE - 1)); + rs += plane; + w += HMX_FP16_TILE_N_ELMS; + } + asm volatile("{ activation.hf = mxmem(%0, %1):single\n weight.hf = mxmem(%2, %3) }" + :: "r"(rs), "r"(rt_last), "r"(w), "r"(wr_last)); + w += HMX_FP16_TILE_N_ELMS; + } + } + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(o), "r"(0) : "memory"); + } + } +} + +static inline void conv2d_vdeal_w(HVX_Vector * a, HVX_Vector * b) { + const HVX_VectorPair p = Q6_W_vdeal_VVR(*b, *a, -4); + *a = Q6_V_lo_W(p); + *b = Q6_V_hi_W(p); +} + +#define CONV2D_XPOSE_STAGE(v, m, op) \ + _Pragma("unroll") for (uint32_t i = 0; i < 16; i++) { \ + if (!(i & (m))) { \ + op(&(v)[i], &(v)[i | (m)]); \ + } \ + } + +// 16x32 word transpose: in[i] = [rows 2i, 2i + 1 of 16 words each], out[slot(v)] word c = row c word v +static inline void conv2d_xpose_w16(HVX_Vector * restrict v) { + CONV2D_XPOSE_STAGE(v, 8, conv2d_vshuff_w); + CONV2D_XPOSE_STAGE(v, 4, conv2d_vshuff_w); + CONV2D_XPOSE_STAGE(v, 2, conv2d_vshuff_w); + CONV2D_XPOSE_STAGE(v, 1, conv2d_vshuff_w); + CONV2D_XPOSE_STAGE(v, 8, conv2d_vshuff_w); +} + +// Inverse of conv2d_xpose_w16 +static inline void conv2d_unxpose_w16(HVX_Vector * restrict v) { + CONV2D_XPOSE_STAGE(v, 8, conv2d_vdeal_w); + CONV2D_XPOSE_STAGE(v, 1, conv2d_vdeal_w); + CONV2D_XPOSE_STAGE(v, 2, conv2d_vdeal_w); + CONV2D_XPOSE_STAGE(v, 4, conv2d_vdeal_w); + CONV2D_XPOSE_STAGE(v, 8, conv2d_vdeal_w); +} + +// conv2d_xpose_w16 output position of row v +static inline uint32_t conv2d_xpose_slot(uint32_t v) { + return ((v & 1) << 3) | (v >> 1); +} + +static inline uint32_t conv2d_hmx_win_chunk_tiles(const struct htp_conv2d_win_context * wc, uint32_t c, uint32_t * ts) { + *ts = wc->t_base + c * wc->kp->ct_chunk; + return MIN(wc->kp->ct_chunk, wc->t_end - *ts); +} + +struct conv2d_win_wpack_kmajor_task { + uint32_t rt; + uint32_t cc; + uint32_t tg; + uint32_t t0; + uint32_t nt; + uint32_t c0; + uint32_t nc; + uint32_t co0; + uint32_t nco; + uint32_t kh0; + uint32_t kw0; +}; + +static inline struct conv2d_win_wpack_kmajor_task conv2d_hmx_win_wpack_kmajor_make_task( + const struct htp_conv2d_win_context * wc, + uint32_t unit, + uint32_t n_tg, + uint32_t n_cc, + const struct fastdiv_values * div_tg, + const struct fastdiv_values * div_cc, + const struct fastdiv_values * div_kw, + uint32_t TG, + uint32_t T, + uint32_t KW, + uint32_t Cin, + uint32_t Cout) { + const uint32_t tg = fastmodulo(unit, n_tg, div_tg); + const uint32_t u_cc = fastdiv(unit, div_tg); + const uint32_t cc = fastmodulo(u_cc, n_cc, div_cc); + const uint32_t rt = fastdiv(u_cc, div_cc); + const uint32_t co0 = (wc->rt0 + rt) * 32; + const uint32_t nco = MIN(32, Cout - co0); + const uint32_t c0 = cc * 32; + const uint32_t nc = MIN(32, Cin - c0); + const uint32_t t0 = tg * TG; + const uint32_t nt = MIN(TG, T - t0); + const uint32_t kh0 = fastdiv(t0, div_kw); + const uint32_t kw0 = fastmodulo(t0, KW, div_kw); + + return (struct conv2d_win_wpack_kmajor_task) { + .rt = rt, + .cc = cc, + .tg = tg, + .t0 = t0, + .nt = nt, + .c0 = c0, + .nc = nc, + .co0 = co0, + .nco = nco, + .kh0 = kh0, + .kw0 = kw0, + }; +} + +static inline uint32_t conv2d_hmx_win_wpack_kmajor_dma(const struct htp_conv2d_win_context * wc, + dma_queue * dmaq, + uint8_t * raw, + const struct conv2d_win_wpack_kmajor_task * task, + uint32_t KW, + uint32_t es) { + const struct htp_tensor * kernel = wc->octx->src[0]; + uint32_t kh = task->kh0; + uint32_t kw = task->kw0; + + for (uint32_t t = 0; t < task->nt; t++) { + const dma_addr_t src = kernel->data + (size_t) kw * kernel->nb[0] + (size_t) kh * kernel->nb[1] + + (size_t) task->c0 * kernel->nb[2] + (size_t) task->co0 * es; + dma_queue_push(dmaq, dma_make_data(raw + (size_t) t * 32 * VLEN, src), VLEN, kernel->nb[2], task->nco * es, task->nc); + if (++kw == KW) { + kw = 0; + kh++; + } + } + return task->nt; +} + +static inline void conv2d_hmx_win_wpack_kmajor_compute(const struct htp_conv2d_win_context * wc, + const uint8_t * raw, + const struct conv2d_win_wpack_kmajor_task * task, + uint32_t n_cc, + uint32_t T, + uint32_t KW, + uint32_t KH, + uint32_t Cin, + uint32_t es, + bool is_f16, + HVX_Vector zero) { + const struct htp_conv2d_kernel_params * kp = wc->kp; + const HVX_VectorPred qco = Q6_Q_vsetq2_R(task->nco * es); + if (kp->window.fold) { + const HVX_Vector * rows = (const HVX_Vector *) raw; + HVX_Vector * tile = (HVX_Vector *) (wc->w_tiles + ((size_t) (task->rt * KH + task->tg) * n_cc + task->cc) * HMX_FP16_TILE_N_ELMS); + uint32_t kw = 0; + uint32_t c = 0; + for (uint32_t p = 0; p < 16; p++) { + const HVX_Vector a = kw < KW ? Q6_V_vmux_QVV(qco, rows[kw * 32 + c], zero) : zero; + if (++c == Cin) { + c = 0; + kw++; + } + const HVX_Vector b = kw < KW ? Q6_V_vmux_QVV(qco, rows[kw * 32 + c], zero) : zero; + if (++c == Cin) { + c = 0; + kw++; + } + tile[p] = is_f16 ? Q6_V_lo_W(Q6_W_vshuff_VVR(b, a, -2)) : conv2d_f32_to_f16_shuff(a, b); + } + return; + } + for (uint32_t t = 0; t < task->nt; t++) { + const HVX_Vector * rows = (const HVX_Vector *) (raw + (size_t) t * 32 * VLEN); + HVX_Vector * tile = (HVX_Vector *) (wc->w_tiles + ((size_t) (task->rt * T + task->t0 + t) * n_cc + task->cc) * HMX_FP16_TILE_N_ELMS); + for (uint32_t p = 0; p < 16; p++) { + const HVX_Vector a = 2 * p + 0 < task->nc ? Q6_V_vmux_QVV(qco, rows[2 * p + 0], zero) : zero; + const HVX_Vector b = 2 * p + 1 < task->nc ? Q6_V_vmux_QVV(qco, rows[2 * p + 1], zero) : zero; + tile[p] = is_f16 ? Q6_V_lo_W(Q6_W_vshuff_VVR(b, a, -2)) : conv2d_f32_to_f16_shuff(a, b); + } + } +} + +// Cout-contiguous kernel: DMA w[t][c][32 cout] rows, tile vector p = (row 2p, row 2p+1) interleaved per cout +static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { + struct htp_ops_context * octx = wc->octx; + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * kernel = octx->src[0]; + + const uint32_t KW = kernel->ne[0]; + const uint32_t KH = kernel->ne[1]; + const uint32_t Cin = kernel->ne[2]; + const uint32_t Cout = kernel->ne[3]; + const uint32_t T = KW * KH; + const uint32_t n_cc = kp->window.n_cc; + const uint32_t TG = kp->window.w_cs; + const bool is_f16 = kernel->type == HTP_TYPE_F16; + const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); + + dma_queue * dmaq = th->dma_q; + const size_t w_raw_size = (size_t) TG * 32 * VLEN; + uint8_t * raws[2] = { wc->thr_base + (size_t) ith * kp->thr_size, + wc->thr_base + (size_t) ith * kp->thr_size + w_raw_size }; + const HVX_Vector zero = Q6_V_vzero(); + + struct htp_thread_trace * tr = th->trace; + + const uint32_t n_tg = (T + TG - 1) / TG; + const uint32_t n_units = wc->n_rt * n_cc * n_tg; + + struct conv2d_win_wpack_kmajor_task tasks[2]; + uint32_t nd[2] = { 0, 0 }; + for (uint32_t k = 0, unit = ith; k < 2 && unit < n_units; k++, unit += nth) { + tasks[k] = conv2d_hmx_win_wpack_kmajor_make_task(wc, unit, n_tg, n_cc, &kp->window.div_tg, &kp->window.div_cc, &kp->window.div_kw, TG, T, KW, Cin, Cout); + nd[k] = conv2d_hmx_win_wpack_kmajor_dma(wc, dmaq, raws[k], &tasks[k], KW, es); + } + for (uint32_t unit = ith, k = 0; unit < n_units; unit += nth, k++) { + const uint32_t bi = k & 1; + uint8_t * raw = raws[bi]; + + for (uint32_t p = 0; p < nd[bi]; p++) { + dma_queue_pop(dmaq); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) unit); + conv2d_hmx_win_wpack_kmajor_compute(wc, raw, &tasks[bi], n_cc, T, KW, KH, Cin, es, is_f16, zero); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) unit); + + if (unit + 2 * nth < n_units) { + tasks[bi] = conv2d_hmx_win_wpack_kmajor_make_task(wc, unit + 2 * nth, n_tg, n_cc, &kp->window.div_tg, &kp->window.div_cc, &kp->window.div_kw, TG, T, KW, Cin, Cout); + nd[bi] = conv2d_hmx_win_wpack_kmajor_dma(wc, dmaq, raw, &tasks[bi], KW, es); + } + } +} + +// K-contiguous kernel: DMA 32 cout rows of the w_cs cin of a packing unit, a dummy DMA if the unit has nothing to load +static void conv2d_hmx_win_wrows_dma(const struct htp_conv2d_win_context * wc, dma_queue * dmaq, uint8_t * raw, uint32_t unit, uint32_t n_cs) { + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * kernel = wc->octx->src[0]; + + const uint32_t T = kernel->ne[0] * kernel->ne[1]; + const uint32_t Cin = kernel->ne[2]; + const uint32_t CS = kp->window.w_cs; + const uint32_t es = kernel->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float); + const uint32_t kpw = VLEN / es; + + const uint32_t cs = fastmodulo(unit, n_cs, &kp->window.div_cs); + const uint32_t u_cc = fastdiv(unit, &kp->window.div_cs); + const uint32_t cc = fastmodulo(u_cc, kp->window.n_cc, &kp->window.div_cc); + const uint32_t rt = fastdiv(u_cc, &kp->window.div_cc); + const uint32_t co0 = (wc->rt0 + rt) * 32; + const uint32_t nco = MIN(32, (uint32_t) kernel->ne[3] - co0); + const uint32_t c0 = cc * 32 + cs * CS; + const uint32_t nk = c0 < Cin ? MIN(CS, Cin - c0) * T : 0; + if (nk == 0 || (kp->window.fold && cs > 0)) { + dma_queue_push(dmaq, dma_make_data(raw, raw), 0, 0, 0, 0); + return; + } + dma_queue_push(dmaq, dma_make_data(raw, kernel->data + (size_t) co0 * kernel->nb[3] + (size_t) c0 * kernel->nb[2]), + (size_t) ((nk + kpw - 1) / kpw) * VLEN, kernel->nb[3], (size_t) nk * es, nco); +} + +// K-contiguous kernel (k = cin * T + t): transpose the 32 cout rows of a packing unit to k rows of 32 cout, then into tiles +__attribute__((noinline)) +static void conv2d_hmx_win_wrows_pack(const struct htp_conv2d_win_context * wc, const uint8_t * raw, uint8_t * xr, HVX_Vector * vin, + HVX_Vector * scratch, uint32_t unit, uint32_t n_cs) { + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * kernel = wc->octx->src[0]; + + const uint32_t KW = kernel->ne[0]; + const uint32_t KH = kernel->ne[1]; + const uint32_t Cin = kernel->ne[2]; + const uint32_t Cout = kernel->ne[3]; + const uint32_t T = KW * KH; + const uint32_t n_cc = kp->window.n_cc; + const uint32_t CS = kp->window.w_cs; + const bool is_f16 = kernel->type == HTP_TYPE_F16; + const uint32_t kpw = is_f16 ? 64 : 32; // k per vector + const uint32_t rstep = is_f16 ? 64 : 128; // bytes per transposed k row + + const uint32_t cs = fastmodulo(unit, n_cs, &kp->window.div_cs); + const uint32_t u_cc = fastdiv(unit, &kp->window.div_cs); + const uint32_t cc = fastmodulo(u_cc, n_cc, &kp->window.div_cc); + const uint32_t rt = fastdiv(u_cc, &kp->window.div_cc); + const uint32_t co0 = (wc->rt0 + rt) * 32; + const uint32_t nco = MIN(32, Cout - co0); + const uint32_t c0 = cc * 32 + cs * CS; + const uint32_t nc = c0 < Cin ? MIN(CS, Cin - c0) : 0; + const uint32_t nk = nc * T; + const uint32_t nvec = (nk + kpw - 1) / kpw; + const size_t rs = (size_t) nvec * VLEN; + + if (kp->window.fold && cs > 0) { + return; + } + + for (uint32_t b = 0; b < nvec; b++) { + // rows past nco are in the buffer but not loaded + for (uint32_t o = 0; o < 32; o++) { + vin[o] = conv2d_keep(*(const HVX_Vector *) (raw + o * rs + (size_t) b * VLEN), o < nco); + } + HVX_Vector * xo = (HVX_Vector *) (xr + (size_t) b * 32 * VLEN); + conv2d_transpose_w32(vin, 1, scratch, xo, xo + 16); + if (is_f16) { + for (uint32_t i = 0; i < 32; i++) { + xo[i] = Q6_Vh_vdeal_Vh(xo[i]); + } + } + } + + // rows past the valid ones are still inside xr, so they are loaded and masked like the rows of vin + const HVX_Vector zero = Q6_V_vzero(); + if (kp->window.fold) { + // tile kh, crouton channel kw * Cin + c is k row c * T + kh * KW + kw + for (uint32_t kh = 0; kh < KH; kh++) { + HVX_Vector * tile = (HVX_Vector *) (wc->w_tiles + ((size_t) (rt * KH + kh) * n_cc + cc) * HMX_FP16_TILE_N_ELMS); + uint32_t kw = 0; + uint32_t c = 0; + for (uint32_t p = 0; p < 16; p++) { + const HVX_Vector a = conv2d_keep(hvx_vmemu(xr + (size_t) (c * T + kh * KW + kw) * rstep), kw < KW); + if (++c == Cin) { + c = 0; + kw++; + } + const HVX_Vector b = conv2d_keep(hvx_vmemu(xr + (size_t) (c * T + kh * KW + kw) * rstep), kw < KW); + if (++c == Cin) { + c = 0; + kw++; + } + tile[p] = is_f16 ? Q6_V_lo_W(Q6_W_vshuff_VVR(b, a, -2)) : conv2d_f32_to_f16_shuff_z(a, b, zero); + } + } + return; + } + + // tile (t, cc) vector p takes cin rows 2p, 2p + 1 of this step + for (uint32_t t = 0; t < T; t++) { + HVX_Vector * tile = (HVX_Vector *) (wc->w_tiles + ((size_t) (rt * T + t) * n_cc + cc) * HMX_FP16_TILE_N_ELMS); + for (uint32_t pc = 0; pc < CS; pc += 2) { + const uint32_t c = cs * CS + pc; + const uint32_t p = c / 2; + const HVX_Vector a = conv2d_keep(hvx_vmemu(xr + (size_t) ((pc + 0) * T + t) * rstep), pc + 0 < nc); + const HVX_Vector b = conv2d_keep(hvx_vmemu(xr + (size_t) ((pc + 1) * T + t) * rstep), pc + 1 < nc); + tile[p] = is_f16 ? Q6_V_lo_W(Q6_W_vshuff_VVR(b, a, -2)) : conv2d_f32_to_f16_shuff_z(a, b, zero); + } + } +} + +static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { + struct htp_ops_context * octx = wc->octx; + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * kernel = octx->src[0]; + + const uint32_t T = kernel->ne[0] * kernel->ne[1]; + const uint32_t CS = kp->window.w_cs; + const uint32_t kpw = kernel->type == HTP_TYPE_F16 ? 64 : 32; // k per vector + + const size_t wraw = (size_t) 32 * ((CS * T + kpw - 1) / kpw) * VLEN; + dma_queue * dmaq = th->dma_q; + uint8_t * raws[2] = { wc->thr_base + (size_t) ith * kp->thr_size, wc->thr_base + (size_t) ith * kp->thr_size + wraw }; + uint8_t * xr = raws[1] + wraw; + HVX_Vector * vin = (HVX_Vector *) (xr + wraw); + HVX_Vector * scratch = vin + 32; + + const uint32_t n_cs = (32 + CS - 1) / CS; + const uint32_t n_units = wc->n_rt * kp->window.n_cc * n_cs; + + struct htp_thread_trace * tr = th->trace; + + // one DMA per unit: push(N), push(N + 1), pop(N), pack(N), push(N + 2), ... + for (uint32_t k = 0, unit = ith; k < 2 && unit < n_units; k++, unit += nth) { + conv2d_hmx_win_wrows_dma(wc, dmaq, raws[k], unit, n_cs); + } + for (uint32_t unit = ith, k = 0; unit < n_units; unit += nth, k++) { + uint8_t * raw = raws[k & 1]; + + dma_queue_pop(dmaq); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) unit); + conv2d_hmx_win_wrows_pack(wc, raw, xr, vin, scratch, unit, n_cs); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) unit); + + if (unit + 2 * nth < n_units) { + conv2d_hmx_win_wrows_dma(wc, dmaq, raw, unit + 2 * nth, n_cs); + } + } +} + +static void conv2d_hmx_win_wpack_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; + + if (wc->kp->w_kmajor) { + conv2d_hmx_win_wpack_kmajor(wc, th, nth, ith); + } else { + conv2d_hmx_win_wpack_rows(wc, th, nth, ith); + } +} + +// Fill stage with input pixels [ua, ub): whcn stage[c][u - ua], cwhn stage[u - ua][c]. Zero outside the image. +// Returns the number of DMAs pushed. +static uint32_t conv2d_hmx_win_stage(const struct htp_conv2d_win_context * wc, dma_queue * dmaq, float * stage, uint32_t ua, uint32_t ub) { + const struct htp_ops_context * octx = wc->octx; + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * input = octx->src[1]; + + const uint32_t IW = input->ne[0]; + const uint32_t IH = input->ne[1]; + const uint32_t Cin = input->ne[2]; + const uint32_t N = input->ne[3]; + const uint32_t SR = kp->window.sr; + const uint32_t IHS = kp->window.ihs; + const uint32_t p0 = octx->op_params[2]; + const uint32_t p1 = octx->op_params[3]; + const uint32_t span = kp->window.seg_b * 32; + const bool cwhn = kp->cwhn; + + // rows of one image are contiguous when there is no column padding + const bool rows_contig = p0 == 0 && input->nb[1] == (size_t) IW * input->nb[0]; + + uint32_t nd = 0; + uint32_t u = ua; + while (u < ub) { + const uint32_t vr = fastdiv(u, &kp->window.div_sr); + const uint32_t col = u - vr * SR; + const uint32_t n = fastdiv(vr, &kp->window.div_ihs); + const uint32_t r = vr - n * IHS; + const int32_t ih = (int32_t) r - (int32_t) p1; + + uint32_t end = MIN(ub, (vr + 1) * SR); + if (n >= N || ih < 0 || ih >= (int32_t) IH) { + const uint32_t len = end - u; + if (cwhn) { + hvx_splat_f32_u(stage + (size_t) (u - ua) * Cin, 0.0f, len * Cin); + } else { + for (uint32_t c = 0; c < Cin; c++) { + hvx_splat_f32_u(stage + (size_t) c * span + (u - ua), 0.0f, len); + } + } + u = end; + continue; + } + + if (rows_contig) { + end = MIN(ub, (vr + (IH - (uint32_t) ih)) * SR); + } + + // valid columns [a, b) of the first row, all columns of the following rows + const uint32_t a = MAX(col, p0); + const uint32_t len = end - u; + uint32_t b = MIN(col + len, p0 + IW); + if (rows_contig) { + b = col + len; + } + + if (a > col) { + const uint32_t z = MIN(a, col + len) - col; + if (cwhn) { + hvx_splat_f32_u(stage + (size_t) (u - ua) * Cin, 0.0f, z * Cin); + } else { + for (uint32_t c = 0; c < Cin; c++) { + hvx_splat_f32_u(stage + (size_t) c * span + (u - ua), 0.0f, z); + } + } + } + if (b < col + len) { + const uint32_t z0 = MAX(a, b); + const uint32_t z = col + len - z0; + if (cwhn) { + hvx_splat_f32_u(stage + (size_t) (u - ua + z0 - col) * Cin, 0.0f, z * Cin); + } else { + for (uint32_t c = 0; c < Cin; c++) { + hvx_splat_f32_u(stage + (size_t) c * span + (u - ua + z0 - col), 0.0f, z); + } + } + } + if (b > a) { + const dma_addr_t src = input->data + (size_t) n * input->nb[3] + (size_t) ih * input->nb[1] + (size_t) (a - p0) * input->nb[0]; + const uint32_t off = u - ua + (a - col); + if (cwhn) { + const size_t sz = (size_t) (b - a) * Cin * sizeof(float); + dma_queue_push(dmaq, dma_make_data(stage + (size_t) off * Cin, src), sz, sz, sz, 1); + } else { + dma_queue_push(dmaq, dma_make_data(stage + off, src), (size_t) span * sizeof(float), input->nb[2], + (size_t) (b - a) * sizeof(float), Cin); + } + nd++; + } + u = end; + } + return nd; +} + +// Fold staging: stage holds the raw input rows of the virtual rows of [ua, ub), after 128 bytes of slack for reads before +// the first row. whcn: raw[c][vr - vr_a][iw], cwhn: raw[vr - vr_a][iw][c]. +// Folded pixel (kw * Cin + c, u = vr * OW + ow) is input pixel (vr, ow + kw * d0 - p0), channel c. +static inline float * conv2d_hmx_win_fold_raw(float * stage) { + return stage + VLEN_FP32; +} + +static inline uint32_t conv2d_hmx_win_fold_rplane(const struct htp_conv2d_win_context * wc) { + const uint32_t IW = wc->octx->src[1]->ne[0]; + return ((wc->kp->window.seg_b * 32 + wc->kp->window.sr - 1) / wc->kp->window.sr + 1) * IW; +} + +// DMA the raw input rows of [ua, ub), returns the number of DMAs pushed +static uint32_t conv2d_hmx_win_stage_fold(const struct htp_conv2d_win_context * wc, dma_queue * dmaq, float * stage, uint32_t ua, uint32_t ub) { + const struct htp_ops_context * octx = wc->octx; + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * input = octx->src[1]; + + const uint32_t IW = input->ne[0]; + const int32_t IH = input->ne[1]; + const uint32_t Cin = input->ne[2]; + const uint32_t N = input->ne[3]; + const uint32_t IHS = kp->window.ihs; + const int32_t p1 = octx->op_params[3]; + const uint32_t RP = conv2d_hmx_win_fold_rplane(wc); + const bool cont = input->nb[1] == (size_t) IW * input->nb[0]; + float * raw = conv2d_hmx_win_fold_raw(stage); + + const uint32_t vr_a = fastdiv(ua, &kp->window.div_sr); + const uint32_t vr_b = fastdiv(ub - 1, &kp->window.div_sr) + 1; + + uint32_t nd = 0; + for (uint32_t vr = vr_a; vr < vr_b;) { + const uint32_t n = fastdiv(vr, &kp->window.div_ihs); + const int32_t ih = (int32_t) (vr - n * IHS) - p1; + if (n >= N || ih < 0 || ih >= IH) { + vr++; + continue; + } + // rows of one image are contiguous in DDR + const uint32_t nr = cont ? MIN(vr_b - vr, (uint32_t) (IH - ih)) : 1; + const dma_addr_t src = input->data + (size_t) n * input->nb[3] + (size_t) ih * input->nb[1]; + if (kp->cwhn) { + // raw[vr - vr_a][iw][c] + const size_t sz = (size_t) nr * IW * Cin * sizeof(float); + dma_queue_push(dmaq, dma_make_data(raw + (size_t) (vr - vr_a) * IW * Cin, src), sz, sz, sz, 1); + } else { + dma_queue_push(dmaq, dma_make_data(raw + (size_t) (vr - vr_a) * IW, src), (size_t) RP * sizeof(float), input->nb[2], + (size_t) nr * IW * sizeof(float), Cin); + } + nd++; + vr += nr; + } + return nd; +} + +// Build croutons [ba, ba + nb) (relative to the chunk crouton base b0) from a filled stage +static void conv2d_hmx_win_croutons(const struct htp_conv2d_win_context * wc, const float * stage, __fp16 * act, uint32_t bo, uint32_t nb) { + const struct htp_conv2d_kernel_params * kp = wc->kp; + const uint32_t Cin = wc->octx->src[1]->ne[2] * (kp->window.fold ? wc->octx->src[0]->ne[0] : 1); + const uint32_t n_cc = kp->window.n_cc; + const uint32_t span = kp->window.seg_b * 32; + const size_t pl = (size_t) kp->window.n_bc * HMX_FP16_TILE_N_ELMS; + const HVX_Vector zero = Q6_V_vzero(); + + if (wc->n_scat) { + // word j of the [c = 2k | c = 2k + 1] pixel pairs goes to (j % 16) * 128 + (2k + j / 16) * 4 + const HVX_Vector soff = *(const HVX_Vector *) conv2d_hmx_win_scat_off; + for (uint32_t i = 0; i < nb; i++) { + const size_t dst = (size_t) (act + (size_t) (bo + i) * HMX_FP16_TILE_N_ELMS); + for (uint32_t k = 0; k < wc->n_scat; k++) { + const HVX_Vector a = *(const HVX_Vector *) (stage + (size_t) (2 * k) * span + i * 32); + const HVX_Vector b = 2 * k + 1 < Cin ? *(const HVX_Vector *) (stage + (size_t) (2 * k + 1) * span + i * 32) : zero; + const HVX_VectorPred q = Q6_Q_vsetq2_R(2 * k + 1 < Cin ? VLEN : VLEN / 2); + Q6_vscatter_QRMVwV(q, dst, HMX_FP16_TILE_SIZE - 1, Q6_Vw_vadd_VwVw(soff, Q6_V_vsplat_R(8 * k)), conv2d_f32_to_f16(a, b)); + } + } + if (nb > 0) { + hvx_gather_sync(act + (size_t) (bo + nb - 1) * HMX_FP16_TILE_N_ELMS); + } + return; + } + + if (kp->cwhn) { + for (uint32_t cc = 0; cc < n_cc; cc++) { + const uint32_t nc = MIN(32, Cin - cc * 32); + const HVX_VectorPred qc = Q6_Q_vsetq2_R(nc * sizeof(float)); + for (uint32_t i = 0; i < nb; i++) { + HVX_Vector * dst = (HVX_Vector *) (act + cc * pl + (size_t) (bo + i) * HMX_FP16_TILE_N_ELMS); + const float * src = stage + (size_t) i * 32 * Cin + cc * 32; + for (uint32_t v = 0; v < 16; v++) { + const HVX_Vector a = Q6_V_vmux_QVV(qc, hvx_vmemu(src + (size_t) (2 * v + 0) * Cin), zero); + const HVX_Vector b = Q6_V_vmux_QVV(qc, hvx_vmemu(src + (size_t) (2 * v + 1) * Cin), zero); + dst[v] = conv2d_f32_to_f16_shuff(a, b); + } + } + } + return; + } + + HVX_Vector y[16]; + for (uint32_t cc = 0; cc < n_cc; cc++) { + const uint32_t nv = MIN(32, Cin - cc * 32); + const uint32_t sv = span / VLEN_FP32; + for (uint32_t i = 0; i < nb; i++) { + // channel rows are span apart, walk them with one pointer + const HVX_Vector * p = (const HVX_Vector *) (stage + (size_t) cc * 32 * span + i * 32); + HVX_Vector * dst = (HVX_Vector *) (act + cc * pl + (size_t) (bo + i) * HMX_FP16_TILE_N_ELMS); + if (nv == 32) { + #pragma unroll + for (uint32_t k = 0; k < 16; k++) { + const HVX_Vector a = p[0]; + const HVX_Vector b = p[sv]; + p += 2 * sv; + y[k] = conv2d_f32_to_f16(a, b); + } + } else { + // partial chunk: channel pairs go to the crouton memory first, the transpose runs on the reloaded rows + uint32_t k = 0; + for (; k < nv / 2; k++) { + dst[k] = conv2d_f32_to_f16(p[0], p[sv]); + p += 2 * sv; + } + if (nv & 1) { + dst[k++] = conv2d_f32_to_f16(p[0], zero); + } + for (; k < 16; k++) { + dst[k] = zero; + } + #pragma unroll + for (uint32_t k = 0; k < 16; k++) { + y[k] = dst[k]; + } + } + conv2d_xpose_w16(y); + #pragma unroll + for (uint32_t v = 0; v < 16; v++) { + dst[v] = y[conv2d_xpose_slot(v)]; + } + } + } +} + +// Fold: one crouton channel vector (32 pixels u0..u0+31), assembled from the row pieces of the block +struct conv2d_hmx_win_fold_piece { + int32_t off; // raw offset of lane 0 for kw = 0, c = 0 + int32_t l0; // lanes [l0, l1) of this row + int32_t l1; + int32_t lb; // lane of iw = 0 for kw = 0 +}; + +// Lanes of a row piece that read inside the image for column shift sh +static inline HVX_VectorPred conv2d_hmx_win_fold_q(const struct conv2d_hmx_win_fold_piece * pc, int32_t sh, int32_t IW) { + const int32_t lo = MAX(pc->l0, pc->lb - sh); + const int32_t hi = MIN(pc->l1, pc->lb - sh + IW); + if (hi <= lo) { + return Q6_Q_vsetq_R(0); + } + return Q6_Q_and_QQn(Q6_Q_vsetq2_R(hi * sizeof(float)), Q6_Q_vsetq_R(lo * sizeof(float))); +} + +static void conv2d_hmx_win_croutons_fold(const struct htp_conv2d_win_context * wc, const float * stage, __fp16 * act, uint32_t bo, + uint32_t nb, uint32_t ua) { + const struct htp_ops_context * octx = wc->octx; + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * input = octx->src[1]; + + const int32_t IW = input->ne[0]; + const int32_t IH = input->ne[1]; + const uint32_t Cin = input->ne[2]; + const uint32_t N = input->ne[3]; + const uint32_t KW = octx->src[0]->ne[0]; + const uint32_t nch = KW * Cin; + const uint32_t SR = kp->window.sr; + const uint32_t IHS = kp->window.ihs; + const int32_t p0 = octx->op_params[2]; + const int32_t p1 = octx->op_params[3]; + const int32_t d0 = octx->op_params[4]; + const uint32_t RP = conv2d_hmx_win_fold_rplane(wc); + const float * raw = conv2d_hmx_win_fold_raw((float *) stage); + + const uint32_t vr_a = fastdiv(ua, &kp->window.div_sr); + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector soff = *(const HVX_Vector *) conv2d_hmx_win_scat_off; + + for (uint32_t i = 0; i < nb; i++) { + const uint32_t u0 = ua + 32 * i; + + struct conv2d_hmx_win_fold_piece pc[4]; + uint32_t np = 0; + for (uint32_t vr = fastdiv(u0, &kp->window.div_sr); vr * SR < u0 + 32 && np < 4; vr++) { + const uint32_t n = fastdiv(vr, &kp->window.div_ihs); + const int32_t ih = (int32_t) (vr - n * IHS) - p1; + if (n >= N || ih < 0 || ih >= IH) { + continue; + } + const int32_t l0 = (int32_t) (MAX(u0, vr * SR) - u0); + const int32_t ow0 = (int32_t) (MAX(u0, vr * SR) - vr * SR); + pc[np].l0 = l0; + pc[np].l1 = (int32_t) (MIN(u0 + 32, (vr + 1) * SR) - u0); + pc[np].lb = l0 - ow0 + p0; + pc[np].off = (int32_t) ((vr - vr_a) * IW) + ow0 - l0 - p0; + np++; + } + + __fp16 * dst = act + (size_t) (bo + i) * HMX_FP16_TILE_N_ELMS; + if (np == 0) { + HVX_Vector * d = (HVX_Vector *) dst; + for (uint32_t v = 0; v < 16; v++) { + d[v] = zero; + } + continue; + } + + // channel ch = kw * Cin + c. Channel pairs go to the crouton memory first, the transpose runs on the reloaded rows. + // Lanes and offsets of the row pieces depend on kw only. + HVX_Vector * d = (HVX_Vector *) dst; + const bool scat = wc->n_scat != 0; + HVX_Vector pa = zero; + uint32_t ch = 0; + for (uint32_t kw = 0; kw < KW; kw++) { + const int32_t sh = (int32_t) kw * d0; + const HVX_VectorPred q0 = conv2d_hmx_win_fold_q(&pc[0], sh, IW); + const HVX_VectorPred q1 = np > 1 ? conv2d_hmx_win_fold_q(&pc[1], sh, IW) : q0; + const HVX_VectorPred q2 = np > 2 ? conv2d_hmx_win_fold_q(&pc[2], sh, IW) : q0; + const HVX_VectorPred q3 = np > 3 ? conv2d_hmx_win_fold_q(&pc[3], sh, IW) : q0; + const int32_t f0 = pc[0].off + sh; + const int32_t f1 = np > 1 ? pc[1].off + sh : f0; + const int32_t f2 = np > 2 ? pc[2].off + sh : f0; + const int32_t f3 = np > 3 ? pc[3].off + sh : f0; + + for (uint32_t c = 0; c < Cin; c++, ch++) { + const float * r = raw + (size_t) c * RP; + HVX_Vector x = Q6_V_vmux_QVV(q0, hvx_vmemu(r + f0), zero); + if (np > 1) { + x = Q6_V_vmux_QVV(q1, hvx_vmemu(r + f1), x); + } + if (np > 2) { + x = Q6_V_vmux_QVV(q2, hvx_vmemu(r + f2), x); + } + if (np > 3) { + x = Q6_V_vmux_QVV(q3, hvx_vmemu(r + f3), x); + } + if (!(ch & 1)) { + pa = x; + continue; + } + const HVX_Vector y = conv2d_f32_to_f16(pa, x); + if (scat) { + Q6_vscatter_QRMVwV(Q6_Q_vsetq2_R(VLEN), (size_t) dst, HMX_FP16_TILE_SIZE - 1, + Q6_Vw_vadd_VwVw(soff, Q6_V_vsplat_R(4 * (ch - 1))), y); + } else { + d[ch / 2] = y; + } + } + } + if (ch & 1) { + const HVX_Vector y = conv2d_f32_to_f16(pa, zero); + if (scat) { + Q6_vscatter_QRMVwV(Q6_Q_vsetq2_R(VLEN / 2), (size_t) dst, HMX_FP16_TILE_SIZE - 1, + Q6_Vw_vadd_VwVw(soff, Q6_V_vsplat_R(4 * (ch - 1))), y); + } else { + d[ch / 2] = y; + } + ch++; + } + if (!scat) { + for (uint32_t k = ch / 2; k < 16; k++) { + d[k] = zero; + } + } + if (scat) { + continue; + } + + HVX_Vector y[16]; + #pragma unroll + for (uint32_t k = 0; k < 16; k++) { + y[k] = d[k]; + } + conv2d_xpose_w16(y); + #pragma unroll + for (uint32_t v = 0; v < 16; v++) { + d[v] = y[conv2d_xpose_slot(v)]; + } + } + if (wc->n_scat && nb > 0) { + hvx_gather_sync(act + (size_t) (bo + nb - 1) * HMX_FP16_TILE_N_ELMS); + } +} + +// Fold, cwhn with d0 = 1: the KW * Cin folded channels of a pixel are contiguous in the raw row +static void conv2d_hmx_win_croutons_fold_cwhn(const struct htp_conv2d_win_context * wc, const float * stage, __fp16 * act, uint32_t bo, + uint32_t nb, uint32_t ua) { + const struct htp_ops_context * octx = wc->octx; + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * input = octx->src[1]; + + const int32_t IW = input->ne[0]; + const int32_t IH = input->ne[1]; + const int32_t Cin = input->ne[2]; + const uint32_t N = input->ne[3]; + const int32_t KW = octx->src[0]->ne[0]; + const uint32_t SR = kp->window.sr; + const uint32_t IHS = kp->window.ihs; + const int32_t p0 = octx->op_params[2]; + const int32_t p1 = octx->op_params[3]; + const float * raw = conv2d_hmx_win_fold_raw((float *) stage); + + const uint32_t vr_a = fastdiv(ua, &kp->window.div_sr); + const HVX_Vector zero = Q6_V_vzero(); + + uint32_t vr = vr_a; + uint32_t ow = ua - vr * SR; + bool ok = false; + for (uint32_t i = 0; i < nb; i++) { + HVX_Vector * dst = (HVX_Vector *) (act + (size_t) (bo + i) * HMX_FP16_TILE_N_ELMS); + HVX_Vector pa = zero; + for (uint32_t j = 0; j < 32; j++) { + if (ow == 0 || (i == 0 && j == 0)) { + const uint32_t n = fastdiv(vr, &kp->window.div_ihs); + const int32_t ih = (int32_t) (vr - n * IHS) - p1; + ok = n < N && ih >= 0 && ih < IH; + } + HVX_Vector x = zero; + if (ok) { + // folded channels kw * Cin + c read input column ow + kw - p0, valid for kw in [p0 - ow, p0 + IW - ow) + const int32_t lo = MAX(0, p0 - (int32_t) ow) * Cin; + const int32_t hi = MIN(KW, p0 + IW - (int32_t) ow) * Cin; + if (hi > lo) { + const float * src = raw + ((int32_t) ((vr - vr_a) * IW + ow) - p0) * Cin; + const HVX_VectorPred q = Q6_Q_and_QQn(Q6_Q_vsetq2_R(hi * sizeof(float)), Q6_Q_vsetq_R(lo * sizeof(float))); + x = Q6_V_vmux_QVV(q, hvx_vmemu(src), zero); + } + } + if (j & 1) { + dst[j / 2] = conv2d_f32_to_f16_shuff(pa, x); + } else { + pa = x; + } + if (++ow == SR) { + ow = 0; + vr++; + } + } + } +} + +static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { + struct htp_ops_context * octx = wc->octx; + const struct htp_conv2d_kernel_params * kp = wc->kp; + + uint32_t ts; + const uint32_t nt = conv2d_hmx_win_chunk_tiles(wc, wc->pack_chunk, &ts); + const uint32_t ncb = MIN(kp->window.n_bc, nt + (kp->window.max_off >> 5) + 1); + const uint32_t bpt = (ncb + nth - 1) / nth; + const uint32_t b0 = MIN(ith * bpt, ncb); + const uint32_t b1 = MIN(b0 + bpt, ncb); + + dma_queue * dmaq = th->dma_q; + uint8_t * thr = wc->thr_base + (size_t) ith * kp->thr_size; + float * stages[2] = { (float *) thr, (float *) (thr + kp->stage_size) }; + __fp16 * act = wc->act[wc->pack_chunk & 1]; + + uint32_t (*stage_fn)(const struct htp_conv2d_win_context *, dma_queue *, float *, uint32_t, uint32_t) = + kp->window.fold ? conv2d_hmx_win_stage_fold : conv2d_hmx_win_stage; + + struct htp_thread_trace * tr = th->trace; + + const uint32_t SB = kp->window.seg_b; + + if (b0 >= b1) { + return; + } + + // push(N), push(N + 1), pop(N), croutons(N), push(N + 2), ... + uint32_t nd[2] = { 0, 0 }; + for (uint32_t i = 0; i < 2 && b0 + i * SB < b1; i++) { + const uint32_t b = b0 + i * SB; + nd[i] += stage_fn(wc, dmaq, stages[i], (ts + b) * 32, (ts + MIN(b + SB, b1)) * 32); + } + + uint32_t si = 0; + for (uint32_t b = b0; b < b1; b += SB) { + const uint32_t nb = MIN(SB, b1 - b); + + for (uint32_t p = 0; p < nd[si]; p++) { + dma_queue_pop(dmaq); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) b); + if (kp->window.fold && kp->cwhn) { + conv2d_hmx_win_croutons_fold_cwhn(wc, stages[si], act, b, nb, (ts + b) * 32); + } else if (kp->window.fold) { + conv2d_hmx_win_croutons_fold(wc, stages[si], act, b, nb, (ts + b) * 32); + } else { + conv2d_hmx_win_croutons(wc, stages[si], act, b, nb); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) b); + + const uint32_t bn = b + 2 * SB; + nd[si] = bn < b1 ? stage_fn(wc, dmaq, stages[si], (ts + bn) * 32, (ts + MIN(bn + SB, b1)) * 32) : 0; + si ^= 1; + } +} + +static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { + struct htp_ops_context * octx = wc->octx; + const struct htp_conv2d_kernel_params * kp = wc->kp; + const struct htp_tensor * dst = octx->dst; + + const uint32_t OW = dst->ne[0]; + const uint32_t OH = dst->ne[1]; + const uint32_t Cout = dst->ne[2]; + const uint32_t SR = kp->window.sr; + const uint32_t IHS = kp->window.ihs; + const uint32_t ST = kp->window.seg_t; + const uint32_t OS = ST * 32; + + uint32_t ts; + const uint32_t nt = conv2d_hmx_win_chunk_tiles(wc, wc->unpack_chunk, &ts); + const uint32_t tpt = (nt + nth - 1) / nth; + const uint32_t i0 = MIN(ith * tpt, nt); + const uint32_t i1 = MIN(i0 + tpt, nt); + + dma_queue * dmaq = th->dma_q; + uint8_t * thr = wc->thr_base + (size_t) ith * kp->thr_size + 2 * (size_t) kp->stage_size; + float * obuf = (float *) thr; + __fp16 * out = wc->out[wc->unpack_chunk & 1]; + + struct htp_thread_trace * tr = th->trace; + + // nd[bi] = DMAs pushed from obuf bi, popped before it is reused HTP_CONV2D_WIN_OBUF steps later + uint32_t nd[HTP_CONV2D_WIN_OBUF] = { 0 }; + uint32_t bi = 0; + + const bool rows_contig = SR == OW && dst->nb[1] == OW * sizeof(float); + + for (uint32_t i = i0; i < i1; i += ST) { + const uint32_t ni = MIN(ST, i1 - i); + const uint32_t q_lo = (ts + i) * 32; + const uint32_t q_hi = q_lo + ni * 32; + + for (uint32_t rt = 0; rt < wc->n_rt; rt++) { + const uint32_t co0 = (wc->rt0 + rt) * 32; + const uint32_t nco = MIN(32, Cout - co0); + const uint32_t npr = (nco + 1) / 2; + const HVX_Vector restore = conv2d_hmx_rt_restore(wc->scales, rt); + + for (uint32_t p = 0; p < nd[bi]; p++) { + dma_queue_pop(dmaq); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) i); + float * ob = obuf + (size_t) bi * 32 * OS; + for (uint32_t g = 0; g < ni; g++) { + HVX_Vector * tv = (HVX_Vector *) (out + ((size_t) rt * kp->ct_chunk + i + g) * HMX_FP16_TILE_N_ELMS); + HVX_Vector y[16]; + #pragma unroll + for (uint32_t v = 0; v < 16; v++) { + y[conv2d_xpose_slot(v)] = tv[v]; + } + conv2d_unxpose_w16(y); + // rows go back in place and are converted by a second loop, both in one loop spill vector registers + #pragma unroll + for (uint32_t k = 0; k < 16; k++) { + tv[k] = y[k]; + } + HVX_Vector * o = (HVX_Vector *) (ob + g * 32); + for (uint32_t k = 0; k < npr; k++) { + const HVX_VectorPair pr = hvx_vec_f16_to_f32(tv[k]); + o[(2 * k + 0) * (OS / 32)] = hvx_vec_mul_f32_f32(Q6_V_lo_W(pr), restore); + o[(2 * k + 1) * (OS / 32)] = hvx_vec_mul_f32_f32(Q6_V_hi_W(pr), restore); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) i); + + // write the valid part of each virtual row in [q_lo, q_hi) + nd[bi] = 0; + uint32_t vr = fastdiv(q_lo, &kp->window.div_sr); + for (; vr * SR < q_hi; vr++) { + const uint32_t n = fastdiv(vr, &kp->window.div_ihs); + const uint32_t oh = vr - n * IHS; + if (vr < wc->vr0 || vr >= wc->vr1 || oh >= OH) { + continue; + } + const uint32_t base = vr * SR; + const uint32_t a = q_lo > base ? q_lo - base : 0; + uint32_t b = MIN(OW, q_hi - base); + if (b <= a) { + continue; + } + const dma_addr_t ddr = dst->data + (size_t) n * dst->nb[3] + (size_t) oh * dst->nb[1] + (size_t) co0 * dst->nb[2] + (size_t) a * sizeof(float); + + // no gap between rows: extend over the following rows of this image + if (rows_contig) { + uint32_t r = 1; + while (oh + r < OH && vr + r < wc->vr1 && (vr + r) * SR < q_hi) { + r++; + } + b = MIN(q_hi - base, (r - 1) * SR + OW); + vr += r - 1; + } + dma_queue_push(dmaq, dma_make_data(ddr, ob + (base + a - q_lo)), dst->nb[2], (size_t) OS * sizeof(float), + (size_t) (b - a) * sizeof(float), nco); + nd[bi]++; + } + bi = bi + 1 == HTP_CONV2D_WIN_OBUF ? 0 : bi + 1; + } + } + + for (uint32_t k = 0; k < HTP_CONV2D_WIN_OBUF; k++) { + for (uint32_t p = 0; p < nd[k]; p++) { + dma_queue_pop(dmaq); + } + } +} + +// per-thread callback for one pipeline phase +static void conv2d_hmx_win_phase_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; + + if (wc->unpack_chunk >= 0) { + conv2d_hmx_win_unpack(wc, th, nth, ith); + } + // weights of this rt block are packed before its first chunk + if (wc->pack_chunk == 0) { + const struct htp_tensor * kernel = wc->octx->src[0]; + const uint32_t n_wt = (wc->kp->window.fold ? kernel->ne[1] : kernel->ne[0] * kernel->ne[1]) * wc->kp->window.n_cc; + for (uint32_t rt = ith; rt < wc->n_rt; rt += nth) { + conv2d_hmx_rt_scale(wc->scales + rt * CONV2D_HMX_SCALES_N, wc->w_tiles + (size_t) rt * n_wt * HMX_FP16_TILE_N_ELMS, n_wt, + kernel->ne[0] * kernel->ne[1] * kernel->ne[2]); + } + } + if (wc->pack_chunk >= 0) { + conv2d_hmx_win_pack(wc, th, nth, ith); + } +} + +// zero out 2 act buffer regions in VTCM before packing +static void conv2d_hmx_win_zero_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; + + const size_t n = 2 * wc->layout.act_size / sizeof(float); + const size_t per = hex_round_up((n + nth - 1) / nth, VLEN_FP32); + const size_t i0 = MIN((size_t) ith * per, n); + const size_t i1 = MIN(i0 + per, n); + + struct htp_thread_trace * tr = th->trace; + const uint16_t block = (uint16_t) (i0 / VLEN_FP32); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, block); + if (i1 > i0) { + hvx_splat_f32_a((uint8_t *) wc->act[0] + i0 * sizeof(float), 0.0f, i1 - i0); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, block); +} + +static void conv2d_hmx_win_run_phase(struct htp_conv2d_win_context * wc, int32_t pack, int32_t unpack) { + wc->pack_chunk = pack; + wc->unpack_chunk = unpack; + work_queue_run(wc->octx->ctx->hvx_queue, conv2d_hmx_win_phase_thread, wc, wc->kp->n_hvx); +} + + +// live[i] = tile ts + i has a row that is written. Dead tiles lie inside the rows [OH, IHS) of an image. +static void conv2d_hmx_win_live(const struct htp_conv2d_win_context * wc, uint8_t * live, uint32_t ts, uint32_t nt) { + const struct htp_conv2d_kernel_params * kp = wc->kp; + + const uint32_t SR = kp->window.sr; + const uint32_t IHS = kp->window.ihs; + const uint32_t OH = wc->octx->dst->ne[1]; + const uint32_t q0 = ts * 32; + const uint32_t q1 = (ts + nt) * 32; + + memset(live, 1, nt); + for (uint32_t n = fastdiv(fastdiv(q0, &kp->window.div_sr), &kp->window.div_ihs); n * IHS * SR < q1; n++) { + const uint32_t g0 = MAX((n * IHS + OH) * SR, q0); + const uint32_t g1 = MIN((n + 1) * IHS * SR, q1); + const uint32_t t0 = (g0 + 31) / 32; + const uint32_t t1 = g1 / 32; + if (t1 > t0) { + memset(live + (t0 - ts), 0, t1 - t0); + } + } +} + +static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp) { + const struct htp_tensor * kernel = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t KW = kernel->ne[0]; + const uint32_t KH = kernel->ne[1]; + const uint32_t OW = dst->ne[0]; + const uint32_t OH = dst->ne[1]; + const uint32_t SR = kp->window.sr; + const uint32_t IHS = kp->window.ihs; + const uint32_t d0 = octx->op_params[4]; + const uint32_t d1 = octx->op_params[5]; + + uint32_t row_base, n_rows; + conv2d_mdev_rows(octx, dst->ne[3] * dst->ne[1], &row_base, &n_rows); + if (n_rows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t ra = row_base; + const uint32_t rb = row_base + n_rows - 1; + const uint32_t vr0 = (ra / OH) * IHS + ra % OH; + const uint32_t vr1 = (rb / OH) * IHS + rb % OH + 1; + + const size_t tile = 2048; + const uint32_t n_wt = kp->window.fold ? KH : (KW * KH); + const size_t w_size = (size_t) kp->rt_block * n_wt * kp->window.n_cc * tile; + const size_t act_size = (size_t) kp->window.n_cc * kp->window.n_bc * tile; + const size_t out_size = (size_t) kp->rt_block * kp->ct_chunk * tile; + + struct htp_conv2d_vtcm_layout layout; + htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_hvx); + + uint8_t * base = (uint8_t *) octx->ctx->vtcm_base; + + struct htp_conv2d_win_context wctx = { + .octx = octx, + .kp = kp, + .layout = layout, + .scales = VTCM_LAYOUT_PTR(__fp16, base, layout.off_scales), + .w_tiles = VTCM_LAYOUT_PTR(__fp16, base, layout.off_w), + .act = { VTCM_LAYOUT_PTR(__fp16, base, layout.off_act[0]), VTCM_LAYOUT_PTR(__fp16, base, layout.off_act[1]) }, + .out = { VTCM_LAYOUT_PTR(__fp16, base, layout.off_out[0]), VTCM_LAYOUT_PTR(__fp16, base, layout.off_out[1]) }, + .thr_base = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_thr), + .t_base = (vr0 * SR) / 32, + .t_end = ((vr1 - 1) * SR + OW + 31) / 32, + .vr0 = vr0, + .vr1 = vr1, + }; + + // small Cin: croutons are built by scattering the staged values into zeroed croutons + const uint32_t n_ch = octx->src[1]->ne[2] * (kp->window.fold ? KW : 1); + if (!kp->cwhn && n_ch <= 2 * CONV2D_WIN_SCAT_MAX) { + wctx.n_scat = (n_ch + 1) / 2; + work_queue_run(octx->ctx->hvx_queue, conv2d_hmx_win_zero_thread, &wctx, kp->n_hvx); + } + + const int32_t n_chunks = (int32_t) ((wctx.t_end - wctx.t_base + kp->ct_chunk - 1) / kp->ct_chunk); + const uint32_t rem = n_ch - (kp->window.n_cc - 1) * 32; + + // fold: kw is in the channels, taps are kh only + const uint32_t TW = kp->window.fold ? 1 : KW; + uint32_t * taps = (uint32_t *) octx->ctx->ddr_spad_base; + for (uint32_t kh = 0; kh < KH; kh++) { + for (uint32_t kw = 0; kw < TW; kw++) { + taps[kh * TW + kw] = conv2d_hmx_win_act_off(kh * d1 * SR + kw * d0); + } + } + + wctx.live[0] = (uint8_t *) octx->ctx->ddr_spad_base + hex_round_up(KH * KW * sizeof(uint32_t), 128); + wctx.live[1] = wctx.live[0] + hex_round_up(kp->ct_chunk, 128); + + struct htp_conv2d_win_task task = { + .scales = wctx.scales, + .taps = taps, + .n_taps = TW * KH, + .out_stride = kp->ct_chunk * HMX_FP16_TILE_N_ELMS, + .n_cc = kp->window.n_cc, + .plane = kp->window.n_bc * HMX_FP16_TILE_SIZE, + .rt_last = kp->window.act_rt_last, + .wr_last = (hex_round_up(rem, 8) / 2) * VLEN - 1, + }; + + for (uint32_t rt0 = 0; rt0 < kp->n_row_tiles; rt0 += kp->rt_block) { + wctx.rt0 = rt0; + wctx.n_rt = MIN(kp->rt_block, kp->n_row_tiles - rt0); + work_queue_run(octx->ctx->hvx_queue, conv2d_hmx_win_wpack_thread, &wctx, kp->n_hvx); + + conv2d_hmx_win_run_phase(&wctx, 0, -1); + + for (int32_t c = 0; c < n_chunks; c++) { + uint32_t ts; + const uint32_t nt = conv2d_hmx_win_chunk_tiles(&wctx, c, &ts); + conv2d_hmx_win_live(&wctx, wctx.live[c & 1], ts, nt); + task.live = wctx.live[c & 1]; + task.out = wctx.out[c & 1]; + task.w = wctx.w_tiles; + task.act = wctx.act[c & 1]; + task.n_rt = wctx.n_rt; + task.n_tiles = nt; + hmx_queue_push(octx->ctx->hmx_queue, hmx_queue_make_desc(conv2d_hmx_win_worker, &task)); + + const int32_t pack = c + 1 < n_chunks ? c + 1 : -1; + const int32_t unpack = c - 1; + if (pack >= 0 || unpack >= 0) { + conv2d_hmx_win_run_phase(&wctx, pack, unpack); + } + + hmx_queue_pop(octx->ctx->hmx_queue); + } + + conv2d_hmx_win_run_phase(&wctx, -1, n_chunks - 1); + } + + return HTP_STATUS_OK; +} + +int op_conv_2d(struct htp_ops_context * octx) { + const struct htp_conv2d_kernel_params * kparams = + (const struct htp_conv2d_kernel_params *) octx->kernel_params; + + if (kparams->total_rows == 0) { + return HTP_STATUS_OK; + } + + if (kparams->kernel_type == HTP_CONV2D_KERNEL_HMX_WINDOW) { + return conv2d_hmx_win(octx, kparams); + } + if (kparams->kernel_type == HTP_CONV2D_KERNEL_HMX_TILED) { + return conv2d_hmx_tiled(octx, kparams); + } + + return HTP_STATUS_NO_SUPPORT; +} + +struct conv2d_dw_task_slice { + dma_addr_t src_addr; + dma_addr_t dst_addr; + uint32_t src_bytes; + uint32_t dst_bytes; + uint32_t ih_count; + uint32_t top_pad; + uint32_t bot_pad; + uint32_t oh_start; + uint32_t oh_len; + int32_t ih_first; + uint32_t c; +}; + +static inline void conv2d_dw_get_slice( + struct conv2d_dw_task_slice * slice, + uint32_t task, + const struct htp_tensor * src1, + const struct htp_tensor * dst, + const struct htp_conv2d_dw_kernel_params * kp) { + + uint32_t chunk_idx; + uint32_t n; + uint32_t c = 0; + + if (kp->cwhn) { + n = fastdiv(task, &kp->div_n_chunks); + chunk_idx = task - n * kp->n_chunks; + } else { + const uint32_t plane = fastdiv(task, &kp->div_n_chunks); + chunk_idx = task - plane * kp->n_chunks; + n = fastdiv(plane, &kp->div_c); + c = plane - n * kp->c; + } + + const uint32_t oh_start = chunk_idx * kp->oh_chunk; + const uint32_t oh_len = MIN(kp->oh_chunk, kp->oh - oh_start); + + const int32_t ih_min = (int32_t)(oh_start * kp->s1) - kp->p1; + const int32_t ih_max = (int32_t)((oh_start + oh_len - 1) * kp->s1 + (kp->kh - 1) * kp->d1) - kp->p1; + const uint32_t ih_span = (uint32_t)(ih_max - ih_min + 1); + + const int32_t ih_first = MAX(0, ih_min); + const int32_t ih_last = MIN((int32_t)kp->h - 1, ih_max); + const uint32_t ih_count = (ih_first <= ih_last) ? (uint32_t)(ih_last - ih_first + 1) : 0; + const uint32_t top_pad = (ih_count > 0) ? (uint32_t)(ih_first - ih_min) : ih_span; + const uint32_t bot_pad = ih_span - top_pad - ih_count; + + slice->src_bytes = ih_count * kp->row_src_bytes; + slice->dst_bytes = oh_len * kp->row_dst_bytes; + slice->ih_count = ih_count; + slice->top_pad = top_pad; + slice->bot_pad = bot_pad; + slice->oh_start = oh_start; + slice->oh_len = oh_len; + slice->ih_first = ih_first; + slice->c = c; + + if (kp->cwhn) { + slice->src_addr = src1->data + (size_t) n * src1->nb[3] + (size_t) ih_first * src1->nb[1]; + slice->dst_addr = dst->data + (size_t) n * dst->nb[3] + (size_t) oh_start * dst->nb[1]; + } else { + slice->src_addr = src1->data + (size_t) n * src1->nb[3] + (size_t) c * src1->nb[2] + (size_t) ih_first * src1->nb[1]; + slice->dst_addr = dst->data + (size_t) n * dst->nb[3] + (size_t) c * dst->nb[2] + (size_t) oh_start * dst->nb[1]; + } +} + +static inline HVX_Vector conv2d_dw_splat_w_f32(const void * ptr) { + return hvx_vec_splat_f32(*(const float *) ptr); +} + +static inline HVX_Vector conv2d_dw_splat_w_f16(const void * ptr) { + const HVX_Vector vh = Q6_Vh_vsplat_R(*(const uint16_t *) ptr); + const HVX_Vector v1 = Q6_Vh_vsplat_R(0x3c00); + return Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(vh, v1))); +} + +static inline HVX_Vector conv2d_dw_load_w_f32(const void * ptr) { + return hvx_vmemu(ptr); +} + +static inline HVX_Vector conv2d_dw_load_w_f16(const void * ptr) { + return Q6_V_lo_W(hvx_vec_f16_to_f32(hvx_vmemu(ptr))); +} + +static inline HVX_Vector conv2d_dw_qf32_mpy_add3( + HVX_Vector acc, + HVX_Vector x0, HVX_Vector w0, + HVX_Vector x1, HVX_Vector w1, + HVX_Vector x2, HVX_Vector w2) { + const HVX_Vector p01 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(x0, w0), Q6_Vqf32_vmpy_VsfVsf(x1, w1)); + const HVX_Vector p2 = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x2, w2)); + return Q6_Vqf32_vadd_Vqf32Vqf32(p01, p2); +} + +static inline void conv2d_dw_pad_vtcm_k3x3( + float * in_vtcm, + const struct conv2d_dw_task_slice * slice, + uint32_t w_in, + uint32_t extra_bot_rows) { + if (slice->top_pad > 0) { + hvx_splat_f32_a(in_vtcm, 0.0f, slice->top_pad * w_in); + } + const uint32_t tail_zero_rows = slice->bot_pad + extra_bot_rows; + if (tail_zero_rows > 0) { + hvx_splat_f32_a(in_vtcm + (size_t) (slice->top_pad + slice->ih_count) * w_in, 0.0f, tail_zero_rows * w_in); + } +} + +#define DEFINE_CONV2D_DW_WHCN_S1_K3X3(SUFFIX, SPLAT_W_FN) \ +__attribute__((noinline)) \ +static void conv2d_dw_whcn_s1_k3x3_##SUFFIX( \ + const float * restrict in_vtcm, \ + float * restrict out_vtcm, \ + const void * w_vtcm, \ + const struct conv2d_dw_task_slice * slice, \ + const struct htp_ops_context * octx, \ + const struct htp_conv2d_dw_kernel_params * kp) { \ + \ + const struct htp_tensor * src0 = octx->src[0]; \ + const size_t w_nb0 = src0->nb[0]; \ + const size_t w_nb1 = src0->nb[1]; \ + const char * restrict w_chan = (const char *) w_vtcm + (size_t) slice->c * src0->nb[3]; \ + const uint32_t oh_len = slice->oh_len; \ + const uint32_t w_in = kp->w; \ + \ + conv2d_dw_pad_vtcm_k3x3((float *) in_vtcm, slice, w_in, 0); \ + \ + const HVX_Vector w00 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w01 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w02 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector w10 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w11 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w12 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector w20 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w21 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w22 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector zero = Q6_V_vzero(); \ + \ + uint32_t ohi = 0; \ + for (; ohi + 4 <= oh_len; ohi += 4) { \ + const float * restrict r0 = in_vtcm + (size_t) ohi * w_in; \ + const float * restrict r1 = r0 + w_in; \ + const float * restrict r2 = r1 + w_in; \ + const float * restrict r3 = r2 + w_in; \ + const float * restrict r4 = r3 + w_in; \ + const float * restrict r5 = r4 + w_in; \ + \ + float * restrict out0 = out_vtcm + (size_t) (ohi + 0) * w_in; \ + float * restrict out1 = out0 + w_in; \ + float * restrict out2 = out1 + w_in; \ + float * restrict out3 = out2 + w_in; \ + \ + HVX_Vector p0 = zero, c0 = hvx_vmem(r0); \ + HVX_Vector p1 = zero, c1 = hvx_vmem(r1); \ + HVX_Vector p2 = zero, c2 = hvx_vmem(r2); \ + HVX_Vector p3 = zero, c3 = hvx_vmem(r3); \ + HVX_Vector p4 = zero, c4 = hvx_vmem(r4); \ + HVX_Vector p5 = zero, c5 = hvx_vmem(r5); \ + \ + for (uint32_t ox = 0; ox < w_in; ox += 32) { \ + const bool has_next = (ox + 32 < w_in); \ + \ + const HVX_Vector n0 = has_next ? hvx_vmem(r0 + ox + 32) : zero; \ + const HVX_Vector l0 = Q6_V_valign_VVR(c0, p0, 124); \ + const HVX_Vector rg0 = Q6_V_valign_VVR(n0, c0, 4); \ + HVX_Vector acc0 = conv2d_dw_qf32_mpy_add3(zero, l0, w00, c0, w01, rg0, w02); \ + p0 = c0; c0 = n0; \ + \ + const HVX_Vector n1 = has_next ? hvx_vmem(r1 + ox + 32) : zero; \ + const HVX_Vector l1 = Q6_V_valign_VVR(c1, p1, 124); \ + const HVX_Vector rg1 = Q6_V_valign_VVR(n1, c1, 4); \ + acc0 = conv2d_dw_qf32_mpy_add3(acc0, l1, w10, c1, w11, rg1, w12); \ + HVX_Vector acc1 = conv2d_dw_qf32_mpy_add3(zero, l1, w00, c1, w01, rg1, w02); \ + p1 = c1; c1 = n1; \ + \ + const HVX_Vector n2 = has_next ? hvx_vmem(r2 + ox + 32) : zero; \ + const HVX_Vector l2 = Q6_V_valign_VVR(c2, p2, 124); \ + const HVX_Vector rg2 = Q6_V_valign_VVR(n2, c2, 4); \ + acc0 = conv2d_dw_qf32_mpy_add3(acc0, l2, w20, c2, w21, rg2, w22); \ + hvx_vmem(out0 + ox) = Q6_Vsf_equals_Vqf32(acc0); \ + acc1 = conv2d_dw_qf32_mpy_add3(acc1, l2, w10, c2, w11, rg2, w12); \ + HVX_Vector acc2 = conv2d_dw_qf32_mpy_add3(zero, l2, w00, c2, w01, rg2, w02); \ + p2 = c2; c2 = n2; \ + \ + const HVX_Vector n3 = has_next ? hvx_vmem(r3 + ox + 32) : zero; \ + const HVX_Vector l3 = Q6_V_valign_VVR(c3, p3, 124); \ + const HVX_Vector rg3 = Q6_V_valign_VVR(n3, c3, 4); \ + acc1 = conv2d_dw_qf32_mpy_add3(acc1, l3, w20, c3, w21, rg3, w22); \ + hvx_vmem(out1 + ox) = Q6_Vsf_equals_Vqf32(acc1); \ + acc2 = conv2d_dw_qf32_mpy_add3(acc2, l3, w10, c3, w11, rg3, w12); \ + HVX_Vector acc3 = conv2d_dw_qf32_mpy_add3(zero, l3, w00, c3, w01, rg3, w02); \ + p3 = c3; c3 = n3; \ + \ + const HVX_Vector n4 = has_next ? hvx_vmem(r4 + ox + 32) : zero; \ + const HVX_Vector l4 = Q6_V_valign_VVR(c4, p4, 124); \ + const HVX_Vector rg4 = Q6_V_valign_VVR(n4, c4, 4); \ + acc2 = conv2d_dw_qf32_mpy_add3(acc2, l4, w20, c4, w21, rg4, w22); \ + hvx_vmem(out2 + ox) = Q6_Vsf_equals_Vqf32(acc2); \ + acc3 = conv2d_dw_qf32_mpy_add3(acc3, l4, w10, c4, w11, rg4, w12); \ + p4 = c4; c4 = n4; \ + \ + const HVX_Vector n5 = has_next ? hvx_vmem(r5 + ox + 32) : zero; \ + const HVX_Vector l5 = Q6_V_valign_VVR(c5, p5, 124); \ + const HVX_Vector rg5 = Q6_V_valign_VVR(n5, c5, 4); \ + acc3 = conv2d_dw_qf32_mpy_add3(acc3, l5, w20, c5, w21, rg5, w22); \ + hvx_vmem(out3 + ox) = Q6_Vsf_equals_Vqf32(acc3); \ + p5 = c5; c5 = n5; \ + } \ + } \ + \ + for (; ohi < oh_len; ++ohi) { \ + const float * restrict r0 = in_vtcm + (size_t) ohi * w_in; \ + const float * restrict r1 = r0 + w_in; \ + const float * restrict r2 = r1 + w_in; \ + float * restrict out_row = out_vtcm + (size_t) ohi * w_in; \ + \ + HVX_Vector p0 = zero, c0 = hvx_vmem(r0); \ + HVX_Vector p1 = zero, c1 = hvx_vmem(r1); \ + HVX_Vector p2 = zero, c2 = hvx_vmem(r2); \ + \ + for (uint32_t ox = 0; ox < w_in; ox += 32) { \ + const bool has_next = (ox + 32 < w_in); \ + \ + const HVX_Vector n0 = has_next ? hvx_vmem(r0 + ox + 32) : zero; \ + const HVX_Vector l0 = Q6_V_valign_VVR(c0, p0, 124); \ + const HVX_Vector rg0 = Q6_V_valign_VVR(n0, c0, 4); \ + HVX_Vector acc = conv2d_dw_qf32_mpy_add3(zero, l0, w00, c0, w01, rg0, w02); \ + p0 = c0; c0 = n0; \ + \ + const HVX_Vector n1 = has_next ? hvx_vmem(r1 + ox + 32) : zero; \ + const HVX_Vector l1 = Q6_V_valign_VVR(c1, p1, 124); \ + const HVX_Vector rg1 = Q6_V_valign_VVR(n1, c1, 4); \ + acc = conv2d_dw_qf32_mpy_add3(acc, l1, w10, c1, w11, rg1, w12); \ + p1 = c1; c1 = n1; \ + \ + const HVX_Vector n2 = has_next ? hvx_vmem(r2 + ox + 32) : zero; \ + const HVX_Vector l2 = Q6_V_valign_VVR(c2, p2, 124); \ + const HVX_Vector rg2 = Q6_V_valign_VVR(n2, c2, 4); \ + acc = conv2d_dw_qf32_mpy_add3(acc, l2, w20, c2, w21, rg2, w22); \ + p2 = c2; c2 = n2; \ + \ + hvx_vmem(out_row + ox) = Q6_Vsf_equals_Vqf32(acc); \ + } \ + } \ +} + +DEFINE_CONV2D_DW_WHCN_S1_K3X3(f32, conv2d_dw_splat_w_f32) +DEFINE_CONV2D_DW_WHCN_S1_K3X3(f16, conv2d_dw_splat_w_f16) + +#define DEFINE_CONV2D_DW_WHCN_S2_K3X3_W32(SUFFIX, SPLAT_W_FN) \ +__attribute__((noinline)) \ +static void conv2d_dw_whcn_s2_k3x3_w32_##SUFFIX( \ + const float * restrict in_vtcm, \ + float * restrict out_vtcm, \ + const void * w_vtcm, \ + const struct conv2d_dw_task_slice * slice, \ + const struct htp_ops_context * octx, \ + const struct htp_conv2d_dw_kernel_params * kp) { \ + \ + (void) kp; \ + const struct htp_tensor * src0 = octx->src[0]; \ + const size_t w_nb0 = src0->nb[0]; \ + const size_t w_nb1 = src0->nb[1]; \ + const char * restrict w_chan = (const char *) w_vtcm + (size_t) slice->c * src0->nb[3]; \ + const uint32_t oh_len = slice->oh_len; \ + \ + conv2d_dw_pad_vtcm_k3x3((float *) in_vtcm, slice, 32, (oh_len & 1) ? 2 : 0); \ + \ + const HVX_Vector w00 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w01 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w02 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector w10 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w11 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w12 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector w20 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w21 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w22 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector zero = Q6_V_vzero(); \ + \ + const HVX_VectorPred q_not_lane16 = Q6_Q_or_QQn(Q6_Q_vsetq_R(64), Q6_Q_vsetq2_R(68)); \ + const float * restrict in_row = in_vtcm; \ + HVX_Vector v_prev_b2 = hvx_vmem(in_row); \ + in_row += 32; \ + \ + for (uint32_t ohi = 0; ohi < oh_len; ohi += 2) { \ + const HVX_Vector v_a0 = v_prev_b2; \ + const HVX_Vector v_a1 = hvx_vmem(in_row); \ + const HVX_Vector v_a2 = hvx_vmem(in_row + 32); \ + const HVX_Vector v_b0 = v_a2; \ + const HVX_Vector v_b1 = hvx_vmem(in_row + 64); \ + const HVX_Vector v_b2 = hvx_vmem(in_row + 96); \ + in_row += 128; \ + v_prev_b2 = v_b2; \ + \ + const HVX_VectorPair vp0 = Q6_W_vdeal_VVR(v_b0, v_a0, -4); \ + const HVX_Vector ev0 = Q6_V_lo_W(vp0); \ + const HVX_Vector od0 = Q6_V_hi_W(vp0); \ + const HVX_Vector lf0 = Q6_V_vand_QV(q_not_lane16, Q6_V_valign_VVR(od0, zero, 124)); \ + HVX_Vector acc = conv2d_dw_qf32_mpy_add3(zero, lf0, w00, ev0, w01, od0, w02); \ + \ + const HVX_VectorPair vp1 = Q6_W_vdeal_VVR(v_b1, v_a1, -4); \ + const HVX_Vector ev1 = Q6_V_lo_W(vp1); \ + const HVX_Vector od1 = Q6_V_hi_W(vp1); \ + const HVX_Vector lf1 = Q6_V_vand_QV(q_not_lane16, Q6_V_valign_VVR(od1, zero, 124)); \ + acc = conv2d_dw_qf32_mpy_add3(acc, lf1, w10, ev1, w11, od1, w12); \ + \ + const HVX_VectorPair vp2 = Q6_W_vdeal_VVR(v_b2, v_a2, -4); \ + const HVX_Vector ev2 = Q6_V_lo_W(vp2); \ + const HVX_Vector od2 = Q6_V_hi_W(vp2); \ + const HVX_Vector lf2 = Q6_V_vand_QV(q_not_lane16, Q6_V_valign_VVR(od2, zero, 124)); \ + acc = conv2d_dw_qf32_mpy_add3(acc, lf2, w20, ev2, w21, od2, w22); \ + \ + hvx_vmem(out_vtcm + (size_t) ohi * 16) = Q6_Vsf_equals_Vqf32(acc); \ + } \ +} + +DEFINE_CONV2D_DW_WHCN_S2_K3X3_W32(f32, conv2d_dw_splat_w_f32) +DEFINE_CONV2D_DW_WHCN_S2_K3X3_W32(f16, conv2d_dw_splat_w_f16) + +#define DEFINE_CONV2D_DW_WHCN_S2_K3X3_W64(SUFFIX, SPLAT_W_FN) \ +__attribute__((noinline)) \ +static void conv2d_dw_whcn_s2_k3x3_w64_##SUFFIX( \ + const float * restrict in_vtcm, \ + float * restrict out_vtcm, \ + const void * w_vtcm, \ + const struct conv2d_dw_task_slice * slice, \ + const struct htp_ops_context * octx, \ + const struct htp_conv2d_dw_kernel_params * kp) { \ + \ + (void) kp; \ + const struct htp_tensor * src0 = octx->src[0]; \ + const size_t w_nb0 = src0->nb[0]; \ + const size_t w_nb1 = src0->nb[1]; \ + const char * restrict w_chan = (const char *) w_vtcm + (size_t) slice->c * src0->nb[3]; \ + const uint32_t oh_len = slice->oh_len; \ + \ + conv2d_dw_pad_vtcm_k3x3((float *) in_vtcm, slice, 64, 0); \ + \ + const HVX_Vector w00 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w01 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w02 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector w10 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w11 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w12 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector w20 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w21 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w22 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector zero = Q6_V_vzero(); \ + \ + const float * restrict in_row = in_vtcm; \ + const HVX_VectorPair vp0_init = Q6_W_vdeal_VVR(hvx_vmem(in_row + 32), hvx_vmem(in_row), -4); \ + in_row += 64; \ + HVX_Vector ev0 = Q6_V_lo_W(vp0_init); \ + HVX_Vector od0 = Q6_V_hi_W(vp0_init); \ + HVX_Vector lf0 = Q6_V_valign_VVR(od0, zero, 124); \ + \ + for (uint32_t ohi = 0; ohi < oh_len; ++ohi) { \ + const float * restrict r1_ptr = in_row; \ + const float * restrict r2_ptr = in_row + 64; \ + in_row += 128; \ + \ + HVX_Vector acc = conv2d_dw_qf32_mpy_add3(zero, lf0, w00, ev0, w01, od0, w02); \ + \ + const HVX_VectorPair vp1 = Q6_W_vdeal_VVR(hvx_vmem(r1_ptr + 32), hvx_vmem(r1_ptr), -4); \ + const HVX_Vector ev1 = Q6_V_lo_W(vp1); \ + const HVX_Vector od1 = Q6_V_hi_W(vp1); \ + const HVX_Vector lf1 = Q6_V_valign_VVR(od1, zero, 124); \ + acc = conv2d_dw_qf32_mpy_add3(acc, lf1, w10, ev1, w11, od1, w12); \ + \ + const HVX_VectorPair vp2 = Q6_W_vdeal_VVR(hvx_vmem(r2_ptr + 32), hvx_vmem(r2_ptr), -4); \ + const HVX_Vector ev2 = Q6_V_lo_W(vp2); \ + const HVX_Vector od2 = Q6_V_hi_W(vp2); \ + const HVX_Vector lf2 = Q6_V_valign_VVR(od2, zero, 124); \ + acc = conv2d_dw_qf32_mpy_add3(acc, lf2, w20, ev2, w21, od2, w22); \ + \ + hvx_vmem(out_vtcm + (size_t) ohi * 32) = Q6_Vsf_equals_Vqf32(acc); \ + \ + ev0 = ev2; \ + od0 = od2; \ + lf0 = lf2; \ + } \ +} + +DEFINE_CONV2D_DW_WHCN_S2_K3X3_W64(f32, conv2d_dw_splat_w_f32) +DEFINE_CONV2D_DW_WHCN_S2_K3X3_W64(f16, conv2d_dw_splat_w_f16) + +#define DEFINE_CONV2D_DW_WHCN_S2_K3X3(SUFFIX, SPLAT_W_FN) \ +__attribute__((noinline)) \ +static void conv2d_dw_whcn_s2_k3x3_##SUFFIX( \ + const float * restrict in_vtcm, \ + float * restrict out_vtcm, \ + const void * w_vtcm, \ + const struct conv2d_dw_task_slice * slice, \ + const struct htp_ops_context * octx, \ + const struct htp_conv2d_dw_kernel_params * kp) { \ + \ + const struct htp_tensor * src0 = octx->src[0]; \ + const size_t w_nb0 = src0->nb[0]; \ + const size_t w_nb1 = src0->nb[1]; \ + const char * restrict w_chan = (const char *) w_vtcm + (size_t) slice->c * src0->nb[3]; \ + const uint32_t oh_len = slice->oh_len; \ + const uint32_t w_in = kp->w; \ + const uint32_t ow = kp->ow; \ + \ + conv2d_dw_pad_vtcm_k3x3((float *) in_vtcm, slice, w_in, 0); \ + \ + const HVX_Vector w00 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w01 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w02 = SPLAT_W_FN(w_chan + 0 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector w10 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w11 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w12 = SPLAT_W_FN(w_chan + 1 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector w20 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 0 * w_nb0); \ + const HVX_Vector w21 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 1 * w_nb0); \ + const HVX_Vector w22 = SPLAT_W_FN(w_chan + 2 * w_nb1 + 2 * w_nb0); \ + const HVX_Vector zero = Q6_V_vzero(); \ + \ + for (uint32_t ohi = 0; ohi < oh_len; ++ohi) { \ + const float * restrict r0 = in_vtcm + (size_t) (2 * ohi) * w_in; \ + const float * restrict r1 = r0 + w_in; \ + const float * restrict r2 = r1 + w_in; \ + float * restrict out_row = out_vtcm + (size_t) ohi * ow; \ + \ + HVX_Vector prev_od0 = zero; \ + HVX_Vector prev_od1 = zero; \ + HVX_Vector prev_od2 = zero; \ + \ + for (uint32_t ox = 0; ox < ow; ox += 32) { \ + const uint32_t ix = 2 * ox; \ + const bool has1 = (ix + 32 < w_in); \ + \ + const HVX_VectorPair vp0 = Q6_W_vdeal_VVR(has1 ? hvx_vmem(r0 + ix + 32) : zero, hvx_vmem(r0 + ix), -4); \ + const HVX_Vector ev0 = Q6_V_lo_W(vp0); \ + const HVX_Vector od0 = Q6_V_hi_W(vp0); \ + const HVX_Vector lf0 = Q6_V_valign_VVR(od0, prev_od0, 124); \ + prev_od0 = od0; \ + HVX_Vector acc = conv2d_dw_qf32_mpy_add3(zero, lf0, w00, ev0, w01, od0, w02); \ + \ + const HVX_VectorPair vp1 = Q6_W_vdeal_VVR(has1 ? hvx_vmem(r1 + ix + 32) : zero, hvx_vmem(r1 + ix), -4); \ + const HVX_Vector ev1 = Q6_V_lo_W(vp1); \ + const HVX_Vector od1 = Q6_V_hi_W(vp1); \ + const HVX_Vector lf1 = Q6_V_valign_VVR(od1, prev_od1, 124); \ + prev_od1 = od1; \ + acc = conv2d_dw_qf32_mpy_add3(acc, lf1, w10, ev1, w11, od1, w12); \ + \ + const HVX_VectorPair vp2 = Q6_W_vdeal_VVR(has1 ? hvx_vmem(r2 + ix + 32) : zero, hvx_vmem(r2 + ix), -4); \ + const HVX_Vector ev2 = Q6_V_lo_W(vp2); \ + const HVX_Vector od2 = Q6_V_hi_W(vp2); \ + const HVX_Vector lf2 = Q6_V_valign_VVR(od2, prev_od2, 124); \ + prev_od2 = od2; \ + acc = conv2d_dw_qf32_mpy_add3(acc, lf2, w20, ev2, w21, od2, w22); \ + \ + if (ox + 32 <= ow) { \ + hvx_vmem(out_row + ox) = Q6_Vsf_equals_Vqf32(acc); \ + } else { \ + const uint32_t rem = ow - ox; \ + const HVX_VectorPred q = Q6_Q_vsetq_R(rem * sizeof(float)); \ + hvx_vec_store_u(out_row + ox, rem * sizeof(float), Q6_V_vmux_QVV(q, Q6_Vsf_equals_Vqf32(acc), zero)); \ + } \ + } \ + } \ +} + +DEFINE_CONV2D_DW_WHCN_S2_K3X3(f32, conv2d_dw_splat_w_f32) +DEFINE_CONV2D_DW_WHCN_S2_K3X3(f16, conv2d_dw_splat_w_f16) + +#define DEFINE_CONV2D_DW_WHCN_S1(SUFFIX, SPLAT_W_FN) \ +__attribute__((noinline)) \ +static void conv2d_dw_whcn_s1_##SUFFIX( \ + const float * in_vtcm, \ + float * out_vtcm, \ + const void * w_vtcm, \ + const struct conv2d_dw_task_slice * slice, \ + const struct htp_ops_context * octx, \ + const struct htp_conv2d_dw_kernel_params * kp) { \ + \ + const struct htp_tensor * src0 = octx->src[0]; \ + const char * w_chan = (const char *) w_vtcm + (size_t) slice->c * src0->nb[3]; \ + const int32_t p0 = kp->p0; \ + const int32_t p1 = kp->p1; \ + const int32_t s1 = kp->s1; \ + const int32_t d1 = kp->d1; \ + const int32_t w = (int32_t) kp->w; \ + const HVX_Vector zero = Q6_V_vzero(); \ + \ + for (uint32_t ohi = 0; ohi < slice->oh_len; ++ohi) { \ + const uint32_t oh = slice->oh_start + ohi; \ + float * out_row = out_vtcm + (size_t) ohi * kp->ow; \ + \ + for (uint32_t ox = 0; ox < kp->ow; ox += 32) { \ + const uint32_t rem = kp->ow - ox; \ + const uint32_t nbytes = (rem < 32) ? (rem * sizeof(float)) : 128; \ + HVX_Vector acc0 = zero; \ + HVX_Vector acc1 = zero; \ + HVX_Vector acc2 = zero; \ + HVX_Vector acc3 = zero; \ + \ + for (uint32_t kh = 0; kh < kp->kh; ++kh) { \ + const int32_t ih = (int32_t) (oh * s1 + kh * d1) - p1; \ + if (ih < 0 || ih >= (int32_t) kp->h) { \ + continue; \ + } \ + const float * in_row = in_vtcm + (size_t) (ih - slice->ih_first) * kp->w; \ + const char * w_row = w_chan + (size_t) kh * src0->nb[1]; \ + \ + for (uint32_t kw = 0; kw < kp->kw; ++kw) { \ + const HVX_Vector w_v = SPLAT_W_FN(w_row + (size_t) kw * src0->nb[0]); \ + const int32_t base_iw = (int32_t) ox - p0 + (int32_t) kw; \ + HVX_Vector in_v; \ + \ + if (base_iw >= 0) { \ + in_v = hvx_vmemu(in_row + base_iw); \ + } else { \ + const int32_t pad = -base_iw; \ + const HVX_Vector v = hvx_vmemu(in_row); \ + in_v = Q6_V_valign_VVR(v, zero, 128 - (uint32_t) pad * sizeof(float)); \ + } \ + if (base_iw + 31 >= w) { \ + const int32_t valid = w - base_iw; \ + const HVX_VectorPred q = Q6_Q_vsetq_R(MAX(0, valid) * sizeof(float)); \ + in_v = Q6_V_vmux_QVV(q, in_v, zero); \ + } \ + \ + const HVX_Vector prod = Q6_Vqf32_vmpy_VsfVsf(in_v, w_v); \ + switch (kw & 3) { \ + case 0: acc0 = Q6_Vqf32_vadd_Vqf32Vqf32(acc0, prod); break; \ + case 1: acc1 = Q6_Vqf32_vadd_Vqf32Vqf32(acc1, prod); break; \ + case 2: acc2 = Q6_Vqf32_vadd_Vqf32Vqf32(acc2, prod); break; \ + case 3: acc3 = Q6_Vqf32_vadd_Vqf32Vqf32(acc3, prod); break; \ + } \ + } \ + } \ + \ + const HVX_Vector acc01 = Q6_Vqf32_vadd_Vqf32Vqf32(acc0, acc1); \ + const HVX_Vector acc23 = Q6_Vqf32_vadd_Vqf32Vqf32(acc2, acc3); \ + HVX_Vector acc_out = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(acc01, acc23)); \ + if (rem < 32) { \ + const HVX_VectorPred q = Q6_Q_vsetq_R(rem * sizeof(float)); \ + acc_out = Q6_V_vmux_QVV(q, acc_out, zero); \ + } \ + \ + hvx_vec_store_u(out_row + ox, nbytes, acc_out); \ + } \ + } \ +} + +DEFINE_CONV2D_DW_WHCN_S1(f32, conv2d_dw_splat_w_f32) +DEFINE_CONV2D_DW_WHCN_S1(f16, conv2d_dw_splat_w_f16) + +#define DEFINE_CONV2D_DW_WHCN_S2(SUFFIX, SPLAT_W_FN) \ +__attribute__((noinline)) \ +static void conv2d_dw_whcn_s2_##SUFFIX( \ + const float * in_vtcm, \ + float * out_vtcm, \ + const void * w_vtcm, \ + const struct conv2d_dw_task_slice * slice, \ + const struct htp_ops_context * octx, \ + const struct htp_conv2d_dw_kernel_params * kp) { \ + \ + const struct htp_tensor * src0 = octx->src[0]; \ + const char * w_chan = (const char *) w_vtcm + (size_t) slice->c * src0->nb[3]; \ + const int32_t p0 = kp->p0; \ + const int32_t p1 = kp->p1; \ + const int32_t s1 = kp->s1; \ + const int32_t d1 = kp->d1; \ + const int32_t w = (int32_t) kp->w; \ + const HVX_Vector zero = Q6_V_vzero(); \ + \ + for (uint32_t ohi = 0; ohi < slice->oh_len; ++ohi) { \ + const uint32_t oh = slice->oh_start + ohi; \ + float * out_row = out_vtcm + (size_t) ohi * kp->ow; \ + \ + for (uint32_t ox = 0; ox < kp->ow; ox += 32) { \ + const uint32_t rem = kp->ow - ox; \ + const uint32_t nbytes = (rem < 32) ? (rem * sizeof(float)) : 128; \ + HVX_Vector acc0 = zero; \ + HVX_Vector acc1 = zero; \ + HVX_Vector acc2 = zero; \ + HVX_Vector acc3 = zero; \ + \ + for (uint32_t kh = 0; kh < kp->kh; ++kh) { \ + const int32_t ih = (int32_t) (oh * s1 + kh * d1) - p1; \ + if (ih < 0 || ih >= (int32_t) kp->h) { \ + continue; \ + } \ + const float * in_row = in_vtcm + (size_t) (ih - slice->ih_first) * kp->w; \ + const char * w_row = w_chan + (size_t) kh * src0->nb[1]; \ + \ + for (uint32_t kw = 0; kw < kp->kw; ++kw) { \ + const HVX_Vector w_v = SPLAT_W_FN(w_row + (size_t) kw * src0->nb[0]); \ + const int32_t base_iw = (int32_t) (2 * ox) - p0 + (int32_t) kw; \ + HVX_Vector in_v; \ + \ + if (base_iw >= 0) { \ + const HVX_Vector v0 = hvx_vmemu(in_row + base_iw); \ + const HVX_Vector v1 = hvx_vmemu(in_row + base_iw + 32); \ + in_v = Q6_V_lo_W(Q6_W_vdeal_VVR(v1, v0, -4)); \ + } else { \ + const int32_t pad = -base_iw; \ + const HVX_Vector v0 = hvx_vmemu(in_row); \ + const HVX_Vector v1 = hvx_vmemu(in_row + 32); \ + const HVX_VectorPair vp = Q6_W_vdeal_VVR(v1, v0, -4); \ + if (pad & 1) { \ + const uint32_t shift_lanes = (uint32_t) (pad + 1) / 2; \ + const HVX_Vector v_odd = Q6_V_hi_W(vp); \ + in_v = Q6_V_valign_VVR(v_odd, zero, 128 - shift_lanes * sizeof(float)); \ + } else { \ + const uint32_t shift_lanes = (uint32_t) pad / 2; \ + const HVX_Vector v_even = Q6_V_lo_W(vp); \ + in_v = Q6_V_valign_VVR(v_even, zero, 128 - shift_lanes * sizeof(float)); \ + } \ + } \ + if (base_iw + 62 >= w) { \ + const int32_t valid = (w - base_iw + 1) / 2; \ + const HVX_VectorPred q = Q6_Q_vsetq_R(MAX(0, valid) * sizeof(float)); \ + in_v = Q6_V_vmux_QVV(q, in_v, zero); \ + } \ + \ + const HVX_Vector prod = Q6_Vqf32_vmpy_VsfVsf(in_v, w_v); \ + switch (kw & 3) { \ + case 0: acc0 = Q6_Vqf32_vadd_Vqf32Vqf32(acc0, prod); break; \ + case 1: acc1 = Q6_Vqf32_vadd_Vqf32Vqf32(acc1, prod); break; \ + case 2: acc2 = Q6_Vqf32_vadd_Vqf32Vqf32(acc2, prod); break; \ + case 3: acc3 = Q6_Vqf32_vadd_Vqf32Vqf32(acc3, prod); break; \ + } \ + } \ + } \ + \ + const HVX_Vector acc01 = Q6_Vqf32_vadd_Vqf32Vqf32(acc0, acc1); \ + const HVX_Vector acc23 = Q6_Vqf32_vadd_Vqf32Vqf32(acc2, acc3); \ + HVX_Vector acc_out = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(acc01, acc23)); \ + if (rem < 32) { \ + const HVX_VectorPred q = Q6_Q_vsetq_R(rem * sizeof(float)); \ + acc_out = Q6_V_vmux_QVV(q, acc_out, zero); \ + } \ + \ + hvx_vec_store_u(out_row + ox, nbytes, acc_out); \ + } \ + } \ +} + +DEFINE_CONV2D_DW_WHCN_S2(f32, conv2d_dw_splat_w_f32) +DEFINE_CONV2D_DW_WHCN_S2(f16, conv2d_dw_splat_w_f16) + +#define DEFINE_CONV2D_DW_CWHN(SUFFIX, LOAD_W_FN) \ +__attribute__((noinline)) \ +static void conv2d_dw_cwhn_##SUFFIX( \ + const float * in_vtcm, \ + float * out_vtcm, \ + const void * w_vtcm, \ + const struct conv2d_dw_task_slice * slice, \ + const struct htp_ops_context * octx, \ + const struct htp_conv2d_dw_kernel_params * kp) { \ + \ + const struct htp_tensor * src0 = octx->src[0]; \ + const uint32_t in_row_stride = kp->w * kp->c; \ + const uint32_t out_row_stride = kp->ow * kp->c; \ + const HVX_Vector zero = Q6_V_vzero(); \ + \ + for (uint32_t cb = 0; cb < kp->c; cb += 32) { \ + const uint32_t rem = MIN(32, kp->c - cb); \ + const char * w_base = (const char *) w_vtcm + (size_t) cb * src0->nb[3]; \ + \ + for (uint32_t ohi = 0; ohi < slice->oh_len; ++ohi) { \ + const uint32_t oh = slice->oh_start + ohi; \ + float * out_row = out_vtcm + (size_t) ohi * out_row_stride + cb; \ + \ + for (uint32_t ow = 0; ow < kp->ow; ++ow) { \ + HVX_Vector acc0 = zero; \ + HVX_Vector acc1 = zero; \ + HVX_Vector acc2 = zero; \ + HVX_Vector acc3 = zero; \ + \ + for (uint32_t kh = 0; kh < kp->kh; ++kh) { \ + const int32_t ih = (int32_t)(oh * kp->s1 + kh * kp->d1) - kp->p1; \ + if (ih < 0 || ih >= (int32_t)kp->h) { \ + continue; \ + } \ + const float * in_row = in_vtcm + (size_t)(ih - slice->ih_first) * in_row_stride + cb; \ + const char * w_row = w_base + (size_t) kh * src0->nb[1]; \ + \ + for (uint32_t kw = 0; kw < kp->kw; ++kw) { \ + const int32_t iw = (int32_t)(ow * kp->s0 + kw * kp->d0) - kp->p0; \ + if (iw < 0 || iw >= (int32_t)kp->w) { \ + continue; \ + } \ + \ + const float * in_ptr = in_row + (size_t) iw * kp->c; \ + const char * w_ptr = w_row + (size_t) kw * src0->nb[0]; \ + const HVX_Vector v_in = hvx_vmemu(in_ptr); \ + const HVX_Vector v_w = LOAD_W_FN(w_ptr); \ + const HVX_Vector prod = Q6_Vqf32_vmpy_VsfVsf(v_in, v_w); \ + switch (kw & 3) { \ + case 0: acc0 = Q6_Vqf32_vadd_Vqf32Vqf32(acc0, prod); break; \ + case 1: acc1 = Q6_Vqf32_vadd_Vqf32Vqf32(acc1, prod); break; \ + case 2: acc2 = Q6_Vqf32_vadd_Vqf32Vqf32(acc2, prod); break; \ + case 3: acc3 = Q6_Vqf32_vadd_Vqf32Vqf32(acc3, prod); break; \ + } \ + } \ + } \ + \ + const HVX_Vector acc01 = Q6_Vqf32_vadd_Vqf32Vqf32(acc0, acc1); \ + const HVX_Vector acc23 = Q6_Vqf32_vadd_Vqf32Vqf32(acc2, acc3); \ + const HVX_Vector acc_out = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(acc01, acc23)); \ + \ + float * dst_ptr = out_row + (size_t) ow * kp->c; \ + if (rem == 32) { \ + hvx_vmemu(dst_ptr) = acc_out; \ + } else { \ + hvx_vec_store_u(dst_ptr, rem * sizeof(float), acc_out); \ + } \ + } \ + } \ + } \ +} + +DEFINE_CONV2D_DW_CWHN(f32, conv2d_dw_load_w_f32) +DEFINE_CONV2D_DW_CWHN(f16, conv2d_dw_load_w_f16) + +typedef void (*conv2d_dw_compute_fn_t)( + const float * in_vtcm, + float * out_vtcm, + const void * w_vtcm, + const struct conv2d_dw_task_slice * slice, + const struct htp_ops_context * octx, + const struct htp_conv2d_dw_kernel_params * kp); + +static const conv2d_dw_compute_fn_t conv2d_dw_kernels[HTP_CONV2D_DW_KERNEL_COUNT][2] = { + [HTP_CONV2D_DW_KERNEL_UNSUPPORTED] = { NULL, NULL }, + [HTP_CONV2D_DW_KERNEL_CWHN] = { conv2d_dw_cwhn_f32, conv2d_dw_cwhn_f16 }, + [HTP_CONV2D_DW_KERNEL_WHCN_S1] = { conv2d_dw_whcn_s1_f32, conv2d_dw_whcn_s1_f16 }, + [HTP_CONV2D_DW_KERNEL_WHCN_S2] = { conv2d_dw_whcn_s2_f32, conv2d_dw_whcn_s2_f16 }, + [HTP_CONV2D_DW_KERNEL_WHCN_S1_K3X3] = { conv2d_dw_whcn_s1_k3x3_f32, conv2d_dw_whcn_s1_k3x3_f16 }, + [HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W32] = { conv2d_dw_whcn_s2_k3x3_w32_f32, conv2d_dw_whcn_s2_k3x3_w32_f16 }, + [HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W64] = { conv2d_dw_whcn_s2_k3x3_w64_f32, conv2d_dw_whcn_s2_k3x3_w64_f16 }, + [HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3] = { conv2d_dw_whcn_s2_k3x3_f32, conv2d_dw_whcn_s2_k3x3_f16 }, +}; + +struct conv2d_dw_dma_context { + struct htp_ops_context * octx; + const struct htp_conv2d_dw_kernel_params * kp; + struct htp_conv2d_dw_vtcm_layout layout; + const void * w_vtcm; + conv2d_dw_compute_fn_t compute_fn; + uint32_t task_start; + uint32_t task_count; + uint32_t tasks_per_hvx; +}; + +static void conv2d_dw_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct conv2d_dw_dma_context * ctx = (const struct conv2d_dw_dma_context *) data; + const struct htp_ops_context * octx = ctx->octx; + const struct htp_conv2d_dw_kernel_params * kp = ctx->kp; + const struct htp_conv2d_dw_vtcm_layout * L = &ctx->layout; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + conv2d_dw_compute_fn_t compute_fn = ctx->compute_fn; + + const uint32_t first = ctx->task_start + ith * ctx->tasks_per_hvx; + const uint32_t last = MIN(first + ctx->tasks_per_hvx, ctx->task_start + ctx->task_count); + if (first >= last) { + return; + } + + (void) nth; + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size; + const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; + const uint32_t pad_top_stride = kp->pad_top_stride; + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; + + float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; + float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) }; + struct conv2d_dw_task_slice sl2[2]; + + const uint32_t total = last - first; + + for (uint32_t i = 0; i < total && i < 2; ++i) { + conv2d_dw_get_slice(&sl2[i], first + i, src1, dst, kp); + + dma_queue_push(dma_q, + dma_make_data(dst->data, dstb2[i]), + sl2[i].dst_bytes, dst_spad_half, + sl2[i].dst_bytes, 0); + + float * src_dst = srcb2[i] + (size_t) sl2[i].top_pad * pad_top_stride; + dma_queue_push(dma_q, + dma_make_data(src_dst, sl2[i].src_addr), + src_spad_half, sl2[i].src_bytes, + sl2[i].src_bytes, sl2[i].src_bytes > 0 ? 1 : 0); + } + + for (uint32_t i = 0; i < total; ++i) { + const uint32_t task = first + i; + const uint32_t buf = i & 1u; + float * srcb = srcb2[buf]; + float * dstb = dstb2[buf]; + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) task); + compute_fn(srcb, dstb, ctx->w_vtcm, &sl2[buf], octx, kp); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) task); + + dma_queue_push(dma_q, + dma_make_data(sl2[buf].dst_addr, dstb), + sl2[buf].dst_bytes, dst_spad_half, + sl2[buf].dst_bytes, sl2[buf].dst_bytes > 0 ? 1 : 0); + + if (i + 2 < total) { + conv2d_dw_get_slice(&sl2[buf], task + 2, src1, dst, kp); + + float * src_dst = srcb + (size_t) sl2[buf].top_pad * pad_top_stride; + dma_queue_push(dma_q, + dma_make_data(src_dst, sl2[buf].src_addr), + src_spad_half, sl2[buf].src_bytes, + sl2[buf].src_bytes, sl2[buf].src_bytes > 0 ? 1 : 0); + } + } + + dma_queue_flush(dma_q); + + FARF(HIGH, "conv2d_dw-f32-dma %d/%d: %ux%ux%ux%u -> %ux%ux%ux%u (%u:%u)\n", + ith, nth, kp->w, kp->h, kp->c, kp->n, kp->ow, kp->oh, kp->c, kp->n, first, last); +} + +int op_conv_2d_dw(struct htp_ops_context * octx) { + const struct htp_conv2d_dw_kernel_params * kp = + (const struct htp_conv2d_dw_kernel_params *) octx->kernel_params; + + if (kp->total_tasks == 0) { + return HTP_STATUS_OK; + } + + if (kp->kernel_type <= HTP_CONV2D_DW_KERNEL_UNSUPPORTED || + kp->kernel_type >= HTP_CONV2D_DW_KERNEL_COUNT) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t pad_rows = (kp->pad_top_stride > 0) ? 4 : 0; + const uint32_t max_ih_chunk = MIN(kp->h, (kp->oh_chunk - 1) * (uint32_t) kp->s1 + (kp->kh - 1) * (uint32_t) kp->d1 + 1); + const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * kp->row_src_bytes + 128; + const uint32_t max_dst_bytes = kp->oh_chunk * kp->row_dst_bytes + 128; + + const struct htp_tensor * src0 = octx->src[0]; + const uint32_t elem_size = kp->is_f16 ? sizeof(__fp16) : sizeof(float); + const uint32_t w_size = src0->size; + + struct htp_conv2d_dw_vtcm_layout layout; + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, kp->n_hvx); + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + void * w_vtcm = VTCM_LAYOUT_PTR(void, vtcm_base, layout.off_w); + + struct htp_tensor w_vtcm_tensor = *src0; + w_vtcm_tensor.data = (dma_addr_t) w_vtcm; + + if (src0->nb[0] == elem_size) { + dma_cpy_sametype_sameshape(octx->ctx->dma[0], &w_vtcm_tensor, src0, elem_size); + } else { + dma_sync_read(octx->ctx->dma[0], w_vtcm, src0->data, w_size); + } + dma_queue_flush(octx->ctx->dma[0]); + + uint32_t task_start = 0; + uint32_t task_count = kp->total_tasks; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + task_count, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + task_count = range.count; + } + if (task_count == 0) { + return HTP_STATUS_OK; + } + + const uint32_t nth = MIN(kp->n_hvx, task_count); + const uint32_t tasks_per_hvx = (task_count + nth - 1) / nth; + + struct conv2d_dw_dma_context ctx = { + .octx = octx, + .kp = kp, + .layout = layout, + .w_vtcm = w_vtcm, + .compute_fn = conv2d_dw_kernels[kp->kernel_type][kp->is_f16 ? 1 : 0], + .task_start = task_start, + .task_count = task_count, + .tasks_per_hvx = tasks_per_hvx, + }; + + work_queue_run(octx->ctx->hvx_queue, conv2d_dw_dma_thread, &ctx, nth); + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.h b/ggml/src/ggml-hexagon/htp/conv2d-ops.h new file mode 100644 index 000000000..29dfa812f --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.h @@ -0,0 +1,199 @@ +#ifndef HTP_CONV2D_OPS_H +#define HTP_CONV2D_OPS_H + +#include <stddef.h> +#include <stdint.h> + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-ops.h" +#include "htp-vtcm.h" + +enum htp_conv2d_kernel_type { + HTP_CONV2D_KERNEL_UNSUPPORTED = 0, + HTP_CONV2D_KERNEL_HMX_WINDOW, // HMX convolution with sliding-window staging + HTP_CONV2D_KERNEL_HMX_TILED, // tiled implicit-GEMM convolution on HMX +}; + +// HMX_WINDOW output staging buffers per thread +#define HTP_CONV2D_WIN_OBUF 2 + +// HMX scales of one row tile at the start of VTCM: 32 (scale, bias) fp16 pairs and 128 zero bytes +#define HTP_CONV2D_HMX_SCALES_SIZE 256 + +struct htp_conv2d_tiled_params { + uint32_t w_kstride; + uint32_t k_size; + uint32_t k_group; + uint32_t k_group_pad; + uint32_t n_groups; + uint32_t n_k_tiles; + uint32_t n_ow_tiles; + uint32_t seg_tiles; + uint32_t stage_row; + uint32_t n_stage; + uint32_t wk_chunk; + struct fastdiv_values div_ow_tiles; + struct fastdiv_values div_oh; + struct fastdiv_values div_khr; // kernel ne[1], unfolds conv_3d kh = kd * KHr + kh' + struct fastdiv_values div_ohr; // dst ne[1], unfolds conv_3d oh = od * OHr + oh' +}; + +struct htp_conv2d_window_params { + uint32_t fold; + uint32_t n_cc; + uint32_t act_rt_last; + uint32_t sr; + uint32_t ihs; + uint32_t max_off; + uint32_t n_bc; + uint32_t seg_b; + uint32_t seg_t; + uint32_t w_cs; + struct fastdiv_values div_sr; + struct fastdiv_values div_ihs; + struct fastdiv_values div_kw; + struct fastdiv_values div_tg; + struct fastdiv_values div_cc; + struct fastdiv_values div_cs; +}; + +struct htp_conv2d_kernel_params { + uint8_t kernel_type; + uint8_t n_hvx; + uint8_t cwhn; + uint8_t w_kmajor; + + uint32_t total_rows; + uint32_t n_row_tiles; + uint32_t ct_chunk; + uint32_t rt_block; + uint32_t stage_size; + uint32_t thr_size; + uint32_t vtcm_size; + + union { + struct htp_conv2d_tiled_params tiled; + struct htp_conv2d_window_params window; + }; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_conv2d_kernel_params) <= 128, "htp_conv2d_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_conv2d_kernel_params) <= 128, "htp_conv2d_kernel_params is too large for kernel_params blob"); +#endif + +enum htp_conv2d_dw_kernel_type { + HTP_CONV2D_DW_KERNEL_UNSUPPORTED = 0, + HTP_CONV2D_DW_KERNEL_CWHN, // CWHN channel-contiguous layout + HTP_CONV2D_DW_KERNEL_WHCN_S1, // WHCN general stride-1 + HTP_CONV2D_DW_KERNEL_WHCN_S2, // WHCN general stride-2 + HTP_CONV2D_DW_KERNEL_WHCN_S1_K3X3, // WHCN aligned w%32==0, 3x3 s=1 p=1 d=1 + HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W32, // WHCN w=32->ow=16, 3x3 s=2 p=1 d=1 (2 rows per vector) + HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W64, // WHCN w=64->ow=32, 3x3 s=2 p=1 d=1 (1 vector rolling kh) + HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3, // WHCN aligned w%32==0 (w>=96), 3x3 s=2 p=1 d=1 + HTP_CONV2D_DW_KERNEL_COUNT, +}; + +struct htp_conv2d_dw_kernel_params { + uint8_t kernel_type; + uint8_t n_hvx; + uint8_t cwhn; + uint8_t is_f16; + + uint32_t total_tasks; + uint32_t vtcm_size; + uint32_t w; + uint32_t h; + uint32_t c; + uint32_t n; + uint32_t kw; + uint32_t kh; + uint32_t ow; + uint32_t oh; + int32_t s0; + int32_t s1; + int32_t p0; + int32_t p1; + int32_t d0; + int32_t d1; + uint32_t oh_chunk; + uint32_t n_chunks; + uint32_t row_src_bytes; + uint32_t row_dst_bytes; + uint32_t pad_top_stride; + struct fastdiv_values div_c; + struct fastdiv_values div_n_chunks; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_conv2d_dw_kernel_params) <= 128, "htp_conv2d_dw_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_conv2d_dw_kernel_params) <= 128, "htp_conv2d_dw_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_conv2d_dw_vtcm_layout { + size_t total_bytes; + size_t off_w; + size_t off_src; + size_t off_dst; + size_t src_spad_half_size; + size_t dst_spad_half_size; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; +}; + +static inline void htp_conv2d_dw_vtcm_layout_build( + struct htp_conv2d_dw_vtcm_layout * L, + uint32_t max_src_bytes, + uint32_t max_dst_bytes, + uint32_t w_size, + uint32_t n_hvx +) { + L->src_spad_half_size = hex_round_up(max_src_bytes + 256, 128); + L->dst_spad_half_size = hex_round_up(max_dst_bytes + 128, 128); + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_w, hex_round_up(w_size, 128)); + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + L->total_bytes = off; +} + +struct htp_conv2d_vtcm_layout { + size_t total_bytes; + size_t off_scales; + size_t off_w; + size_t off_act[2]; + size_t off_out[2]; + size_t off_thr; + + size_t act_size; +}; + +static inline void htp_conv2d_vtcm_layout_build( + struct htp_conv2d_vtcm_layout * L, + uint32_t rt_block, + size_t w_size, + size_t act_size, + size_t out_size, + size_t thr_size, + uint32_t n_hvx +) { + const size_t sc = hex_round_up((size_t) rt_block * HTP_CONV2D_HMX_SCALES_SIZE, 2048); + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_scales, sc); + VTCM_LAYOUT_ALLOC(off, off_w, w_size); + VTCM_LAYOUT_ALLOC(off, off_act[0], act_size); + VTCM_LAYOUT_ALLOC(off, off_act[1], act_size); + VTCM_LAYOUT_ALLOC(off, off_out[0], out_size); + VTCM_LAYOUT_ALLOC(off, off_out[1], out_size); + VTCM_LAYOUT_ALLOC(off, off_thr, (size_t) n_hvx * thr_size); + L->act_size = act_size; + L->total_bytes = off; +} + +#endif // HTP_CONV2D_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index df38d3eed..cdf4d8ec6 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -11,42 +11,34 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" -#include "hex-cpy-dma.h" +#include "cpy-ops.h" +#include "dma-copy.h" #include "htp-ctx.h" #include "htp-fence.h" #include "htp-ops.h" #include "htp-tensor.h" #include "hvx-utils.h" +#include "gather-rows.h" struct htp_copy_context { - struct htp_ops_context * octx; + struct htp_ops_context * octx; + const struct htp_copy_kernel_params * kparams; - uint32_t src0_type_size; - uint32_t src0_block_size; + uint32_t row_start; + uint32_t nrows; + uint32_t src0_nrows_per_hvx; - uint32_t dst_type_size; - uint32_t dst_block_size; + uint32_t elem_start; + uint32_t nelem; + uint32_t elem_per_hvx; - uint32_t src0_blocks_per_row; - uint32_t dst_blocks_per_row; + struct htp_copy_convert_vtcm_layout layout; + uint8_t * vtcm_src0; + uint8_t * vtcm_dst; - uint32_t elem_start; - uint32_t nelem; - uint32_t elem_per_thread; - - uint32_t src0_nrows_per_thread; - uint32_t row_start; - uint32_t nrows; - - struct fastdiv_values div_ne01; - struct fastdiv_values div_ne02_ne01; - - struct fastdiv_values div_ne0; - struct fastdiv_values div_ne1_ne0; - struct fastdiv_values div_ne2_ne1_ne0; - struct fastdiv_values div_ne00; - struct fastdiv_values div_ne01_ne00; - struct fastdiv_values div_ne02_ne01_ne00; + uint32_t tile_start; + uint32_t ntiles; + uint32_t tiles_per_hvx; }; #define cpy_preamble \ @@ -73,108 +65,56 @@ struct htp_copy_context { const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; -#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_copy_context * ct = (struct htp_copy_context *) data; \ - struct htp_ops_context * octx = ct->octx; \ - cpy_preamble; \ - const uint32_t dr = ct->src0_nrows_per_thread; \ - const uint32_t ir0 = ct->row_start + dr * ith; \ - const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ - if (ir0 >= ir1) return; \ - const bool contiguous = htp_tensor_is_contiguous(src0, ELEM_SIZE) && htp_tensor_is_contiguous(dst, ELEM_SIZE); \ - if (contiguous) { \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - dma_addr_t dst_addr = dst->data + ir0 * ne00 * ELEM_SIZE; \ - dma_addr_t src0_addr = src0->data + ir0 * ne00 * ELEM_SIZE; \ - cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (ir1 - ir0) * ne00 * ELEM_SIZE); \ - dma_queue_flush(dma_q); \ - return; \ - } \ - const uint32_t ne02_ne01 = ne02 * ne01; \ - uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); \ - uint32_t rem = ir0 - i03 * ne02_ne01; \ - uint32_t i02 = fastdiv(rem, &ct->div_ne01); \ - uint32_t i01 = rem - i02 * ne01; \ - uint8_t * dst_ptr = (uint8_t *) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \ - uint8_t * src0_ptr = (uint8_t *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ - for (uint32_t r = ir0; r < ir1; r++) { \ - hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \ - hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ - dst_ptr += nb1; \ - src0_ptr += nb01; \ - if (++i01 == ne01) { \ - i01 = 0; \ - if (++i02 == ne02) { \ - i02 = 0; \ - i03++; \ - } \ - dst_ptr = (uint8_t *) dst->data + i02*nb2 + i03*nb3; \ - src0_ptr = (uint8_t *) src0->data + i02*nb02 + i03*nb03; \ - } \ - } \ -} - -DEFINE_CPY_SAMESHAPE(f32, float, 4) -DEFINE_CPY_SAMESHAPE(f16, __fp16, 2) -DEFINE_CPY_SAMESHAPE(i32, int32_t, 4) - #define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ +static void cpy_thread_##NAME##_reshape(const htp_thread_t * th, unsigned int nth, unsigned int ith, \ + void * data) { \ struct htp_copy_context * ct = (struct htp_copy_context *) data; \ struct htp_ops_context * octx = ct->octx; \ cpy_preamble; \ - const uint32_t th_nelem = ct->elem_per_thread; \ + const uint32_t th_nelem = ct->elem_per_hvx; \ const uint32_t th_start = ct->elem_start + ith * th_nelem; \ const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \ if (th_start >= th_end) return; \ \ - if (htp_tensor_is_contiguous(src0, ELEM_SIZE) && htp_tensor_is_contiguous(dst, ELEM_SIZE)) { \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - dma_addr_t dst_addr = dst->data + th_start * ELEM_SIZE; \ - dma_addr_t src0_addr = src0->data + th_start * ELEM_SIZE; \ - cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (th_end - th_start) * ELEM_SIZE); \ - dma_queue_flush(dma_q); \ - return; \ - } \ + dma_queue * dma_q = th->dma_q; \ \ const uint32_t ne01_ne00 = ne01 * ne00; \ const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \ const uint32_t ne1_ne0 = ne1 * ne0; \ const uint32_t ne2_ne1_ne0 = ne2 * ne1_ne0; \ \ + const struct htp_copy_reshape_params * rsh = &ct->kparams->u.reshape; \ uint32_t e = th_start; \ - uint32_t i13 = fastdiv(e, &ct->div_ne2_ne1_ne0); \ + uint32_t i13 = fastdiv(e, &rsh->div_ne2_ne1_ne0); \ uint32_t rem = e - i13 * ne2_ne1_ne0; \ - uint32_t i12 = fastdiv(rem, &ct->div_ne1_ne0); \ + uint32_t i12 = fastdiv(rem, &rsh->div_ne1_ne0); \ uint32_t rem2 = rem - i12 * ne1_ne0; \ - uint32_t i11 = fastdiv(rem2, &ct->div_ne0); \ + uint32_t i11 = fastdiv(rem2, &rsh->div_ne0); \ uint32_t i10 = rem2 - i11 * ne0; \ \ - uint32_t i03 = fastdiv(e, &ct->div_ne02_ne01_ne00); \ + uint32_t i03 = fastdiv(e, &rsh->div_ne02_ne01_ne00); \ uint32_t rem_s = e - i03 * ne02_ne01_ne00; \ - uint32_t i02 = fastdiv(rem_s, &ct->div_ne01_ne00); \ + uint32_t i02 = fastdiv(rem_s, &rsh->div_ne01_ne00); \ uint32_t rem2_s = rem_s - i02 * ne01_ne00; \ - uint32_t i01 = fastdiv(rem2_s, &ct->div_ne00); \ + uint32_t i01 = fastdiv(rem2_s, &rsh->div_ne00); \ uint32_t i00 = rem2_s - i01 * ne00; \ \ - char * dst_ptr = (char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \ - const char * src0_ptr = (const char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \ + dma_addr_t dst_addr = dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \ + dma_addr_t src0_addr = src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \ \ const bool rows_contig = (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \ \ while (e < th_end) { \ - uint32_t run = 1; \ + const uint32_t run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \ if (rows_contig) { \ - run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \ - hvx_copy_uu((uint8_t *) dst_ptr, (const uint8_t *) src0_ptr, run, ELEM_SIZE); \ + dma_cpy_sametype_reshape_contig(dma_q, dst_addr, src0_addr, run * ELEM_SIZE); \ } else { \ - *((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \ + dma_cpy_push_2d_chunked(dma_q, dst_addr, src0_addr, nb0, nb00, ELEM_SIZE, run); \ } \ e += run; \ \ - dst_ptr += run * nb0; \ - i10 += run; \ + dst_addr += run * nb0; \ + i10 += run; \ if (i10 == ne0) { \ i10 = 0; \ if (++i11 == ne1) { \ @@ -184,11 +124,11 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void i13++; \ } \ } \ - dst_ptr = (char *) dst->data + i11*nb1 + i12*nb2 + i13*nb3; \ + dst_addr = dst->data + i11*nb1 + i12*nb2 + i13*nb3; \ } \ \ - src0_ptr += run * nb00; \ - i00 += run; \ + src0_addr += run * nb00; \ + i00 += run; \ if (i00 == ne00) { \ i00 = 0; \ if (++i01 == ne01) { \ @@ -198,371 +138,560 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void i03++; \ } \ } \ - src0_ptr = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ + src0_addr = src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ } \ } \ + dma_queue_flush(dma_q); \ } DEFINE_CPY_RESHAPE(f32, float, 4) DEFINE_CPY_RESHAPE(f16, __fp16, 2) DEFINE_CPY_RESHAPE(i32, int32_t, 4) -static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, void * data) { - struct htp_copy_context * ct = (struct htp_copy_context *) data; - struct htp_ops_context * octx = ct->octx; - cpy_preamble; - - const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = ct->row_start + dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); - if (ir0 >= ir1) return; - - const uint32_t ne02_ne01 = ne02 * ne01; - uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); - uint32_t rem = ir0 - i03 * ne02_ne01; - uint32_t i02 = fastdiv(rem, &ct->div_ne01); - uint32_t i01 = rem - i02 * ne01; - - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - - for (uint32_t r = ir0; r < ir1; r++) { - hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); - hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); - dst_ptr += nb1; - src0_ptr += nb01; - if (++i01 == ne01) { - i01 = 0; - if (++i02 == ne02) { - i02 = 0; - i03++; - } - dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; - src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; +struct cpy_row_cursor { + uint32_t r; + uint32_t i01; + uint32_t i02; + uint32_t i03; +}; + +static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uint32_t ne01, uint32_t ne02) { + c->r += n; + c->i01 += n; + if (c->i01 == ne01) { + c->i01 = 0; + if (++c->i02 == ne02) { + c->i02 = 0; + c->i03++; } } } -static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, void * data) { - struct htp_copy_context * ct = (struct htp_copy_context *) data; - struct htp_ops_context * octx = ct->octx; - cpy_preamble; - - const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = ct->row_start + dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); - if (ir0 >= ir1) return; - - const uint32_t ne02_ne01 = ne02 * ne01; - uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); - uint32_t rem = ir0 - i03 * ne02_ne01; - uint32_t i02 = fastdiv(rem, &ct->div_ne01); - uint32_t i01 = rem - i02 * ne01; - - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - - for (uint32_t r = ir0; r < ir1; r++) { - hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2); - hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00); - dst_ptr += nb1; - src0_ptr += nb01; - if (++i01 == ne01) { - i01 = 0; - if (++i02 == ne02) { - i02 = 0; - i03++; +struct cpy_tile_cursor { + uint32_t ibk; + uint32_t ib0; + uint32_t i2; + uint32_t i3; +}; + +static inline void cpy_tile_cursor_advance(struct cpy_tile_cursor * c, const struct htp_copy_transpose_params * tp) { + if (++c->ibk == tp->nkb) { + c->ibk = 0; + if (++c->ib0 == tp->n0b) { + c->ib0 = 0; + if (++c->i2 == tp->ne[2]) { + c->i2 = 0; + c->i3++; } - dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; - src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; } } } -static void cpy_thread_i32_f32_sameshape(unsigned int nth, unsigned int ith, void * data) { - struct htp_copy_context * ct = (struct htp_copy_context *) data; - struct htp_ops_context * octx = ct->octx; - cpy_preamble; - - const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = ct->row_start + dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); - if (ir0 >= ir1) return; - - const uint32_t ne02_ne01 = ne02 * ne01; - uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); - uint32_t rem = ir0 - i03 * ne02_ne01; - uint32_t i02 = fastdiv(rem, &ct->div_ne01); - uint32_t i01 = rem - i02 * ne01; - - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - - for (uint32_t r = ir0; r < ir1; r++) { - hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); - const float * restrict src_row = (const float *) src0_ptr; - int32_t * restrict dst_row = (int32_t *) dst_ptr; - for (uint32_t i = 0; i < ne00; i++) { - dst_row[i] = (int32_t) src_row[i]; - } - dst_ptr += nb1; - src0_ptr += nb01; - if (++i01 == ne01) { - i01 = 0; - if (++i02 == ne02) { - i02 = 0; - i03++; - } - dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; - src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; +// Gather offsets for element e = l >> log2_m at (e % r, e / r) +static inline HVX_Vector cpy_transpose_offsets(const struct htp_copy_transpose_params * tp) { + const uint32_t r_mask = (1u << tp->log2_r) - 1; + if (tp->elem_size != 2) { + const uint32_t m_mask = (1u << tp->log2_m) - 1; + int32_t offsets[32] __attribute__((aligned(128))); + for (uint32_t l = 0; l < 32; l++) { + const uint32_t e = l >> tp->log2_m; + offsets[l] = (e & r_mask) * tp->in_stride + (e >> tp->log2_r) * tp->elem_size + (l & m_mask) * 4; } + return *(HVX_Vector *) offsets; + } + int16_t offsets[64] __attribute__((aligned(128))); + for (uint32_t l = 0; l < 64; l++) { + offsets[l] = (l & r_mask) * tp->in_stride + (l >> tp->log2_r) * 2; } + return *(HVX_Vector *) offsets; } -static void cpy_thread_f32_i32_sameshape(unsigned int nth, unsigned int ith, void * data) { +static inline void cpy_transpose_push_in(dma_queue * q, const struct htp_copy_transpose_params * tp, dma_addr_t src, + const struct cpy_tile_cursor * c, uint8_t * buf) { + const uint32_t a0 = c->ib0 * tp->blk0; + const uint32_t ak = c->ibk * tp->blkk; + const uint32_t off = a0 * tp->src_nb[0] + ak * tp->elem_size + c->i2 * tp->src_nb[1] + c->i3 * tp->src_nb[2]; + dma_queue_push(q, dma_make_data(buf, src + off), tp->in_stride, tp->src_nb[0], + MIN(tp->blkk, tp->ne[1] - ak) * tp->elem_size, MIN(tp->blk0, tp->ne[0] - a0)); +} + +// Pipelined tile worker: DMA in -> vgather transpose -> DMA out +static void cpy_thread_transpose(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_copy_context * ct = (struct htp_copy_context *) data; struct htp_ops_context * octx = ct->octx; - cpy_preamble; - - const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = ct->row_start + dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); - if (ir0 >= ir1) return; - - const uint32_t ne02_ne01 = ne02 * ne01; - uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); - uint32_t rem = ir0 - i03 * ne02_ne01; - uint32_t i02 = fastdiv(rem, &ct->div_ne01); - uint32_t i01 = rem - i02 * ne01; - - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - - for (uint32_t r = ir0; r < ir1; r++) { - hex_l2fetch(src0_ptr, ne00 * sizeof(int32_t), nb01, 2); - const int32_t * restrict src_row = (const int32_t *) src0_ptr; - float * restrict dst_row = (float *) dst_ptr; - for (uint32_t i = 0; i < ne00; i++) { - dst_row[i] = (float) src_row[i]; - } - dst_ptr += nb1; - src0_ptr += nb01; - if (++i01 == ne01) { - i01 = 0; - if (++i02 == ne02) { - i02 = 0; - i03++; - } - dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; - src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; - } + const struct htp_copy_transpose_params * tp = &ct->kparams->u.transpose; + + const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_hvx; + const uint32_t t1 = MIN(t0 + ct->tiles_per_hvx, ct->tile_start + ct->ntiles); + if (t0 >= t1) { + return; } -} -static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { - cpy_preamble; - *use_dma = false; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + + const dma_addr_t src = octx->src[0]->data; + const dma_addr_t dst = octx->dst->data; + + const uint32_t es = tp->elem_size; + const uint32_t ne0 = tp->ne[0]; + const uint32_t nek = tp->ne[1]; + const uint32_t in_stride = tp->in_stride; + const uint32_t out_stride = tp->out_stride; + + const HVX_Vector offsets = cpy_transpose_offsets(tp); + const uint32_t packed_mu = (ne0 - 1) * in_stride + 127; + const uint32_t packed_step = 128 >> tp->log2_r; + const uint32_t epv = 1u << tp->log2_r; + + uint8_t * in_buf[2]; + uint8_t * out_buf[2]; + in_buf[0] = ct->vtcm_src0 + ith * 2 * tp->in_buf_size; + in_buf[1] = in_buf[0] + tp->in_buf_size; + out_buf[0] = ct->vtcm_dst + ith * 2 * tp->out_buf_size; + out_buf[1] = out_buf[0] + tp->out_buf_size; + + struct cpy_tile_cursor cur; + { + const uint32_t q0 = fastdiv(t0, &tp->div_nkb); + const uint32_t q1 = fastdiv(q0, &tp->div_n0b); + cur.ibk = t0 - q0 * tp->nkb; + cur.ib0 = q0 - q1 * tp->n0b; + cur.i3 = fastdiv(q1, &tp->div_ne2); + cur.i2 = q1 - cur.i3 * tp->ne[2]; + } - const uint32_t total_elems_src = ne00 * ne01 * ne02 * ne03; - const uint32_t total_elems_dst = ne0 * ne1 * ne2 * ne3; - if (total_elems_src == 1 && total_elems_dst == 1) { - if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { - return HTP_STATUS_OK; - } - if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_I32) { - ((int32_t *) dst->data)[0] = (int32_t) (((const float *) src0->data)[0]); - return HTP_STATUS_OK; - } - if (src0->type == HTP_TYPE_I32 && dst->type == HTP_TYPE_F32) { - ((float *) dst->data)[0] = (float) (((const int32_t *) src0->data)[0]); - return HTP_STATUS_OK; - } - if (src0->type == HTP_TYPE_I32 && dst->type == HTP_TYPE_I32) { - ((int32_t *) dst->data)[0] = ((const int32_t *) src0->data)[0]; - return HTP_STATUS_OK; - } - if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_F32) { - ((float *) dst->data)[0] = ((const float *) src0->data)[0]; - return HTP_STATUS_OK; + struct cpy_tile_cursor nxt = cur; + + cpy_transpose_push_in(dma_q, tp, src, &cur, in_buf[0]); + + for (uint32_t t = t0, i = 0; t < t1; t++, i++) { + while (dma_queue_depth(dma_q) > (i > 0 ? 1u : 0u)) { + dma_queue_pop(dma_q); } - if (src0->type == HTP_TYPE_F16 && dst->type == HTP_TYPE_F16) { - ((__fp16 *) dst->data)[0] = ((const __fp16 *) src0->data)[0]; - return HTP_STATUS_OK; + + uint8_t * in = in_buf[i & 1]; + uint8_t * out = out_buf[i & 1]; + + const uint32_t a0 = cur.ib0 * tp->blk0; + const uint32_t ak = cur.ibk * tp->blkk; + const uint32_t n0 = MIN(tp->blk0, ne0 - a0); + const uint32_t nk = MIN(tp->blkk, nek - ak); + + if (t + 1 < t1) { + cpy_tile_cursor_advance(&nxt, tp); + cpy_transpose_push_in(dma_q, tp, src, &nxt, in_buf[(i + 1) & 1]); } - if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_F16) { - ((__fp16 *) dst->data)[0] = (__fp16) (((const float *) src0->data)[0]); - return HTP_STATUS_OK; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t); + if (tp->packed) { + const uint32_t nvec = (nk * ne0 * es + 127) >> 7; + if (es != 2) { + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets); + } + } else { + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets); + } + } + } else { + for (uint32_t rb = 0; rb < n0; rb += epv) { + const uint32_t mu = (MIN(epv, n0 - rb) - 1) * in_stride + es - 1; + uint8_t * d = out + rb * es; + const uint8_t * s = in + rb * in_stride; + if (es != 2) { + #pragma unroll(4) + for (uint32_t ik = 0; ik < nk; ik++) { + Q6_vgather_ARMVw((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * es), mu, offsets); + } + } else { + #pragma unroll(4) + for (uint32_t ik = 0; ik < nk; ik++) { + Q6_vgather_ARMVh((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * 2), mu, offsets); + } + } + } } - if (src0->type == HTP_TYPE_F16 && dst->type == HTP_TYPE_F32) { - ((float *) dst->data)[0] = (float) (((const __fp16 *) src0->data)[0]); - return HTP_STATUS_OK; + hvx_gather_sync(out); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t); + + const uint32_t doff = a0 * es + ak * tp->dst_nb[0] + cur.i2 * tp->dst_nb[1] + cur.i3 * tp->dst_nb[2]; + if (tp->packed) { + const uint32_t n = nk * ne0 * es; + dma_queue_push(dma_q, dma_make_data(dst + doff, out), n, n, n, 1); + } else { + dma_queue_push(dma_q, dma_make_data(dst + doff, out), tp->dst_nb[0], out_stride, n0 * es, nk); } + + cur = nxt; } - struct htp_copy_context ct; - ct.octx = octx; + dma_queue_flush(dma_q); +} - switch (src0->type) { - case HTP_TYPE_F32: ct.src0_type_size = 4; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break; - case HTP_TYPE_F16: ct.src0_type_size = 2; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break; - case HTP_TYPE_I32: ct.src0_type_size = 4; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break; - default: - return HTP_STATUS_NO_SUPPORT; +#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \ +static void cpy_thread_##NAME##_sameshape(const htp_thread_t * th, unsigned int nth, \ + unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + \ + const uint32_t dr = ct->src0_nrows_per_hvx; \ + const uint32_t ir0 = ct->row_start + dr * ith; \ + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ + if (ir0 >= ir1) return; \ + \ + dma_queue * dma_q = th->dma_q; \ + struct htp_thread_trace * tr = th->trace; \ + \ + const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \ + const struct htp_copy_convert_vtcm_layout * L = &ct->layout; \ + const uint32_t src0_buf_size = L->src0_buf_size; \ + const uint32_t dst_buf_size = L->dst_buf_size; \ + const uint32_t src0_row_stride = L->src0_row_stride; \ + const uint32_t dst_row_stride = L->dst_row_stride; \ + const uint32_t blk_rows = L->blk_rows; \ + uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_hvx; \ + uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_hvx; \ + const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \ + const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \ + \ + const uint32_t ne02_ne01 = ne02 * ne01; \ + uint32_t i03 = fastdiv(ir0, &cvt->div_ne02_ne01); \ + uint32_t rem = ir0 - i03 * ne02_ne01; \ + uint32_t i02 = fastdiv(rem, &cvt->div_ne01); \ + uint32_t i01 = rem - i02 * ne01; \ + \ + struct cpy_row_cursor f = { ir0, i01, i02, i03 }; \ + struct cpy_row_cursor c = f; \ + \ + for (uint32_t b = 0; b < 2 && f.r < ir1; ++b) { \ + uint8_t * src_spad = vtcm_src0_base + b * src0_buf_size; \ + uint8_t * dst_spad = vtcm_dst_base + b * dst_buf_size; \ + const uint32_t nr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \ + dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \ + dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \ + dst_row_size, dst_buf_size, dst_row_size, 0); \ + dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \ + src0_row_stride, nb01, src0_row_size, nr); \ + cpy_cursor_advance(&f, nr, ne01, ne02); \ + } \ + \ + for (uint32_t blk = 0; c.r < ir1; ++blk) { \ + const uint32_t nr = MIN(blk_rows, MIN(ne01 - c.i01, ir1 - c.r)); \ + uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \ + uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \ + for (uint32_t r = 0; r < nr; ++r) { \ + CONV_FUNC(dst_spad + r * dst_row_stride, src_spad + r * src0_row_stride, ne00); \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \ + \ + dma_addr_t c_dst_addr = dst->data + c.i01*nb1 + c.i02*nb2 + c.i03*nb3; \ + dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \ + nb1, dst_row_stride, dst_row_size, nr); \ + cpy_cursor_advance(&c, nr, ne01, ne02); \ + \ + if (f.r < ir1) { \ + const uint32_t fnr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \ + dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \ + dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \ + src0_row_stride, nb01, src0_row_size, fnr); \ + cpy_cursor_advance(&f, fnr, ne01, ne02); \ + } \ + } \ + dma_queue_flush(dma_q); \ +} + +DEFINE_CPY_CONVERT_SAMESHAPE(f16_f32, hvx_copy_f16_f32_aa) +DEFINE_CPY_CONVERT_SAMESHAPE(f32_f16, hvx_copy_f32_f16_aa) +DEFINE_CPY_CONVERT_SAMESHAPE(i32_f32, hvx_copy_i32_f32_aa) +DEFINE_CPY_CONVERT_SAMESHAPE(f32_i32, hvx_copy_f32_i32_aa) + +static int cpy_scalar(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (src0->type == dst->type) { + dma_cpy_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, kparams->src0_type_size); + dma_queue_flush(octx->ctx->dma[0]); + return HTP_STATUS_OK; } - switch (dst->type) { - case HTP_TYPE_F32: ct.dst_type_size = 4; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break; - case HTP_TYPE_F16: ct.dst_type_size = 2; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break; - case HTP_TYPE_I32: ct.dst_type_size = 4; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break; - default: + dma_queue * dma_q = octx->ctx->dma[0]; + uint8_t * s_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, 0); + uint8_t * d_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, VLEN); + dma_addr_t s_vtcm = (dma_addr_t)(uintptr_t) s_ptr; + dma_addr_t d_vtcm = (dma_addr_t)(uintptr_t) d_ptr; + const uint32_t s_size = kparams->src0_type_size; + const uint32_t d_size = kparams->dst_type_size; + + dma_queue_push(dma_q, dma_make_data(s_vtcm, src0->data), s_size, s_size, s_size, 1); + dma_queue_pop(dma_q); + + const HVX_Vector v_src = hvx_vmem(s_ptr); + HVX_Vector v_dst; + + if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_I32) { + v_dst = Q6_Vw_equals_Vsf(v_src); + } else if (src0->type == HTP_TYPE_I32 && dst->type == HTP_TYPE_F32) { + v_dst = Q6_Vsf_equals_Vw(v_src); + } else if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_F16) { + v_dst = hvx_vec_f32_to_f16(v_src, v_src); + } else if (src0->type == HTP_TYPE_F16 && dst->type == HTP_TYPE_F32) { + v_dst = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src)); + } else { return HTP_STATUS_NO_SUPPORT; } - const bool sametype = (src0->type == dst->type); - const bool transposed = (nb00 > nb01) || (nb0 > nb1) || - (nb00 != ct.src0_type_size) || (nb0 != ct.dst_type_size) || - (nb01 < ne00 * ct.src0_type_size) || (nb1 < ne0 * ct.dst_type_size); - const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3); + hvx_vmem(d_ptr) = v_dst; + + dma_queue_push(dma_q, dma_make_data(dst->data, d_vtcm), d_size, d_size, d_size, 1); + dma_queue_flush(dma_q); + return HTP_STATUS_OK; +} - const uint32_t n_threads = octx->n_threads; +static int cpy_1d_contig(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; - const bool src_is_contiguous = htp_tensor_is_contiguous(src0, ct.src0_type_size); - const bool dst_is_contiguous = htp_tensor_is_contiguous(dst, ct.dst_type_size); + uint32_t elem_start = 0; + uint32_t nelem = kparams->total_elems; - if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { - if (!sametype) { - return HTP_STATUS_NO_SUPPORT; - } - if (!sameshape && !(src_is_contiguous && dst_is_contiguous && octx->ctx->mdev.count <= 1)) { - return HTP_STATUS_NO_SUPPORT; - } + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->total_elems, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelem = range.count; } - if (sameshape) { - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t row_size = ne00 * ct.dst_type_size; + if (nelem > 0) { + dma_queue * q = octx->ctx->dma[0]; + const uint32_t type_size = kparams->src0_type_size; + dma_addr_t dst_addr = dst->data + elem_start * type_size; + dma_addr_t src0_addr = src0->data + elem_start * type_size; + dma_cpy_sametype_reshape_contig(q, dst_addr, src0_addr, nelem * type_size); + dma_queue_flush(q); + } - ct.div_ne01 = init_fastdiv_values(ne01); - ct.div_ne02_ne01 = init_fastdiv_values(ne02 * ne01); + return HTP_STATUS_OK; +} - uint32_t row_start = 0; - uint32_t nrows = total_rows; +static int cpy_sameshape_sametype(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; - if (octx->ctx->mdev.count > 1) { - const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; - const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous; - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, can_split ? rows_per_chunk : 0, - octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); - row_start = range.start; - nrows = range.count; - } + uint32_t row_start = 0; + uint32_t nrows = kparams->total_rows; - if (nrows == 0) { - return HTP_STATUS_OK; - } + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } - ct.row_start = row_start; - ct.nrows = nrows; - ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + if (nrows > 0) { + dma_queue * q = octx->ctx->dma[0]; + dma_cpy_sametype_sameshape_range(q, dst, src0, kparams->src0_type_size, row_start, nrows); + dma_queue_flush(q); + } - if (sametype && (octx->ctx->mdev.count <= 1 || htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { - if (octx->ctx->mdev.idx == 0) { - *use_dma = true; - cpy_dma_sametype_sameshape(octx->ctx->dma[0], dst, src0, ct.src0_type_size); - dma_queue_flush(octx->ctx->dma[0]); - } - } else { - work_queue_func_t copy_fun = NULL; - if (sametype) { - switch (src0->type) { - case HTP_TYPE_F32: copy_fun = cpy_thread_f32_sameshape; break; - case HTP_TYPE_F16: copy_fun = cpy_thread_f16_sameshape; break; - case HTP_TYPE_I32: copy_fun = cpy_thread_i32_sameshape; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } else if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) { - copy_fun = cpy_thread_f16_f32_sameshape; - } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) { - copy_fun = cpy_thread_f32_f16_sameshape; - } else if (dst->type == HTP_TYPE_I32 && src0->type == HTP_TYPE_F32) { - copy_fun = cpy_thread_i32_f32_sameshape; - } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_I32) { - copy_fun = cpy_thread_f32_i32_sameshape; - } else { - return HTP_STATUS_NO_SUPPORT; - } - work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); - } - } else if (sametype) { - const uint32_t total_elems = ne0 * ne1 * ne2 * ne3; - const uint32_t elems_per_line = (ct.dst_type_size == 4) ? 32 : 64; - - if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) { - *use_dma = true; - cpy_dma_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, total_elems * ct.dst_type_size); - dma_queue_flush(octx->ctx->dma[0]); - return HTP_STATUS_OK; - } + return HTP_STATUS_OK; +} - ct.div_ne0 = init_fastdiv_values(ne0); - ct.div_ne1_ne0 = init_fastdiv_values(ne1 * ne0); - ct.div_ne2_ne1_ne0 = init_fastdiv_values(ne2 * ne1 * ne0); - ct.div_ne00 = init_fastdiv_values(ne00); - ct.div_ne01_ne00 = init_fastdiv_values(ne01 * ne00); - ct.div_ne02_ne01_ne00 = init_fastdiv_values(ne02 * ne01 * ne00); +static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; - uint32_t elem_start = 0; - uint32_t nelem = total_elems; + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } - if (octx->ctx->mdev.count > 1) { - const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous; - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elems, can_split ? elems_per_line : 0, - octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); - elem_start = range.start; - nelem = range.count; - } + uint32_t row_start = 0; + uint32_t nrows = kparams->total_rows; - if (nelem == 0) { - return HTP_STATUS_OK; - } + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } - ct.elem_start = elem_start; - ct.nelem = nelem; - ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div); + if (nrows == 0) { + return HTP_STATUS_OK; + } - work_queue_func_t copy_fun = NULL; - switch (src0->type) { - case HTP_TYPE_F32: copy_fun = cpy_thread_f32_reshape; break; - case HTP_TYPE_F16: copy_fun = cpy_thread_f16_reshape; break; - case HTP_TYPE_I32: copy_fun = cpy_thread_i32_reshape; break; - default: return HTP_STATUS_NO_SUPPORT; - } - work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const uint32_t n_hvx = octx->n_hvx; + const struct htp_copy_convert_params * cvt = &kparams->u.convert; + + struct htp_copy_context ct; + htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_hvx); + + ct.octx = octx; + ct.kparams = kparams; + ct.row_start = row_start; + ct.nrows = nrows; + ct.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + ct.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_src0); + ct.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_dst); + + work_queue_func_t copy_fun = NULL; + if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) { + copy_fun = cpy_thread_f16_f32_sameshape; + } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) { + copy_fun = cpy_thread_f32_f16_sameshape; + } else if (dst->type == HTP_TYPE_I32 && src0->type == HTP_TYPE_F32) { + copy_fun = cpy_thread_i32_f32_sameshape; + } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_I32) { + copy_fun = cpy_thread_f32_i32_sameshape; } else { return HTP_STATUS_NO_SUPPORT; } + work_queue_run(octx->ctx->hvx_queue, copy_fun, &ct, n_hvx); + return HTP_STATUS_OK; +} + +static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + uint32_t elem_start = 0; + uint32_t nelem = kparams->total_elems; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->total_elems, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelem = range.count; + } + + if (nelem == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_hvx = octx->n_hvx; + + struct htp_copy_context ct; + ct.octx = octx; + ct.kparams = kparams; + ct.elem_start = elem_start; + ct.nelem = nelem; + ct.elem_per_hvx = fastdiv(nelem + n_hvx - 1, &octx->n_hvx_div); + + work_queue_func_t copy_fun = NULL; + switch (src0->type) { + case HTP_TYPE_F32: copy_fun = cpy_thread_f32_reshape; break; + case HTP_TYPE_F16: copy_fun = cpy_thread_f16_reshape; break; + case HTP_TYPE_I32: copy_fun = cpy_thread_i32_reshape; break; + default: return HTP_STATUS_NO_SUPPORT; + } + + work_queue_run(octx->ctx->hvx_queue, copy_fun, &ct, n_hvx); + return HTP_STATUS_OK; +} + +static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const struct htp_copy_transpose_params * tp = &kparams->u.transpose; + + uint32_t tile_start = 0; + uint32_t ntiles = tp->total_tiles; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + tp->total_tiles, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + tile_start = range.start; + ntiles = range.count; + } + + if (ntiles == 0) { + return HTP_STATUS_OK; + } + + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const uint32_t n_hvx = octx->n_hvx; + + struct htp_copy_context ct; + ct.octx = octx; + ct.kparams = kparams; + ct.tile_start = tile_start; + ct.ntiles = ntiles; + ct.tiles_per_hvx = fastdiv(ntiles + n_hvx - 1, &octx->n_hvx_div); + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + ct.vtcm_src0 = vtcm_base; + ct.vtcm_dst = vtcm_base + (size_t) n_hvx * 2 * tp->in_buf_size; + + work_queue_run(octx->ctx->hvx_queue, cpy_thread_transpose, &ct, n_hvx); return HTP_STATUS_OK; } int op_cpy(struct htp_ops_context * octx) { - bool use_dma = false; - int status = exec_cpy(octx, &use_dma); + const struct htp_copy_kernel_params * kparams = (const struct htp_copy_kernel_params *) octx->kernel_params; + int status = HTP_STATUS_OK; + + switch (kparams->kernel_type) { + case HTP_COPY_KERNEL_SCALAR: + status = cpy_scalar(octx, kparams); + break; + case HTP_COPY_KERNEL_1D_CONTIG: + status = cpy_1d_contig(octx, kparams); + break; + case HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE: + status = cpy_sameshape_sametype(octx, kparams); + break; + case HTP_COPY_KERNEL_SAMESHAPE_CONVERT: + status = cpy_sameshape_convert(octx, kparams); + break; + case HTP_COPY_KERNEL_RESHAPE: + status = cpy_reshape(octx, kparams); + break; + case HTP_COPY_KERNEL_TRANSPOSE: + status = cpy_transpose(octx, kparams); + break; + case HTP_COPY_KERNEL_GATHER_ROWS: + status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_hvx); + break; + default: + status = HTP_STATUS_NO_SUPPORT; + break; + } htp_ops_context_set_status(octx, status); if (octx->op == HTP_OP_CPY_FENCE) { - if (!use_dma) { - htp_flush_dirty_ranges(octx->ctx); + if (octx->ctx->mdev.count > 1) { + htp_mdev_group_barrier(octx); } - htp_mdev_group_barrier(octx); - if (octx->ctx->mdev.idx == 0) { const struct htp_tensor * sync = octx->src[1]; - if (htp_tensor_is_extended(sync)) { - return HTP_STATUS_NO_SUPPORT; - } const uint32_t seq = (uint32_t) octx->op_params[0]; atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; htp_fence_write(sync_fence, seq, octx->status); diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h new file mode 100644 index 000000000..1ea50ca1d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h @@ -0,0 +1,220 @@ +#ifndef HTP_CPY_OPS_H +#define HTP_CPY_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include "gather-rows.h" +#include <stdint.h> + +enum htp_copy_kernel_type { + HTP_COPY_KERNEL_UNSUPPORTED = 0, + HTP_COPY_KERNEL_1D_CONTIG = 1, + HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE = 2, + HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3, + HTP_COPY_KERNEL_RESHAPE = 4, + HTP_COPY_KERNEL_SCALAR = 5, + HTP_COPY_KERNEL_GATHER_ROWS = 6, + HTP_COPY_KERNEL_TRANSPOSE = 7, +}; + +struct htp_copy_convert_params { + uint32_t blk_rows; // rows moved per DMA descriptor + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02_ne01; +}; + +struct htp_copy_reshape_params { + struct fastdiv_values div_ne0; + struct fastdiv_values div_ne1_ne0; + struct fastdiv_values div_ne2_ne1_ne0; + struct fastdiv_values div_ne00; + struct fastdiv_values div_ne01_ne00; + struct fastdiv_values div_ne02_ne01_ne00; +}; + +// Batched 2D transpose of elem_size elements (dim 0 dense in dst, dim 1 dense in src) +struct htp_copy_transpose_params { + uint32_t ne[3]; // dims 0..2 + uint32_t src_nb[3]; // src strides for dims 0, 2, 3 + uint32_t dst_nb[3]; // dst strides for dims 1, 2, 3 + struct fastdiv_values div_nkb; + struct fastdiv_values div_n0b; + struct fastdiv_values div_ne2; + uint32_t blk0; // dim-0 elements per tile + uint32_t blkk; // dim-1 elements per tile + uint32_t n0b; // tiles along dim 0 + uint32_t nkb; // tiles along dim 1 + uint32_t in_stride; // VTCM row pitch of staged src tile + uint32_t out_stride; // VTCM row pitch of transposed tile + uint32_t in_buf_size; + uint32_t out_buf_size; + uint32_t total_tiles; + uint8_t elem_size; + uint8_t packed; // gather straight to dst order + uint8_t log2_m; // log2(gather lanes per element) + uint8_t log2_r; // log2(dim-0 elements per vector) +}; + +struct htp_copy_kernel_params { + uint8_t kernel_type; + uint8_t src0_type_size; + uint8_t dst_type_size; + uint8_t n_hvx; + + uint32_t total_elems; + uint32_t total_rows; + uint32_t vtcm_size; + + union { + struct htp_copy_convert_params convert; + struct htp_copy_reshape_params reshape; + struct htp_gather_rows_params gather; + struct htp_copy_transpose_params transpose; + } u; +}; + +// Size of one VTCM staging buffer: big enough that a DMA descriptor moves many short +// rows at once. The copy kernels keep four per thread (two in flight each way), about +// 1 MB of VTCM with 8 threads. +#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024) + +struct htp_copy_convert_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + uint32_t src0_row_stride; + uint32_t dst_row_stride; + uint32_t blk_rows; + uint32_t src0_buf_size; + uint32_t dst_buf_size; + uint32_t spad0_size_per_hvx; + uint32_t spad1_size_per_hvx; +}; + +static inline void htp_copy_convert_vtcm_layout_build( + struct htp_copy_convert_vtcm_layout * L, + uint32_t ne00, + uint32_t src_type_size, + uint32_t dst_type_size, + uint32_t blk_rows, + uint32_t n_hvx) { + + const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256); + const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); + + L->src0_row_stride = src0_row_stride; + L->dst_row_stride = dst_row_stride; + L->blk_rows = blk_rows; + L->src0_buf_size = blk_rows * src0_row_stride; + L->dst_buf_size = blk_rows * dst_row_stride; + L->spad0_size_per_hvx = 2 * L->src0_buf_size; + L->spad1_size_per_hvx = 2 * L->dst_buf_size; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_hvx * L->spad0_size_per_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_hvx * L->spad1_size_per_hvx); + L->total_bytes = off; +} + +// ne01 bounds a block: rows of one block share the dim-1 stride. +static inline void htp_copy_convert_solve_layout( + struct htp_copy_convert_vtcm_layout * layout, + uint32_t ne00, + uint32_t ne01, + uint32_t src_type_size, + uint32_t dst_type_size, + uint32_t n_hvx) { + + const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256); + const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); + + const uint32_t max_row = src0_row_stride > dst_row_stride ? src0_row_stride : dst_row_stride; + uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row; + if (blk_rows > ne01) { + blk_rows = ne01; + } + if (blk_rows == 0) { + blk_rows = 1; + } + + htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_hvx); +} + +#define HTP_COPY_TRANSPOSE_BLK_BYTES (64 * 1024) + +static inline bool htp_copy_transpose_layout_build(struct htp_copy_transpose_params * tp, uint32_t blk_bytes) { + const uint32_t ne0 = tp->ne[0]; + const uint32_t nek = tp->ne[1]; + const uint32_t es = tp->elem_size; + const uint32_t epv = 128 / es; + + uint32_t blk0; + uint32_t blkk; + if (tp->packed) { + // Packed: vector holds epv / ne0 whole dst rows + blk0 = ne0; + blkk = blk_bytes / (ne0 * es); + if (es == 2 && blkk > 16384 / ne0 - 64) { + blkk = 16384 / ne0 - 64; + } + if (blkk > nek) { + blkk = nek; + } + if (blkk == 0) { + return false; + } + } else if (nek <= epv) { + // Stage short contiguous dim and expand dim 0 + blkk = nek; + blk0 = blk_bytes / (nek * es); + blk0 -= blk0 % epv; + if (blk0 > hex_round_up(ne0, epv)) { + blk0 = hex_round_up(ne0, epv); + } + if (blk0 < epv) { + return false; + } + } else { + blk0 = hex_round_up(ne0, epv); + if (blk0 > 256) { + blk0 = 256; + } + blkk = blk_bytes / (blk0 * es); + blkk -= blkk % epv; + // Cap f16 blkk to 256 for 16-bit gather offsets + const uint32_t max_blkk = (es == 2 ? 512 : 4096) / es; + if (blkk > max_blkk) { + blkk = max_blkk; + } + if (blkk > hex_round_up(nek, epv)) { + blkk = hex_round_up(nek, epv); + } + if (blkk < epv) { + return false; + } + } + + tp->blk0 = blk0; + tp->blkk = blkk; + tp->n0b = (ne0 + blk0 - 1) / blk0; + tp->nkb = (nek + blkk - 1) / blkk; + // Pad row pitch by 1 element to avoid VTCM bank conflicts + const uint32_t pad = (es == 2) ? 2 : 4; + tp->in_stride = blkk * es + pad; + tp->out_stride = blk0 * es; + // Extra vector for packed gather overhang + tp->in_buf_size = hex_round_up(blk0 * tp->in_stride + (tp->packed ? 128 : 0), 128); + tp->out_buf_size = hex_round_up(blkk * tp->out_stride, 128); + tp->log2_m = es == 2 ? 0 : __builtin_ctz(es) - 2; + tp->log2_r = __builtin_ctz(tp->packed ? ne0 : epv); + return true; +} + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_CPY_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c index eaab7d7e5..292a05455 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c @@ -8,13 +8,15 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" +#include "htp-vtcm.h" #include "hvx-types.h" #include "hvx-utils.h" #include "dma-queue.h" +#include "cumsum-ops.h" #define htp_cumsum_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ @@ -46,7 +48,7 @@ struct htp_cumsum_context { size_t dst_row_size; size_t src_row_size_aligned; size_t dst_row_size_aligned; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint32_t total_rows; uint32_t row_start; }; @@ -55,7 +57,7 @@ struct htp_cumsum_context { struct htp_cumsum_context * cctx = (struct htp_cumsum_context *) data; \ struct htp_ops_context * octx = cctx->octx; \ htp_cumsum_tensors_preamble; \ - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; // --------------------------------------------------------------------------- // HVX prefix scan helpers @@ -89,8 +91,8 @@ static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) { } static inline void hvx_cumsum_row_f32(const float * restrict src, float * restrict dst, uint32_t n) { - const uint32_t nvec = n / VLEN_FP32; - const uint32_t nloe = n % VLEN_FP32; + const uint32_t nvec = n >> 5; + const uint32_t nloe = n & (VLEN_FP32 - 1); HVX_Vector carry = Q6_V_vsplat_R(0); @@ -116,11 +118,11 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri // Per thread worker: Double-buffered DMA // --------------------------------------------------------------------------- -static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { +static void cumsum_thread_f32_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_hvx, cctx->row_start + cctx->total_rows); if (ir0 >= ir1) { return; @@ -149,7 +151,7 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat src_row_size_aligned, src_row_size, src_row_size, 1); } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = ir0; ir < ir1; ir++) { float * dst_spad_row = (float *) dma_queue_pop(dma_q).src; @@ -182,16 +184,16 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat // Per thread worker: Direct HVX (no DMA) // --------------------------------------------------------------------------- -static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void cumsum_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_hvx, cctx->row_start + cctx->total_rows); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ir++) { @@ -210,16 +212,19 @@ static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { int op_cumsum_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; + const struct htp_cumsum_kernel_params * kparams = + (const struct htp_cumsum_kernel_params *) octx->kernel_params; - const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + const uint32_t total_rows = kparams->total_rows; + const size_t src_row_size = src0->nb[1]; + const size_t dst_row_size = dst->nb[1]; uint32_t row_start = 0; uint32_t nrows = total_rows; if (octx->ctx->mdev.count > 1) { uint32_t rows_per_chunk = 0; - htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk); const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); row_start = range.start; nrows = range.count; @@ -229,43 +234,45 @@ int op_cumsum_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - - const size_t src_row_size = src0->nb[1]; - const size_t dst_row_size = dst->nb[1]; - const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); - - // 2 ping-pong buffers per thread for src and dst - const size_t spad_per_thread = 2 * (src_row_size_aligned + dst_row_size_aligned); + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } - octx->src0_spad.size_per_thread = src_row_size_aligned * 2; - octx->dst_spad.size_per_thread = dst_row_size_aligned * 2; + const uint32_t n_hvx = octx->n_hvx; - octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread; - octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread; + if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { + return HTP_STATUS_NO_SUPPORT; + } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL; + struct htp_cumsum_vtcm_layout layout = { 0 }; + if (kparams->use_dma) { + htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_hvx); + + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; + octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); + octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); + octx->src0_spad.src = NULL; + octx->dst_spad.src = NULL; + } struct htp_cumsum_context cctx = { .octx = octx, .src_row_size = src_row_size, .dst_row_size = dst_row_size, - .src_row_size_aligned = src_row_size_aligned, - .dst_row_size_aligned = dst_row_size_aligned, - .rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src_row_size_aligned = layout.src_row_size_aligned, + .dst_row_size_aligned = layout.dst_row_size_aligned, + .rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_rows = nrows, .row_start = row_start, }; - if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { - return HTP_STATUS_NO_SUPPORT; - } - work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads); + if (kparams->use_dma) { + work_queue_run(octx->ctx->hvx_queue, cumsum_thread_f32_dma, &cctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, cumsum_thread_f32, &cctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.h b/ggml/src/ggml-hexagon/htp/cumsum-ops.h new file mode 100644 index 000000000..48f4053ed --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.h @@ -0,0 +1,49 @@ +#ifndef HTP_CUMSUM_OPS_H +#define HTP_CUMSUM_OPS_H + +#include "hex-common.h" +#include "htp-vtcm.h" +#include <stdint.h> + +struct htp_cumsum_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + size_t src_row_size_aligned; + size_t dst_row_size_aligned; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; +}; + +static inline void htp_cumsum_vtcm_layout_build( + struct htp_cumsum_vtcm_layout * L, + size_t src_row_size, + size_t dst_row_size, + uint32_t n_hvx +) { + L->src_row_size_aligned = hex_round_up(src_row_size, 128); + L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); + L->src0_bytes_per_hvx = 2 * L->src_row_size_aligned; + L->dst_bytes_per_hvx = 2 * L->dst_row_size_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + + L->total_bytes = off; +} + +struct htp_cumsum_kernel_params { + uint32_t n_hvx; + uint32_t total_rows; + uint32_t use_dma; + uint32_t vtcm_size; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_CUMSUM_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c index 162214d3e..ea1a50316 100644 --- a/ggml/src/ggml-hexagon/htp/diag-ops.c +++ b/ggml/src/ggml-hexagon/htp/diag-ops.c @@ -6,14 +6,16 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" +#include "htp-vtcm.h" #include "hvx-types.h" #include "hex-utils.h" #include "hvx-copy.h" #include "dma-queue.h" +#include "diag-ops.h" #define htp_diag_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ @@ -33,11 +35,12 @@ struct htp_diag_context { struct htp_ops_context * octx; + const struct htp_diag_kernel_params * kparams; size_t src_batch_size; size_t dst_row_size; size_t src_batch_size_aligned; size_t dst_row_size_aligned; - uint32_t batches_per_thread; + uint32_t batches_per_hvx; uint32_t total_batches; uint32_t batch_start; }; @@ -57,12 +60,12 @@ static inline void hvx_diag_row_f32(const float * restrict src, float * restrict // Per thread worker: DMA src fetch, compute in VTCM, DMA dst writeback // --------------------------------------------------------------------------- -static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { +static void diag_thread_f32_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; - const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_hvx * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_hvx, dctx->batch_start + dctx->total_batches); if (ib0 >= ib1) { return; @@ -80,11 +83,11 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned); uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ib = ib0; ib < ib1; ib++) { - const uint32_t i3 = ib / ne02; - const uint32_t i2 = ib % ne02; + const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02); + const uint32_t i2 = ib - i3 * ne02; const dma_addr_t src_batch = src_data + i3 * nb03 + i2 * nb02; @@ -121,21 +124,21 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) // Per thread worker: Direct HVX (no DMA) // --------------------------------------------------------------------------- -static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void diag_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_hvx * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_hvx, dctx->batch_start + dctx->total_batches); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); for (uint32_t ib = ib0; ib < ib1; ib++) { - const uint32_t i3 = ib / ne02; - const uint32_t i2 = ib % ne02; + const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02); + const uint32_t i2 = ib - i3 * ne02; const float * restrict src_batch = (const float *)(src_data + i3 * nb03 + i2 * nb02); @@ -155,29 +158,17 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { int op_diag_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; + const struct htp_diag_kernel_params * kparams = + (const struct htp_diag_kernel_params *) octx->kernel_params; - const uint32_t total_batches = src0->ne[2] * src0->ne[3]; - const size_t dst_batch_size = dst->ne[1] * dst->nb[1]; + const uint32_t total_batches = kparams->total_batches; uint32_t batch_start = 0; uint32_t nbatches = total_batches; if (octx->ctx->mdev.count > 1) { - bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !htp_tensor_is_permuted(dst); - uint32_t batches_per_chunk = 1; - if (can_split) { - if (dst->ne[2] > 1 && (dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && - (dst->ne[3] <= 1 || (dst->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) { - batches_per_chunk = 1; - } else if (dst->nb[2] == dst_batch_size && - (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) { - batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1; - } else { - can_split = false; - } - } - - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + const bool can_split = (kparams->can_split != 0) && htp_tensor_mdev_data_aligned(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); batch_start = range.start; nbatches = range.count; } @@ -186,43 +177,49 @@ int op_diag_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - - const size_t src_batch_size = src0->ne[0] * sizeof(float); - const size_t dst_row_size = dst->ne[0] * sizeof(float); - const size_t src_batch_size_aligned = hex_round_up(src_batch_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); - - // 1 src buffer + 1 dst row buffer per thread - const size_t spad_per_thread = src_batch_size_aligned + dst_row_size_aligned; + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nbatches))) { + return HTP_STATUS_INVAL_PARAMS; + } - octx->src0_spad.size_per_thread = src_batch_size_aligned; - octx->dst_spad.size_per_thread = dst_row_size_aligned; + const uint32_t n_hvx = octx->n_hvx; - octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread; - octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread; + if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { + return HTP_STATUS_NO_SUPPORT; + } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL; + const size_t src_batch_size = src0->ne[0] * sizeof(float); + const size_t dst_row_size = dst->ne[0] * sizeof(float); + + struct htp_diag_vtcm_layout layout = { 0 }; + if (kparams->use_dma) { + htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_hvx); + + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; + octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); + octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); + octx->src0_spad.src = NULL; + octx->dst_spad.src = NULL; + } struct htp_diag_context dctx = { .octx = octx, + .kparams = kparams, .src_batch_size = src_batch_size, .dst_row_size = dst_row_size, - .src_batch_size_aligned = src_batch_size_aligned, - .dst_row_size_aligned = dst_row_size_aligned, - .batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div), + .src_batch_size_aligned = layout.src_batch_size_aligned, + .dst_row_size_aligned = layout.dst_row_size_aligned, + .batches_per_hvx = fastdiv(nbatches + n_hvx - 1, &octx->n_hvx_div), .total_batches = nbatches, .batch_start = batch_start, }; - if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { - return HTP_STATUS_NO_SUPPORT; - } - work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads); + if (kparams->use_dma) { + work_queue_run(octx->ctx->hvx_queue, diag_thread_f32_dma, &dctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, diag_thread_f32, &dctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.h b/ggml/src/ggml-hexagon/htp/diag-ops.h new file mode 100644 index 000000000..609a061c5 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/diag-ops.h @@ -0,0 +1,54 @@ +#ifndef HTP_DIAG_OPS_H +#define HTP_DIAG_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include <stdint.h> + +struct htp_diag_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + size_t src_batch_size_aligned; + size_t dst_row_size_aligned; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; +}; + +static inline void htp_diag_vtcm_layout_build( + struct htp_diag_vtcm_layout * L, + size_t src_batch_size, + size_t dst_row_size, + uint32_t n_hvx +) { + L->src_batch_size_aligned = hex_round_up(src_batch_size, 128); + L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); + L->src0_bytes_per_hvx = L->src_batch_size_aligned; + L->dst_bytes_per_hvx = L->dst_row_size_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + + L->total_bytes = off; +} + +struct htp_diag_kernel_params { + uint32_t n_hvx; + uint32_t total_batches; + uint32_t use_dma; + uint32_t can_split; + uint32_t batches_per_chunk; + uint32_t vtcm_size; + + struct fastdiv_values div_ne02; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_DIAG_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h b/ggml/src/ggml-hexagon/htp/dma-copy.h similarity index 58% rename from ggml/src/ggml-hexagon/htp/hex-cpy-dma.h rename to ggml/src/ggml-hexagon/htp/dma-copy.h index c87f495e2..b2c91a13c 100644 --- a/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h +++ b/ggml/src/ggml-hexagon/htp/dma-copy.h @@ -1,9 +1,9 @@ -#ifndef HEX_CPY_DMA_H -#define HEX_CPY_DMA_H +#ifndef HTP_DMA_COPY_H +#define HTP_DMA_COPY_H // DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides. // Used by CPY for the copy itself and by CONCAT, which is two such copies into -// two views of its destination. Every helper only pushes descriptors; the +// two views of its destination. Every helper only pushes descriptors; the // caller flushes the queue when it needs the data. #include "dma-queue.h" @@ -14,7 +14,7 @@ #include <stdint.h> // Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each. -static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q, +static inline void dma_cpy_sametype_reshape_contig(dma_queue * dma_q, dma_addr_t dst, dma_addr_t src0, uint32_t total_bytes) { @@ -36,7 +36,7 @@ static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q, } // One 2d transfer, split at the 16-bit nrows field. -static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q, +static inline void dma_cpy_push_2d_chunked(dma_queue * dma_q, dma_addr_t dst, dma_addr_t src, size_t dst_stride, @@ -59,12 +59,18 @@ static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q, } } -// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on -// both sides (nb[0] == elem_size). -static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q, - const struct htp_tensor * dst, - const struct htp_tensor * src0, - uint32_t elem_size) { +// Copy a range of rows [row_start, row_start + nrows) from src0 into dst: +// same type, same ne[], any nb[] above dim 0, dim 0 dense on both sides (nb[0] == elem_size). +static inline void dma_cpy_sametype_sameshape_range(dma_queue * dma_q, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t elem_size, + uint32_t row_start, + uint32_t nrows) { + if (nrows == 0) { + return; + } + const uint32_t ne00 = src0->ne[0]; const uint32_t ne01 = src0->ne[1]; const uint32_t ne02 = src0->ne[2]; @@ -85,13 +91,16 @@ static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q, const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size); if (contiguous) { - cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03); + dma_cpy_sametype_reshape_contig(dma_q, + dst->data + (dma_addr_t) row_start * ne00 * elem_size, + src0->data + (dma_addr_t) row_start * ne00 * elem_size, + nrows * ne00 * elem_size); return; } // The single-descriptor path flattens (i01,i02,i03) into one row index, so every // row must sit at a constant stride: nb01 on the source, nb1 on the destination. - // Walk the outer dims and require each to continue that progression. A dim of + // Walk the outer dims and require each to continue that progression. A dim of // extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be // used to justify the next dim's stride, which is what comparing nb03 against // ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view @@ -109,18 +118,52 @@ static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q, } if (contiguous_outer) { - uint32_t total_rows = ne01 * ne02 * ne03; - cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows); + dma_cpy_push_2d_chunked(dma_q, + dst->data + (dma_addr_t) row_start * nb1, + src0->data + (dma_addr_t) row_start * nb01, + nb1, nb01, ne00 * elem_size, nrows); return; } - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3; - dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03; - cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01); + const uint32_t ne02_ne01 = ne02 * ne01; + uint32_t i03 = row_start / ne02_ne01; + uint32_t rem = row_start - i03 * ne02_ne01; + uint32_t i02 = rem / ne01; + uint32_t i01 = rem - i02 * ne01; + + dma_addr_t cur_dst = dst->data + (dma_addr_t) i01 * nb1 + (dma_addr_t) i02 * nb2 + (dma_addr_t) i03 * nb3; + dma_addr_t cur_src0 = src0->data + (dma_addr_t) i01 * nb01 + (dma_addr_t) i02 * nb02 + (dma_addr_t) i03 * nb03; + + uint32_t r = row_start; + const uint32_t row_end = row_start + nrows; + while (r < row_end) { + uint32_t cur_rows = MIN(row_end - r, ne01 - i01); + dma_cpy_push_2d_chunked(dma_q, cur_dst, cur_src0, nb1, nb01, ne00 * elem_size, cur_rows); + r += cur_rows; + i01 += cur_rows; + if (i01 == ne01) { + i01 = 0; + if (++i02 == ne02) { + i02 = 0; + i03++; + } + cur_dst = dst->data + (dma_addr_t) i02 * nb2 + (dma_addr_t) i03 * nb3; + cur_src0 = src0->data + (dma_addr_t) i02 * nb02 + (dma_addr_t) i03 * nb03; + } else { + cur_dst += cur_rows * nb1; + cur_src0 += cur_rows * nb01; } } } -#endif /* HEX_CPY_DMA_H */ +// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on +// both sides (nb[0] == elem_size). +static inline void dma_cpy_sametype_sameshape(dma_queue * dma_q, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t elem_size) { + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + dma_cpy_sametype_sameshape_range(dma_q, dst, src0, elem_size, 0, total_rows); +} + +#endif /* HTP_DMA_COPY_H */ diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.c b/ggml/src/ggml-hexagon/htp/dma-queue.c index 464e4b849..11d52a919 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.c +++ b/ggml/src/ggml-hexagon/htp/dma-queue.c @@ -141,43 +141,9 @@ bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride dma_ring_flush(r1); r0->tail = r1->tail; - return dma_ring_push_single_2d(r0, ddata, 0, 0, 0, /*nrows=*/ 0); -} - -bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total) { - dma_ring * r0 = q->ring0; - dma_ring * r1 = q->ring1; - - if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) { - return false; - } - - r1->tail = r0->tail; - - size_t rem_bytes = total; - dma_addr_t cur_dst = ddata.dst; - dma_addr_t cur_src = ddata.src; - - while (rem_bytes > 0) { - const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE); - dma_data cur_data = dma_make_data(cur_dst, cur_src); - if (!dma_ring_push_single_1d(r1, cur_data, cur_bytes)) { - dma_ring_flush(r1); - dma_ring_push_single_1d(r1, cur_data, cur_bytes); - } - cur_dst += cur_bytes; - cur_src += cur_bytes; - rem_bytes -= cur_bytes; - } - - dma_ring_flush(r1); - r0->tail = r1->tail; - return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0); } -#if __HVX_ARCH__ < 75 - bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { dma_ring * r0 = q->ring0; dma_ring * r1 = q->ring1; @@ -188,19 +154,28 @@ bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride r1->tail = r0->tail; - size_t rem_rows = nrows; - dma_addr_t cur_dst = ddata.dst; - dma_addr_t cur_src = ddata.src; - - while (rem_rows > 0) { - dma_data cur_data = dma_make_data(cur_dst, cur_src); - if (!dma_ring_push_single_1d(r1, cur_data, row_size)) { - dma_ring_flush(r1); - dma_ring_push_single_1d(r1, cur_data, row_size); + dma_addr_t row_dst = ddata.dst; + dma_addr_t row_src = ddata.src; + + for (size_t i = 0; i < nrows; i++) { + size_t rem_bytes = row_size; + dma_addr_t cur_dst = row_dst; + dma_addr_t cur_src = row_src; + + while (rem_bytes > 0) { + const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE); + dma_data cur_data = dma_make_data(cur_dst, cur_src); + if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) { + dma_ring_flush(r1); + dma_ring_push_single_contig(r1, cur_data, cur_bytes); + } + cur_dst += cur_bytes; + cur_src += cur_bytes; + rem_bytes -= cur_bytes; } - cur_dst += dst_stride; - cur_src += src_stride; - rem_rows -= 1; + + row_dst += dst_stride; + row_src += src_stride; } dma_ring_flush(r1); @@ -208,5 +183,3 @@ bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0); } - -#endif diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h index a736eb762..28d06119b 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.h +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -9,7 +9,7 @@ #include <string.h> #include "hex-utils.h" -#include "hex-profile.h" +#include "htp-trace.h" #ifdef __cplusplus extern "C" { @@ -107,6 +107,14 @@ typedef struct { #define DMA_MAX_STRIDE_24B 0x00FFFFFFu // 24-bit HW descriptor limit for strides (16MB - 1) #define DMA_SAFE_CHUNK_SIZE 0x00F00000u // ~15MB safe contiguous chunk size +#if __HVX_ARCH__ < 75 +#define DMA_MAX_2D_ROW_SIZE DMA_MAX_SIZE_16B +#define DMA_MAX_2D_STRIDE DMA_MAX_STRIDE_16B +#else +#define DMA_MAX_2D_ROW_SIZE DMA_MAX_SIZE_24B +#define DMA_MAX_2D_STRIDE DMA_MAX_STRIDE_24B +#endif + #define DMA_FALLBACK_CAPACITY 16u // descriptors in secondary fallback ring typedef struct dma_ring_s dma_ring; @@ -140,10 +148,7 @@ dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q); void dma_queue_alias_free(dma_queue_t q); bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows); -bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total); -#if __HVX_ARCH__ < 75 bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows); -#endif // TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead // but those do not seem to always compiler properly. @@ -216,13 +221,10 @@ static inline bool dma_ring_push_single_1d(dma_ring * r, dma_data ddata, size_t static inline bool dma_ring_push_single_2d(dma_ring * r, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { #if __HVX_ARCH__ > 79 + assert(!((ddata.src | ddata.dst) >> 40) || nrows == 0); const uint32_t src_hi = (uint32_t) (ddata.src >> 32); const uint32_t dst_hi = (uint32_t) (ddata.dst >> 32); const bool is_ext = (src_hi | dst_hi) != 0; - - if (is_ext && ((ddata.src >> 40) || (ddata.dst >> 40))) { - return false; - } #endif if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { @@ -284,6 +286,16 @@ static inline bool dma_ring_push_single_2d(dma_ring * r, dma_data ddata, size_t return true; } +#if __HVX_ARCH__ < 75 +static inline bool dma_ring_push_single_contig(dma_ring * r, dma_data ddata, size_t size) { + return dma_ring_push_single_1d(r, ddata, size); +} +#else +static inline bool dma_ring_push_single_contig(dma_ring * r, dma_data ddata, size_t size) { + return dma_ring_push_single_2d(r, ddata, size, size, size, 1); +} +#endif + static inline dma_data dma_ring_pop(dma_ring * r) { dma_data ddata = { 0 }; @@ -374,58 +386,33 @@ static inline uint32_t dma_queue_capacity(dma_queue * q) { return dma_ring_capacity(q->ring0); } -#if __HVX_ARCH__ < 75 - -static inline bool dma_queue_push(dma_queue *q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - // Fast path: everything fits in 16 bits - if (nrows == 0 || __builtin_expect( - nrows <= DMA_MAX_NROWS && - row_size <= DMA_MAX_SIZE_16B && - src_stride <= DMA_MAX_STRIDE_16B && - dst_stride <= DMA_MAX_STRIDE_16B, 1)) { - return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows); +static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + if (__builtin_expect(nrows == 0, 0)) { + return dma_ring_push_single_1d(q->ring0, ddata, 0); } - // Contiguous block: 1D DMA mode supports up to 24-bit size (16MB) - if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { - size_t total = row_size * nrows; - if (total <= DMA_MAX_SIZE_24B) { - return dma_ring_push_single_1d(q->ring0, ddata, total); + // 1. Hot path: Contiguous or single-row (80-90% of calls) + if (nrows == 1 || !((row_size ^ src_stride) | (row_size ^ dst_stride))) { + const size_t total = row_size * nrows; + if (__builtin_expect(total <= DMA_MAX_SIZE_24B, 1)) { + return dma_ring_push_single_contig(q->ring0, ddata, total); } - return dma_queue_push_fallback_contig(q, ddata, total); - } - - // Row count overflow with 16-bit strides: chunk 2D descriptors via fallback ring - if (row_size <= DMA_MAX_SIZE_16B && src_stride <= DMA_MAX_STRIDE_16B && dst_stride <= DMA_MAX_STRIDE_16B) { - return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); + return dma_queue_push_fallback_1d(q, ddata, 0, 0, total, 1); } - // Stride or row_size overflow: row-by-row 1D via fallback ring - return dma_queue_push_fallback_1d(q, ddata, dst_stride, src_stride, row_size, nrows); -} - -#else // HVX_ARCH >= 75 - -static inline bool dma_queue_push(dma_queue *q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - if (nrows == 0 || __builtin_expect( - nrows <= DMA_MAX_NROWS && - row_size <= DMA_MAX_SIZE_24B && - src_stride <= DMA_MAX_STRIDE_24B && - dst_stride <= DMA_MAX_STRIDE_24B, 1)) { + // 2. Hot path: Standard strided 2D (10-20% of calls) + if (__builtin_expect(nrows <= DMA_MAX_NROWS && + (row_size | src_stride | dst_stride) <= DMA_MAX_2D_ROW_SIZE, 1)) { return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows); } - // Contiguous block exceeding 24 bits - if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { - size_t total = row_size * nrows; - return dma_queue_push_fallback_contig(q, ddata, total); + // 3. Cold path: Descriptor chunking fallbacks (< 0.1%) + if ((row_size | src_stride | dst_stride) <= DMA_MAX_2D_ROW_SIZE) { + return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); } - - return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); + return dma_queue_push_fallback_1d(q, ddata, dst_stride, src_stride, row_size, nrows); } -#endif - static inline void dma_sync_read(dma_queue * dma_q, void * dst, dma_addr_t src, size_t bytes) { const uint32_t b = (uint32_t) bytes; if (b > 0) { @@ -442,10 +429,10 @@ static inline void dma_sync_write(dma_queue * dma_q, dma_addr_t dst, const void } } -#define DMA_CACHE_MAX_SIZE 256U +#define DMA_CACHE_MAX_SIZE 128U // Fully assoc LRU cache -typedef struct { +typedef struct __attribute__((aligned(128))) { uint8_t *base; uint32_t line_size; uint32_t capacity; @@ -491,7 +478,7 @@ static inline bool dma_cache_fa_push(dma_queue *q, dma_cache_fa *c, dma_addr_t s } // Direct mapped cache -typedef struct { +typedef struct __attribute__((aligned(128))) { uint8_t *base; uint32_t line_size; uint32_t capacity; diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c new file mode 100644 index 000000000..b7123923c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c @@ -0,0 +1,675 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" + +#include <HAP_farf.h> + +#include <stdint.h> +#include <string.h> +#include <math.h> + +#include "dma-queue.h" +#include "hex-common.h" +#include "htp-trace.h" +#include "htp-ctx.h" +#include "htp-tensor.h" +#include "hvx-utils.h" +#include "dsv4-hc-ops.h" + +// --------------------------------------------------------------------------- +// DSV4_HC_PRE +// --------------------------------------------------------------------------- + +// hvx_vec_fast_sigmoid_f32_guard_2it with fewer live constants and predicates +static inline HVX_Vector dsv4_hc_sigmoid_f32(HVX_Vector v, HVX_Vector max_exp, HVX_Vector min_exp) { + v = Q6_Vsf_vmin_VsfVsf(Q6_Vsf_vmax_VsfVsf(v, min_exp), max_exp); + v = Q6_Vqf32_vmpy_VsfVsf(v, hvx_vec_splat_f32(0.5f * 1.442695022f)); + + HVX_Vector in_int = Q6_Vw_equals_Vsf(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Newton-Raphson with 2 iterations + HVX_Vector r = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); + for (int i = 0; i < 2; ++i) { + const HVX_Vector rv = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r, v5)); + r = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vadd_VsfVsf(r, r), Q6_Vqf32_vmpy_VsfVsf(r, rv))); + } + + return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, r)); +} + +struct dsv4_hc_pre_context { + struct htp_ops_context * octx; + const struct htp_dsv4_hc_pre_kernel_params * kparams; + uint32_t token_start; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_hvx; + float scale; + uint8_t * vtcm; + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; +}; + +struct dsv4_hc_block { + uint32_t it; + uint32_t chunk; + uint32_t count; +}; + +static inline struct dsv4_hc_block dsv4_hc_pre_block_make( + const struct dsv4_hc_pre_context * dctx, + uint32_t job, + uint32_t end) { + const uint32_t local_token = fastdiv(job, &dctx->kparams->div_n_chunks); + const uint32_t chunk = job - local_token*dctx->n_chunks; + const uint32_t count = hex_smin(dctx->dma_block_chunks, hex_smin(dctx->n_chunks - chunk, end - job)); + return (struct dsv4_hc_block) { + .it = dctx->token_start + local_token, + .chunk = chunk, + .count = count, + }; +} + +static inline void dsv4_hc_pre_push_plain( + dma_queue * dma_q, + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * weights = dctx->octx->src[1]; + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, x->data + (block->it*x->nb[2] + block->chunk*VLEN)), row_stride, x->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 5*row_stride, weights->data + (block->it*weights->nb[1])), 16, 16, 16, 1); +} + +static inline void dsv4_hc_pre_push_gated( + dma_queue * dma_q, + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * g = dctx->octx->src[1]; + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, x->data + (block->it*x->nb[2] + block->chunk*VLEN)), row_stride, x->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 4*row_stride, g->data + (block->it*g->nb[2] + block->chunk*VLEN)), row_stride, g->nb[1], row_size, 4); +} + +static __attribute__((noinline)) void dsv4_hc_pre_plain_block_f32( + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + const uint8_t * x_spad, + uint8_t * d_spad, + const float * wb) { + const uint32_t row_stride = dctx->dma_row_stride; + const HVX_Vector scale_v = hvx_vec_splat_f32(dctx->scale); + const HVX_Vector w0 = hvx_vec_splat_f32(wb[0]); + const HVX_Vector w1 = hvx_vec_splat_f32(wb[1]); + const HVX_Vector w2 = hvx_vec_splat_f32(wb[2]); + const HVX_Vector w3 = hvx_vec_splat_f32(wb[3]); + + for (uint32_t c = 0; c < block->count; ++c) { + const uint8_t * xb = x_spad + c*VLEN; + + HVX_Vector x0 = *(const HVX_Vector *) xb; + HVX_Vector x1 = *(const HVX_Vector *) (xb + row_stride); + HVX_Vector x2 = *(const HVX_Vector *) (xb + 2*row_stride); + HVX_Vector x3 = *(const HVX_Vector *) (xb + 3*row_stride); + + HVX_Vector acc = hvx_vec_mul_f32_f32(x0, w0); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x1, w1)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x2, w2)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x3, w3)); + + *(HVX_Vector *) (d_spad + c*VLEN) = hvx_vec_mul_f32_f32(acc, scale_v); + } +} + +static __attribute__((noinline)) void dsv4_hc_pre_gated_block_f32( + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + const uint8_t * x_spad, + const uint8_t * g_spad, + uint8_t * d_spad) { + const uint32_t row_stride = dctx->dma_row_stride; + const HVX_Vector scale_v = hvx_vec_splat_f32(dctx->scale); + const HVX_Vector max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.0f); + + for (uint32_t c = 0; c < block->count; ++c) { + const uint8_t * xb = x_spad + c*VLEN; + const uint8_t * gb = g_spad + c*VLEN; + + HVX_Vector xv = *(const HVX_Vector *) xb; + HVX_Vector gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) gb, max_exp, min_exp); + HVX_Vector acc = hvx_vec_mul_f32_f32(xv, gv); + + xv = *(const HVX_Vector *) (xb + row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + xv = *(const HVX_Vector *) (xb + 2*row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + 2*row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + xv = *(const HVX_Vector *) (xb + 3*row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + 3*row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + *(HVX_Vector *) (d_spad + c*VLEN) = hvx_vec_mul_f32_f32(acc, scale_v); + } +} + +static void dsv4_hc_pre_thread_plain_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t buffer_size = dctx->dma_buffer_size; + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = th->dma_q; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + // empty dst writeback keeps pop order dst, x, weights in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 4*row_stride), block.count*VLEN, block.count*VLEN, 0, 0); + dsv4_hc_pre_push_plain(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = th->trace; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, job, end); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * wb = (const float *) (spad[curr] + 5*row_stride); + uint8_t * d_spad = spad[curr] + 4*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_pre_plain_block_f32(dctx, &block, spad[curr], d_spad, wb); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[1] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), block.count*VLEN, block.count*VLEN, block.count*VLEN, 1); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + dsv4_hc_pre_push_plain(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +static void dsv4_hc_pre_thread_gated_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t buffer_size = dctx->dma_buffer_size; + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = th->dma_q; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + // empty dst writeback keeps pop order dst, x, g in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 8*row_stride), block.count*VLEN, block.count*VLEN, 0, 0); + dsv4_hc_pre_push_gated(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = th->trace; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, job, end); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + uint8_t * d_spad = spad[curr] + 8*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_pre_gated_block_f32(dctx, &block, spad[curr], spad[curr] + 4*row_stride, d_spad); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[1] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), block.count*VLEN, block.count*VLEN, block.count*VLEN, 1); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + dsv4_hc_pre_push_gated(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +int op_dsv4_hc_pre(struct htp_ops_context * octx) { + const struct htp_tensor * x = octx->src[0]; + const struct htp_tensor * weights = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const struct htp_dsv4_hc_pre_kernel_params * kparams = + (const struct htp_dsv4_hc_pre_kernel_params *) octx->kernel_params; + + const uint32_t n_hvx = kparams->n_hvx; + if (!htp_ops_context_set_n_hvx_div(octx, n_hvx, kparams->div_n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + uint32_t token_start = 0; + uint32_t n_tokens = x->ne[2]; + uint32_t total_jobs = kparams->total_jobs; + uint32_t jobs_per_hvx = kparams->jobs_per_hvx; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + token_start = range.start; + n_tokens = range.count; + total_jobs = n_tokens * kparams->n_chunks; + jobs_per_hvx = fastdiv(total_jobs + n_hvx - 1, &kparams->div_n_hvx); + } + + if (n_tokens == 0) { + return HTP_STATUS_OK; + } + + float scale; + memcpy(&scale, octx->op_params, sizeof(scale)); + + struct dsv4_hc_pre_context dctx = { + .octx = octx, + .kparams = kparams, + .token_start = token_start, + .n_chunks = kparams->n_chunks, + .total_jobs = total_jobs, + .jobs_per_hvx = jobs_per_hvx, + .scale = scale, + .vtcm = (uint8_t *) octx->ctx->vtcm_base, + .dma_block_chunks = kparams->dma_block_chunks, + .dma_row_stride = kparams->dma_row_stride, + .dma_buffer_size = kparams->dma_buffer_size, + }; + + work_queue_func_t worker = NULL; + switch (kparams->kernel_type) { + case HTP_DSV4_HC_PRE_KERNEL_PLAIN: worker = dsv4_hc_pre_thread_plain_f32; break; + case HTP_DSV4_HC_PRE_KERNEL_GATED: worker = dsv4_hc_pre_thread_gated_f32; break; + default: return HTP_STATUS_INVAL_PARAMS; + } + + FARF(HIGH, "dsv4-hc-pre: (%ux%ux%u) -> (%ux%u) : ktype %u tokens %u:%u nhvx %u\n", + x->ne[0], x->ne[1], x->ne[2], dst->ne[0], dst->ne[1], kparams->kernel_type, + token_start, token_start + n_tokens, n_hvx); + + work_queue_run(octx->ctx->hvx_queue, worker, &dctx, n_hvx); + return HTP_STATUS_OK; +} + +// --------------------------------------------------------------------------- +// DSV4_HC_POST +// --------------------------------------------------------------------------- + +struct dsv4_hc_post_context { + struct htp_ops_context * octx; + const struct htp_dsv4_hc_post_kernel_params * kparams; + uint32_t token_start; + uint32_t total_jobs; + uint32_t jobs_per_hvx; + uint8_t * vtcm; +}; + +static inline struct dsv4_hc_block dsv4_hc_post_block_make( + const struct dsv4_hc_post_context * dctx, + uint32_t job, + uint32_t end, + uint32_t block_chunks) { + const uint32_t local_token = fastdiv(job, &dctx->kparams->div_n_chunks); + const uint32_t chunk = job - local_token*dctx->kparams->n_chunks; + const uint32_t count = hex_smin(block_chunks, hex_smin(dctx->kparams->n_chunks - chunk, end - job)); + return (struct dsv4_hc_block) { + .it = dctx->token_start + local_token, + .chunk = chunk, + .count = count, + }; +} + +static __attribute__((noinline)) void dsv4_hc_post_identity_block_f32( + const uint8_t * xb, + const uint8_t * rb, + size_t r_stride, + uint8_t * db, + size_t d_stride, + uint32_t count, + const float * p) { + for (uint32_t id = 0; id < 4; ++id) { + const HVX_Vector pv = hvx_vec_splat_f32(p[id]); + const uint8_t * rr = rb + id*r_stride; + uint8_t * dd = db + id*d_stride; + + for (uint32_t i = 0; i < count; ++i) { + const HVX_Vector xv = *(const HVX_Vector *) (xb + i*VLEN); + const HVX_Vector rv = *(const HVX_Vector *) (rr + i*VLEN); + *(HVX_Vector *) (dd + i*VLEN) = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(xv, pv), rv); + } + } +} + +static __attribute__((noinline)) void dsv4_hc_post_comb_block_f32( + const uint8_t * xb, + const uint8_t * rb, + size_t r_stride, + uint8_t * db, + size_t d_stride, + uint32_t count, + const float * p, + const float * c) { + for (uint32_t id = 0; id < 4; ++id) { + const HVX_Vector pv = hvx_vec_splat_f32(p[id]); + const HVX_Vector c0 = hvx_vec_splat_f32(c[id]); + const HVX_Vector c1 = hvx_vec_splat_f32(c[4 + id]); + const HVX_Vector c2 = hvx_vec_splat_f32(c[8 + id]); + const HVX_Vector c3 = hvx_vec_splat_f32(c[12 + id]); + uint8_t * dd = db + id*d_stride; + + for (uint32_t i = 0; i < count; ++i) { + const uint8_t * r = rb + i*VLEN; + HVX_Vector acc = hvx_vec_mul_f32_f32(*(const HVX_Vector *) (xb + i*VLEN), pv); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) r, c0)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + r_stride), c1)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + 2*r_stride), c2)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + 3*r_stride), c3)); + *(HVX_Vector *) (dd + i*VLEN) = acc; + } + } +} + +static inline void dsv4_hc_post_push_identity( + dma_queue * dma_q, + const struct dsv4_hc_post_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * residual = dctx->octx->src[1]; + const struct htp_tensor * post = dctx->octx->src[2]; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, residual->data + (block->it*residual->nb[2] + block->chunk*VLEN)), + row_stride, residual->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 4*row_stride, x->data + (block->it*x->nb[1] + block->chunk*VLEN)), + row_stride, row_size, row_size, 1); + dma_queue_push(dma_q, dma_make_data(spad + 9*row_stride, post->data + (block->it*post->nb[1])), + 16, 16, 16, 1); +} + +static inline void dsv4_hc_post_push_comb( + dma_queue * dma_q, + const struct dsv4_hc_post_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * comb = dctx->octx->src[3]; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + + dsv4_hc_post_push_identity(dma_q, dctx, block, spad); + dma_queue_push(dma_q, dma_make_data(spad + 9*row_stride + 64, comb->data + (block->it*comb->nb[2])), + 16, comb->nb[1], 16, 4); +} + +static void dsv4_hc_post_thread_identity_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t block_chunks = dctx->kparams->dma_block_chunks; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t buffer_size = dctx->kparams->dma_buffer_size; + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = th->dma_q; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + // empty dst writeback keeps pop order dst, residual, x, post in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 5*row_stride), dst->nb[1], row_stride, 0, 0); + dsv4_hc_post_push_identity(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = th->trace; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, job, end, block_chunks); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * p = (const float *) (spad[curr] + 9*row_stride); + uint8_t * d_spad = spad[curr] + 5*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_post_identity_block_f32(spad[curr] + 4*row_stride, spad[curr], row_stride, d_spad, row_stride, block.count, p); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[2] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), dst->nb[1], row_stride, block.count*VLEN, 4); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + dsv4_hc_post_push_identity(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +static void dsv4_hc_post_thread_comb_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t block_chunks = dctx->kparams->dma_block_chunks; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t buffer_size = dctx->kparams->dma_buffer_size; + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = th->dma_q; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + // empty dst writeback keeps pop order dst, residual, x, post, comb in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 5*row_stride), dst->nb[1], row_stride, 0, 0); + dsv4_hc_post_push_comb(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = th->trace; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, job, end, block_chunks); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * p = (const float *) (spad[curr] + 9*row_stride); + const float * c = (const float *) (spad[curr] + 9*row_stride + 64); + uint8_t * d_spad = spad[curr] + 5*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_post_comb_block_f32(spad[curr] + 4*row_stride, spad[curr], row_stride, d_spad, row_stride, block.count, p, c); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[2] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), dst->nb[1], row_stride, block.count*VLEN, 4); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + dsv4_hc_post_push_comb(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +int op_dsv4_hc_post(struct htp_ops_context * octx) { + const struct htp_tensor * x = octx->src[0]; + const struct htp_tensor * residual = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const struct htp_dsv4_hc_post_kernel_params * kparams = + (const struct htp_dsv4_hc_post_kernel_params *) octx->kernel_params; + + const uint32_t n_hvx = kparams->n_hvx; + if (!htp_ops_context_set_n_hvx_div(octx, n_hvx, kparams->div_n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + uint32_t token_start = 0; + uint32_t n_tokens = x->ne[1]; + uint32_t total_jobs = kparams->total_jobs; + uint32_t jobs_per_hvx = kparams->jobs_per_hvx; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + token_start = range.start; + n_tokens = range.count; + total_jobs = n_tokens * kparams->n_chunks; + jobs_per_hvx = fastdiv(total_jobs + n_hvx - 1, &kparams->div_n_hvx); + } + + if (n_tokens == 0) { + return HTP_STATUS_OK; + } + + struct dsv4_hc_post_context dctx = { + .octx = octx, + .kparams = kparams, + .token_start = token_start, + .total_jobs = total_jobs, + .jobs_per_hvx = jobs_per_hvx, + .vtcm = (uint8_t *) octx->ctx->vtcm_base, + }; + + work_queue_func_t worker = NULL; + switch (kparams->kernel_type) { + case HTP_DSV4_HC_POST_KERNEL_IDENTITY: worker = dsv4_hc_post_thread_identity_f32; break; + case HTP_DSV4_HC_POST_KERNEL_COMB: worker = dsv4_hc_post_thread_comb_f32; break; + default: return HTP_STATUS_INVAL_PARAMS; + } + + FARF(HIGH, "dsv4-hc-post: (%ux%u) (%ux%ux%u) -> (%ux%ux%u) : ktype %u tokens %u:%u nhvx %u\n", + x->ne[0], x->ne[1], residual->ne[0], residual->ne[1], residual->ne[2], + dst->ne[0], dst->ne[1], dst->ne[2], kparams->kernel_type, + token_start, token_start + n_tokens, n_hvx); + + work_queue_run(octx->ctx->hvx_queue, worker, &dctx, n_hvx); + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h new file mode 100644 index 000000000..58a3d0d3c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h @@ -0,0 +1,77 @@ +#ifndef HTP_DSV4_HC_OPS_H +#define HTP_DSV4_HC_OPS_H + +#include <stdint.h> + +#include "hex-fastdiv.h" +#include "htp-ops.h" + +#define HTP_DSV4_HC_CHUNK_ELEMS 32 +#define HTP_DSV4_HC_MAX_CHUNKS 128 +#define HTP_DSV4_HC_PRE_GATED_ROWS 9 +#define HTP_DSV4_HC_PRE_PLAIN_ROWS 6 + +enum htp_dsv4_hc_pre_kernel_type { + HTP_DSV4_HC_PRE_KERNEL_PLAIN = 0, + HTP_DSV4_HC_PRE_KERNEL_GATED = 1, +}; + +struct htp_dsv4_hc_pre_kernel_params { + uint32_t kernel_type; + uint32_t n_hvx; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_hvx; + + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; + uint32_t vtcm_size; + + struct fastdiv_values div_n_chunks; + struct fastdiv_values div_n_hvx; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_dsv4_hc_pre_kernel_params) <= 128, "htp_dsv4_hc_pre_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_dsv4_hc_pre_kernel_params) <= 128, "htp_dsv4_hc_pre_kernel_params is too large for kernel_params blob"); +#endif + +// dma buffer holds residual rows 0..3, x at row 4, dst rows 5..8, post/comb at row 9 +#define HTP_DSV4_HC_POST_ROWS 10 + +enum htp_dsv4_hc_post_kernel_type { + HTP_DSV4_HC_POST_KERNEL_IDENTITY = 0, + HTP_DSV4_HC_POST_KERNEL_COMB = 1, +}; + +struct htp_dsv4_hc_post_kernel_params { + uint32_t kernel_type; + uint32_t n_hvx; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_hvx; + uint32_t has_comb; + + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; + uint32_t vtcm_size; + + struct fastdiv_values div_n_chunks; + struct fastdiv_values div_n_hvx; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_dsv4_hc_post_kernel_params) <= 128, "htp_dsv4_hc_post_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_dsv4_hc_post_kernel_params) <= 128, "htp_dsv4_hc_post_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_ops_context; + +int op_dsv4_hc_pre(struct htp_ops_context * octx); +int op_dsv4_hc_post(struct htp_ops_context * octx); + +#endif // HTP_DSV4_HC_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c index 212104a23..620263063 100644 --- a/ggml/src/ggml-hexagon/htp/fill-ops.c +++ b/ggml/src/ggml-hexagon/htp/fill-ops.c @@ -6,7 +6,7 @@ #include <string.h> #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hvx-copy.h" #include "hvx-utils.h" @@ -16,6 +16,7 @@ #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" +#include "fill-ops.h" // ggml op_params layout for FILL: // op_params[0] (as float) - the scalar fill value @@ -30,75 +31,87 @@ \ const uint32_t nb1 = dst->nb[1]; \ const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const uint32_t nr = ne1 * ne2 * ne3; + const uint32_t nb3 = dst->nb[3]; struct htp_fill_context { struct htp_ops_context * octx; - uint32_t nrows_per_thread; + const struct htp_fill_kernel_params * kparams; + uint32_t nrows_per_hvx; uint32_t total_rows; // ne1 * ne2 * ne3 uint32_t row_start; - bool opt_path; HVX_Vector splat_vec; uint32_t elem_size; }; -static void fill_thread(unsigned int nth, unsigned int ith, void * data) { +static void fill_thread_contiguous(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_fill_context * fctx = (const struct htp_fill_context *) data; struct htp_ops_context * octx = fctx->octx; fill_preamble; // Parallelise over the flat row index spanning ne1*ne2*ne3 - const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_hvx, fctx->row_start + fctx->total_rows); if (ir0 >= ir1) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); - if (fctx->opt_path) { - // Opt path: tensor is fully contiguous, treat as flat array - const uint32_t elem_start = ir0 * ne0; - const uint32_t elem_end = ir1 * ne0; - uint8_t * dst_ptr = (uint8_t *) dst->data + elem_start * fctx->elem_size; - hvx_splat_u(dst_ptr, fctx->splat_vec, elem_end - elem_start, fctx->elem_size); - } else { - // Non-contiguous path: must respect strides - for (uint32_t ir = ir0; ir < ir1; ++ir) { - const uint32_t i1 = ir % ne1; - const uint32_t i2 = (ir / ne1) % ne2; - const uint32_t i3 = ir / (ne1 * ne2); - uint8_t * dst_ptr = (uint8_t *) dst->data + i1*nb1 + i2*nb2 + i3*nb3; - hvx_splat_u(dst_ptr, fctx->splat_vec, ne0, fctx->elem_size); - } - } + const uint32_t elem_start = ir0 * ne0; + const uint32_t elem_end = ir1 * ne0; + uint8_t * dst_ptr = (uint8_t *) dst->data + elem_start * fctx->elem_size; + hvx_splat_u(dst_ptr, fctx->splat_vec, elem_end - elem_start, fctx->elem_size); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); - FARF(HIGH, "fill %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); + FARF(HIGH, "fill-contig %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); } -int op_fill(struct htp_ops_context * octx) { +static void fill_thread_strided(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_fill_context * fctx = (const struct htp_fill_context *) data; + struct htp_ops_context * octx = fctx->octx; + const struct htp_fill_kernel_params * kparams = fctx->kparams; fill_preamble; - if (dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16) { - return HTP_STATUS_NO_SUPPORT; + // Parallelise over the flat row index spanning ne1*ne2*ne3 + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_hvx, fctx->row_start + fctx->total_rows); + + if (ir0 >= ir1) { + return; } + struct htp_thread_trace * tr = th->trace; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + + for (uint32_t ir = ir0; ir < ir1; ++ir) { + const uint32_t i3 = fastdiv(ir, &kparams->div_ne1_ne2); + const uint32_t rem = ir - i3 * (ne1 * ne2); + const uint32_t i2 = fastdiv(rem, &kparams->div_ne1); + const uint32_t i1 = rem - i2 * ne1; + uint8_t * dst_ptr = (uint8_t *) dst->data + i1*nb1 + i2*nb2 + i3*nb3; + hvx_splat_u(dst_ptr, fctx->splat_vec, ne0, fctx->elem_size); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); + FARF(HIGH, "fill-strided %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); +} + +int op_fill(struct htp_ops_context * octx) { + const struct htp_tensor * dst = octx->dst; + const struct htp_fill_kernel_params * kparams = + (const struct htp_fill_kernel_params *) octx->kernel_params; + if (htp_tensor_is_extended(dst)) { return HTP_STATUS_NO_SUPPORT; } uint32_t row_start = 0; - uint32_t nrows = nr; + uint32_t nrows = kparams->total_rows; if (octx->ctx->mdev.count > 1) { - const uint32_t row_size = nb1; - const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(nr, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); row_start = range.start; nrows = range.count; } @@ -107,40 +120,43 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - // nr = ne1*ne2*ne3 (flat row count across all outer dims); parallelise over it. - const uint32_t n_threads = octx->n_threads; + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } - // Optimize if fully contiguous: skip stride arithmetic, treat as flat array - const bool opt_path = (nb2 == nb1 * ne1) && (nb3 == nb2 * ne2); + const uint32_t n_hvx = octx->n_hvx; - FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u opt=%d\n", - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, (int) opt_path); + FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u ktype=%u\n", + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, kparams->kernel_type); float val_f32 = 0.f; memcpy(&val_f32, &octx->op_params[0], sizeof(float)); struct htp_fill_context fctx = { .octx = octx, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .kparams = kparams, + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_rows = nrows, .row_start = row_start, - .opt_path = opt_path, + .elem_size = kparams->elem_size, }; switch (dst->type) { case HTP_TYPE_F32: fctx.splat_vec = hvx_vec_splat_f32(val_f32); - fctx.elem_size = sizeof(float); break; case HTP_TYPE_F16: fctx.splat_vec = hvx_vec_splat_f16((_Float16) val_f32); - fctx.elem_size = sizeof(_Float16); break; default: return HTP_STATUS_NO_SUPPORT; } - work_queue_run(octx->ctx->work_queue, fill_thread, &fctx, n_threads); + work_queue_func_t worker = (kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS) + ? fill_thread_contiguous + : fill_thread_strided; + + work_queue_run(octx->ctx->hvx_queue, worker, &fctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.h b/ggml/src/ggml-hexagon/htp/fill-ops.h new file mode 100644 index 000000000..06fe08209 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/fill-ops.h @@ -0,0 +1,29 @@ +#ifndef HTP_FILL_OPS_H +#define HTP_FILL_OPS_H + +#include "hex-fastdiv.h" +#include <stdint.h> + +enum htp_fill_kernel_type { + HTP_FILL_KERNEL_CONTIGUOUS = 0, + HTP_FILL_KERNEL_STRIDED = 1, +}; + +struct htp_fill_kernel_params { + uint8_t kernel_type; + uint32_t n_hvx; + uint32_t total_rows; + uint32_t elem_size; + uint32_t rows_per_chunk; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne1_ne2; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_FILL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index f079f7389..0b7fc2dae 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -14,7 +14,7 @@ #include "dma-queue.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hmx-queue.h" #include "hmx-utils.h" #include "hvx-utils.h" @@ -39,6 +39,26 @@ // Must be multiple of 32 #define FLASH_ATTN_BLOCK_SIZE (32 * 2) +// Large scratch lives in ctx->ddr_spad to keep stack usage low. +// Ops can run on the dspqueue callback thread, which has a small stack. +struct hvx_fa_ddr_spad { + __fp16 slopes[512]; + uint16_t active_blocks[FA_MAX_KV_BLOCKS]; + uint16_t thread_active_blocks[HTP_MAX_NHVX][FA_MAX_KV_BLOCKS] __attribute__((aligned(128))); +}; + +struct hmx_fa_ddr_spad { + uint16_t active_blocks[FA_MAX_KV_BLOCKS]; + __fp16 slopes[] __attribute__((aligned(128))); // [g_br] +}; + +static inline void * fa_get_ddr_spad(const struct htp_ops_context * octx, size_t size) { + if (!octx->ctx->ddr_spad_base || octx->ctx->ddr_spad_size < size) { + return NULL; + } + return octx->ctx->ddr_spad_base; +} + struct htp_fa_context { const struct htp_ops_context * octx; @@ -61,9 +81,10 @@ struct htp_fa_context { uint32_t n_head_log2; float m0; float m1; - __fp16 slopes[512]; uint32_t n_blocks; + uint32_t n_active_blocks; + bool shared_active_blocks; size_t size_q_row_padded; size_t size_k_row_padded; @@ -74,7 +95,7 @@ struct htp_fa_context { size_t size_m_block; uint32_t qrows; - uint32_t qrows_per_thread; + uint32_t qrows_per_hvx; uint32_t qrow_start; bool is_q_fp32; @@ -82,12 +103,15 @@ struct htp_fa_context { size_t size_q_block; size_t size_vkq_acc; - uint8_t * spad_q; - uint8_t * spad_k; - uint8_t * spad_v; - uint8_t * spad_m; - uint8_t * spad_a; - float * spad_sinks; + uint8_t * spad_q; + uint8_t * spad_k; + uint8_t * spad_v; + uint8_t * spad_m; + uint8_t * spad_a; + float * spad_sinks; + + struct hvx_fa_ddr_spad * ddr; + dma_cache_dm * m_cache; // [n_hvx] const struct htp_tensor * k; const struct htp_tensor * v; @@ -98,8 +122,8 @@ struct hmx_fa_context { const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent const struct htp_tensor * k; const struct htp_tensor * v; - bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2 - uint32_t n_threads; + bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_hvx >= 2 + uint32_t n_hvx; // Op parameters __fp16 scale; @@ -147,23 +171,96 @@ struct hmx_fa_context { HVX_Vector * vtcm_l_vec; // Row sum [g_br] HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br] HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br] - HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64] + HVX_Vector * vtcm_row_bufs; // Per-HVX softmax row scratch [n_hvx][2][Bc/64] uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity) uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale) __fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block __fp16 * vtcm_slopes; // ALiBi slopes [g_br] float * vtcm_sinks; // Attention sinks + __fp16 * ddr_slopes; // ALiBi slopes staging buffer in DDR [g_br] size_t row_buf_stride; // HVX vectors per row buffer (Bc/64) size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffer size_t q_tile_bytes; + size_t k_tile_bytes; size_t o_tile_bytes; size_t col_vec_bytes; size_t d_tile_bytes; bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA) - dma_cache_fa m_cache; + dma_cache_fa * m_cache; }; -static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) { +// Scan mask rows [q_start, q_start + n_rows_q) and build compact list of KV blocks with any entry != -inf. +static uint32_t fa_scan_active_blocks( + dma_queue * dma_q, + const struct htp_tensor * mask, + uint32_t q_start, + uint32_t n_rows_q, + uint32_t im2, + uint32_t im3, + uint32_t nek1, + uint32_t Bc, + uint32_t n_kv_blocks, + void * vtcm_scratch, + size_t scratch_bytes, + uint16_t * active_blocks) { + const size_t row_bytes = (size_t) nek1 * sizeof(__fp16); + const size_t row_stride = hex_round_up(row_bytes, 128); + + if (!mask || n_kv_blocks <= 1 || n_kv_blocks > FA_MAX_KV_BLOCKS || + (size_t) n_rows_q * row_stride > scratch_bytes) { + #pragma clang loop vectorize(disable) + for (uint32_t b = 0; b < n_kv_blocks; ++b) { + active_blocks[b] = (uint16_t) b; + } + return n_kv_blocks; + } + + const dma_addr_t m_src = mask->data + q_start * mask->nb[1] + im2 * mask->nb[2] + im3 * mask->nb[3]; + dma_queue_push(dma_q, dma_make_data(vtcm_scratch, m_src), row_stride, mask->nb[1], row_bytes, n_rows_q); + const uint8_t * m_buf = (const uint8_t *) dma_queue_pop(dma_q).dst; + + const HVX_Vector v_ninf = Q6_Vh_vsplat_R(0xFC00); + uint32_t n_active = 0; + + for (uint32_t b = 0; b < n_kv_blocks; ++b) { + const uint32_t col0 = b * Bc; + const uint32_t cols = MIN(Bc, nek1 - col0); + + for (uint32_t r = 0; r < n_rows_q; ++r) { + const __fp16 * row_ptr = (const __fp16 *) (m_buf + r * row_stride) + col0; + HVX_Vector v_any = Q6_V_vzero(); + uint32_t c = 0; + for (; c + 64 <= cols; c += 64) { + const HVX_Vector v_m = *(const HVX_Vector *) (row_ptr + c); + v_any = Q6_V_vor_VV(v_any, Q6_V_vxor_VV(v_m, v_ninf)); + } + if (c < cols) { + const uint32_t rem = cols - c; + const HVX_VectorPred q_rem = Q6_Q_vsetq2_R(rem * sizeof(__fp16)); + const HVX_Vector v_m = *(const HVX_Vector *) (row_ptr + c); + const HVX_Vector v_diff = Q6_V_vand_QV(q_rem, Q6_V_vxor_VV(v_m, v_ninf)); + v_any = Q6_V_vor_VV(v_any, v_diff); + } + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 64)); + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 32)); + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 16)); + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 8)); + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 4)); + if (Q6_R_vextract_VR(v_any, 0) != 0) { + active_blocks[n_active++] = (uint16_t) b; + break; + } + } + } + + if (n_active == 0 && n_kv_blocks > 0) { + active_blocks[0] = 0; + n_active = 1; + } + return n_active; +} + +static void flash_attn_ext_f16_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_fa_context * factx = (struct htp_fa_context *) data; const struct htp_ops_context * octx = factx->octx; const struct htp_tensor * q = octx->src[0]; @@ -209,15 +306,15 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; // total rows in q - const uint32_t dr = factx->qrows_per_thread; + const uint32_t dr = factx->qrows_per_hvx; const uint32_t ir0 = factx->qrow_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows); if (ir0 >= ir1) return; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t DK = nek0; const uint32_t DV = nev0; @@ -233,8 +330,20 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith; uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith; - dma_cache_dm m_cache; - dma_cache_dm_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE); + dma_cache_dm * m_cache = &factx->m_cache[ith]; + dma_cache_dm_init(m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE); + + uint16_t * thread_active_blocks = factx->ddr->thread_active_blocks[ith]; + const uint16_t * active_blocks = factx->shared_active_blocks ? factx->ddr->active_blocks : thread_active_blocks; + uint32_t n_active_blocks = factx->n_active_blocks; + dma_addr_t prev_mp_base = 0; + + if (!factx->shared_active_blocks && !mask) { + #pragma clang loop vectorize(disable) + for (uint32_t b = 0; b < factx->n_blocks; ++b) { + thread_active_blocks[b] = (uint16_t) b; + } + } const size_t size_vkq_acc_single = hex_round_up(DV * sizeof(float), 128); @@ -271,6 +380,19 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * } } + dma_addr_t mp_base = 0; + if (mask) { + const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2); + const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3); + mp_base = mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]; + if (!factx->shared_active_blocks && mp_base != prev_mp_base) { + n_active_blocks = fa_scan_active_blocks( + dma_q, mask, iq1, 1, im2, im3, nek1, FLASH_ATTN_BLOCK_SIZE, + factx->n_blocks, spad_k, factx->size_k_block * 2, thread_active_blocks); + prev_mp_base = mp_base; + } + } + uint32_t heads[FA_HVX_G_MAX]; HVX_Vector slope_vecs[FA_HVX_G_MAX] __attribute__((aligned(128))); HVX_Vector S_vec[FA_HVX_G_MAX] __attribute__((aligned(128))); @@ -285,7 +407,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t r_iq1 = (r - r_iq3*neq2*neq1 - r_iq2 * neq1); heads[g] = r_iq2; - const __fp16 slope = factx->slopes[r_iq2]; + const __fp16 slope = factx->ddr->slopes[r_iq2]; slope_vecs[g] = hvx_vec_splat_f16(slope); S_vec[g] = hvx_vec_splat_f32(0.0f); @@ -303,32 +425,26 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * dma_queue_push(dma_q, dma_make_data(q_dst, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1); } - dma_addr_t mp_base = 0; - if (mask) { - const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2); - const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3); - mp_base = mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]; - } - - // Prefetch first two blocks - for (uint32_t ib = 0; ib < MIN(factx->n_blocks, 2); ++ib) { + // Prefetch first two active blocks + for (uint32_t step = 0; step < MIN(n_active_blocks, 2); ++step) { + const uint32_t ib = active_blocks[step]; const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE; const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); // K const dma_addr_t k_src = k->data + ic_start*nbk1 + ik2*nbk2 + ik3*nbk3; - uint8_t * k_dst = spad_k + (ib % 2) * factx->size_k_block; + uint8_t * k_dst = spad_k + (step % 2) * factx->size_k_block; dma_queue_push(dma_q, dma_make_data(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size); // V const dma_addr_t v_src = v->data + ic_start*nbv1 + iv2*nbv2 + iv3*nbv3; - uint8_t * v_dst = spad_v + (ib % 2) * factx->size_v_block; + uint8_t * v_dst = spad_v + (step % 2) * factx->size_v_block; dma_queue_push(dma_q, dma_make_data(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size); // Mask if (mask) { const dma_addr_t m_src = mp_base + ic_start * sizeof(__fp16); - dma_cache_dm_push(dma_q, &m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1); + dma_cache_dm_push(dma_q, m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1); } } @@ -348,7 +464,8 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F); const uint32_t stride_v2 = factx->size_v_row_padded * 2; - for (uint32_t ib = 0; ib < factx->n_blocks; ++ib) { + for (uint32_t step = 0; step < n_active_blocks; ++step) { + const uint32_t ib = active_blocks[step]; const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE; const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); @@ -361,7 +478,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir); for (uint32_t r = 0; r < current_block_size; ++r) { __fp16 * row_k = (__fp16 *)(k_base + r * factx->size_k_row_padded); - hvx_dequantize_row_q8_0_f16(row_k, row_k, DK); + hvx_dequantize_row_flat_q8_0_f16(row_k, row_k, DK); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir); } @@ -369,7 +486,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir); for (uint32_t r = 0; r < current_block_size; ++r) { __fp16 * row_v = (__fp16 *)(v_base + r * factx->size_v_row_padded); - hvx_dequantize_row_q8_0_f16(row_v, row_v, DV); + hvx_dequantize_row_flat_q8_0_f16(row_v, row_v, DV); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir); } @@ -411,9 +528,9 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * HVX_Vector v_max = Q6_V_lo_W(hvx_vec_f16_to_f32(v_max_f16)); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_QK, head_ir); - // prefetch K for block ib + 2 after last head finished QK - if (g + 1 == G_local && ib + 2 < factx->n_blocks) { - const uint32_t next_ib = ib + 2; + // prefetch K for step + 2 after last head finished QK + if (g + 1 == G_local && step + 2 < n_active_blocks) { + const uint32_t next_ib = active_blocks[step + 2]; const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE; const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start); @@ -504,6 +621,56 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * ((HVX_VectorPair *) VKQ32)[0] = vkq0; ((HVX_VectorPair *) VKQ32)[1] = vkq1; + } else if (DV == 256) { + HVX_VectorPair vkq0 = ((const HVX_VectorPair *) VKQ32)[0]; + HVX_VectorPair vkq1 = ((const HVX_VectorPair *) VKQ32)[1]; + HVX_VectorPair vkq2 = ((const HVX_VectorPair *) VKQ32)[2]; + HVX_VectorPair vkq3 = ((const HVX_VectorPair *) VKQ32)[3]; + vkq0 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq0), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq0), ms_vec) + ); + vkq1 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq1), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq1), ms_vec) + ); + vkq2 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq2), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq2), ms_vec) + ); + vkq3 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq3), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq3), ms_vec) + ); + + for (uint32_t j = 0; j < current_block_size; j += 2) { + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + const HVX_Vector * vx0 = (const HVX_Vector *) v_ptr; + if (j + 1 == current_block_size) { + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0); + vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx0[2]), S0); + vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx0[3]), S0); + break; + } + + HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); + const HVX_Vector * vx1 = (const HVX_Vector *) (v_ptr + factx->size_v_row_padded); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx1[0]), S1); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx1[1]), S1); + vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx0[2]), S0); + vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx1[2]), S1); + vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx0[3]), S0); + vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx1[3]), S1); + v_ptr += stride_v2; + } + + ((HVX_VectorPair *) VKQ32)[0] = vkq0; + ((HVX_VectorPair *) VKQ32)[1] = vkq1; + ((HVX_VectorPair *) VKQ32)[2] = vkq2; + ((HVX_VectorPair *) VKQ32)[3] = vkq3; } else { hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); @@ -524,9 +691,9 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, head_ir); - // prefetch V and mask for block ib + 2 after last head finished V accumulation - if (g + 1 == G_local && ib + 2 < factx->n_blocks) { - const uint32_t next_ib = ib + 2; + // prefetch V and mask for step + 2 after last head finished V accumulation + if (g + 1 == G_local && step + 2 < n_active_blocks) { + const uint32_t next_ib = active_blocks[step + 2]; const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE; const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start); @@ -537,11 +704,11 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * // Mask if (mask) { const dma_addr_t m_src = mp_base + next_ic_start * sizeof(__fp16); - dma_cache_dm_push(dma_q, &m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1); + dma_cache_dm_push(dma_q, m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1); } } } // end for g - } // end for ib + } // end for step for (uint32_t g = 0; g < G_local; ++g) { const uint32_t head_ir = ir + g; @@ -607,7 +774,7 @@ typedef struct { size_t buf_idx; } fa_k_int_args_t; -static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) { +static void fa_k_interleave_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_k_int_args_t * args = (fa_k_int_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -620,12 +787,12 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); if (factx->k->type == HTP_TYPE_Q8_0) { for (uint32_t r = start; r < end; ++r) { __fp16 * row_k = (__fp16 *)((char *)args->curr_k + r * args->src_stride * sizeof(__fp16)); - hvx_dequantize_row_q8_0_f16(row_k, row_k, factx->DK); + hvx_dequantize_row_flat_q8_0_f16(row_k, row_k, factx->DK); } } hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK_pad, @@ -634,18 +801,14 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) } static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start, size_t buf_idx) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { - n = factx->n_threads; + if (factx->n_hvx > 1 && kv_rows >= factx->n_hvx * 2) { + n = factx->n_hvx; } uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t, buf_idx }; - if (n > 1) { - work_queue_run(wp, fa_k_interleave_thread, &args, n); - } else { - fa_k_interleave_thread(1, 0, &args); - } + work_queue_run(wp, fa_k_interleave_thread, &args, n); } typedef struct { @@ -659,7 +822,7 @@ typedef struct { uint32_t rows_per_t; } fa_v_int_args_t; -static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) { +static void fa_v_interleave_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_v_int_args_t * args = (fa_v_int_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -674,12 +837,12 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) __fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst; - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); if (factx->v->type == HTP_TYPE_Q8_0) { for (uint32_t r = start; r < end; ++r) { __fp16 * row_v = (__fp16 *)((char *)args->v_src + r * args->src_stride * sizeof(__fp16)); - hvx_dequantize_row_q8_0_f16(row_v, row_v, factx->DV); + hvx_dequantize_row_flat_q8_0_f16(row_v, row_v, factx->DV); } } hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV_pad, @@ -694,18 +857,14 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx, void * v_tiles_dst, size_t n_col_tiles, uint32_t kv_start) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { - n = factx->n_threads; + if (factx->n_hvx > 1 && kv_rows >= factx->n_hvx * 2) { + n = factx->n_hvx; } uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t }; - if (n > 1) { - work_queue_run(wp, fa_v_interleave_thread, &args, n); - } else { - fa_v_interleave_thread(1, 0, &args); - } + work_queue_run(wp, fa_v_interleave_thread, &args, n); } typedef struct { @@ -721,7 +880,7 @@ typedef struct { atomic_uint barrier; } fa_q_load_args_t; -static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { +static void fa_q_load_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_q_load_args_t * args = (fa_q_load_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -735,7 +894,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { const size_t start = (size_t) i * rows_per_t; const size_t end = hex_smin(start + rows_per_t, factx->g_br); - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); // Parallel initialization of per-block state @@ -755,40 +914,11 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { const size_t m_start = i * m_bytes_per_t; const size_t m_end = hex_smin(m_start + m_bytes_per_t, col_vec_bytes); - if (factx->sinks) { - const float * sinks_data = factx->vtcm_sinks; - float * m_vec = (float *) factx->vtcm_m_vec; - const size_t r_start = l_start / sizeof(float); - const size_t r_end = l_end / sizeof(float); - const float scale_factor = EXP_LOG2E_F; - - const HVX_Vector v_scale = hvx_vec_splat_f32(scale_factor); - - for (size_t r = r_start; r < r_end; r += 32) { - HVX_VectorAlias local_m; - for (size_t j = 0; j < 32; ++j) { - size_t curr_r = r + j; - if (curr_r < n_rows_g) { - const size_t h_idx = fastmodulo(curr_r, G, &factx->div_G); - const size_t head = args->kv_head * G + h_idx; - local_m.fp32[j] = sinks_data[head]; - } else { - local_m.fp32[j] = HTP_FA_M_INITIAL_VAL; - } - } - HVX_Vector v_scaled = HVX_OP_MUL_F32(local_m.v, v_scale); - *(HVX_Vector *) (m_vec + r) = v_scaled; - } - if (l_start < col_vec_bytes) { - hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start); - } - } else { - if (l_start < col_vec_bytes) { - hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start); - } - if (m_start < col_vec_bytes) { - hvx_splat_f32_a((char *) factx->vtcm_m_vec + m_start, HTP_FA_M_INITIAL_VAL, (m_end - m_start) / sizeof(float)); - } + if (l_start < col_vec_bytes) { + hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start); + } + if (m_start < col_vec_bytes) { + hvx_splat_f32_a((char *) factx->vtcm_m_vec + m_start, HTP_FA_M_INITIAL_VAL, (m_end - m_start) / sizeof(float)); } // Zero the whole rescale region: vtcm_d_tiles[0], the optional vtcm_d_tiles[1] @@ -872,10 +1002,10 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { - n = factx->n_threads; + if (factx->n_hvx > 1 && n_rows_g >= (size_t) (factx->n_hvx * 2)) { + n = factx->n_hvx; } size_t rows_per_t = hex_align_up(hmx_ceil_div(factx->g_br, n), 2); const uint32_t n_rows_q = hex_smin(factx->Br, factx->neq1 - q_start); @@ -890,11 +1020,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, args.n_rows_q = n_rows_q; args.q_transposed = q->nb[1] < q->nb[2]; atomic_init(&args.barrier, n); - if (n > 1) { - work_queue_run(wp, fa_q_load_thread, &args, n); - } else { - fa_q_load_thread(1, 0, &args); - } + work_queue_run(wp, fa_q_load_thread, &args, n); } typedef struct { @@ -908,7 +1034,7 @@ typedef struct { size_t rows_per_t; } fa_o_store_args_t; -static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { +static void fa_o_store_thread_f32(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_o_store_args_t * args = (fa_o_store_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -924,7 +1050,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -975,7 +1101,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); } -static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { +static void fa_o_store_thread_f16(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_o_store_args_t * args = (fa_o_store_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -990,7 +1116,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -1052,19 +1178,15 @@ static void fa_phase_o_store(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { - n = factx->n_threads; + if (factx->n_hvx > 1 && n_rows_g >= (size_t) (factx->n_hvx * 2)) { + n = factx->n_hvx; } size_t rows_per_t = hmx_ceil_div(n_rows_g, n); fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t }; - worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; - if (n > 1) { - work_queue_run(wp, store_fn, &args, n); - } else { - store_fn(1, 0, &args); - } + work_queue_func_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; + work_queue_run(wp, store_fn, &args, n); } typedef struct { @@ -1091,7 +1213,7 @@ typedef struct { } fa_softmax_args_t; static inline void fa_softmax_impl( - unsigned int n, unsigned int i, void * data, + const htp_thread_t * th, unsigned int n, unsigned int i, void * data, const bool has_mask, const bool mask_broadcast, const bool is_g1, @@ -1121,7 +1243,7 @@ static inline void fa_softmax_impl( return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride @@ -1447,21 +1569,10 @@ static inline void fa_softmax_impl( HVX_Vector v_rowsum_acc_f32_0 = Q6_V_lo_W(rowsum_acc_pair); HVX_Vector v_rowsum_acc_f32_1 = Q6_V_hi_W(rowsum_acc_pair); - HVX_Vector v_l_curr0; - HVX_Vector v_l_curr1; - if (args->kv_start == 0 && factx->sinks != NULL) { - // First KV block with sinks: m_prev holds the seeded sink value (not -inf), - // so exp_m_diff = exp2(sink - m_curr) is the sink's contribution to the - // denominator. l_prev is 0 here, so add exp_m_diff directly instead of - // multiplying the (uninitialized) l_prev term. - v_l_curr0 = HVX_OP_ADD_F32(exp_m_diff0, v_rowsum_acc_f32_0); - v_l_curr1 = HVX_OP_ADD_F32(exp_m_diff1, v_rowsum_acc_f32_1); - } else { - HVX_Vector l_prev_v0 = factx->vtcm_l_vec[r_vec_idx * 2 + 0]; - HVX_Vector l_prev_v1 = factx->vtcm_l_vec[r_vec_idx * 2 + 1]; - v_l_curr0 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v0, exp_m_diff0), v_rowsum_acc_f32_0); - v_l_curr1 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v1, exp_m_diff1), v_rowsum_acc_f32_1); - } + HVX_Vector l_prev_v0 = factx->vtcm_l_vec[r_vec_idx * 2 + 0]; + HVX_Vector l_prev_v1 = factx->vtcm_l_vec[r_vec_idx * 2 + 1]; + HVX_Vector v_l_curr0 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v0, exp_m_diff0), v_rowsum_acc_f32_0); + HVX_Vector v_l_curr1 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v1, exp_m_diff1), v_rowsum_acc_f32_1); factx->vtcm_m_vec[r_vec_idx * 2 + 0] = v_m_curr0; factx->vtcm_m_vec[r_vec_idx * 2 + 1] = v_m_curr1; @@ -1492,8 +1603,8 @@ static inline void fa_softmax_impl( htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); } -static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_nomask(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/false, /*mask_broadcast=*/false, /*is_g1=*/false, @@ -1501,8 +1612,8 @@ static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data /*has_softcap=*/false); } -static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_mask_broadcast_g1(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/true, /*mask_broadcast=*/true, /*is_g1=*/true, @@ -1510,8 +1621,8 @@ static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, /*has_softcap=*/false); } -static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_mask_broadcast_gn(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/true, /*mask_broadcast=*/true, /*is_g1=*/false, @@ -1519,7 +1630,7 @@ static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, /*has_softcap=*/false); } -static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { +static void fa_softmax_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_softmax_args_t * args = (fa_softmax_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -1529,12 +1640,69 @@ static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { const bool has_alibi = args->has_alibi; const bool has_softcap = factx->has_softcap; - fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); + fa_softmax_impl(th, n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); +} + +static inline HVX_Vector hvx_fa_load_sinks(const float * sinks_data, size_t idx, size_t count) { + const HVX_Vector * v_src = (const HVX_Vector *) sinks_data + (idx >> 5); + const size_t lane_off = idx & 31; + const size_t byte_off = lane_off * sizeof(float); + return (lane_off + count > 32) + ? Q6_V_valign_VVR(v_src[1], v_src[0], byte_off) + : Q6_V_vror_VR(v_src[0], byte_off); } static __attribute__((noinline)) void fa_build_d_diag_inv_l(struct hmx_fa_context * factx, size_t n_row_tiles, - size_t n_row_tiles_g_br) { + size_t n_row_tiles_g_br, + uint32_t kv_head, + size_t n_rows_g) { + (void) n_row_tiles_g_br; + + // sinks: l += exp(sink - m * ln2) in f32, m and l are in the log2 domain + if (factx->sinks) { + const float * sinks_data = factx->vtcm_sinks; + const size_t G = factx->G; + const size_t head_base = (size_t) kv_head * G; + const HVX_Vector v_ln2 = Q6_V_vsplat_R(EXP_LOGN2); + const HVX_Vector v_head_sinks = (G <= 32) ? hvx_fa_load_sinks(sinks_data, head_base, G) : Q6_V_vzero(); + HVX_Vector v_sink = Q6_V_vzero(); + size_t g_prev = SIZE_MAX; + + for (size_t i = 0; i < n_row_tiles; ++i) { + // the sink pattern of a tile only depends on (i * 32) % G, rebuild it when that changes + const size_t r0 = i * 32; + const size_t g0 = fastmodulo(r0, G, &factx->div_G); + if (g0 != g_prev) { + const size_t rem_g = G - g0; + if (G <= 32) { + const HVX_Vector v_lo = Q6_V_vror_VR(v_head_sinks, g0 * sizeof(float)); + const HVX_Vector v_hi = Q6_V_vror_VR(v_head_sinks, 128 - rem_g * sizeof(float)); + v_sink = Q6_V_vmux_QVV(Q6_Q_vsetq2_R(rem_g * sizeof(float)), v_lo, v_hi); + for (size_t len = G; len < 32; len <<= 1) { + const HVX_VectorPred q_keep = Q6_Q_vsetq_R(len * sizeof(float)); + const HVX_Vector v_shifted = Q6_V_vror_VR(v_sink, 128 - len * sizeof(float)); + v_sink = Q6_V_vmux_QVV(q_keep, v_sink, v_shifted); + } + } else if (rem_g >= 32) { + v_sink = hvx_fa_load_sinks(sinks_data, head_base + g0, 32); + } else { + const HVX_Vector v_lo = hvx_fa_load_sinks(sinks_data, head_base + g0, rem_g); + const HVX_Vector v_hi = hvx_fa_load_sinks(sinks_data, head_base, 32 - rem_g); + v_sink = Q6_V_vmux_QVV(Q6_Q_vsetq_R(rem_g * sizeof(float)), v_lo, Q6_V_vror_VR(v_hi, 128 - rem_g * sizeof(float))); + } + g_prev = g0; + } + + HVX_Vector v_d = HVX_OP_SUB_F32(v_sink, HVX_OP_MUL_F32(factx->vtcm_m_vec[i], v_ln2)); + if (r0 + 32 > n_rows_g) { + const HVX_VectorPred q_valid = Q6_Q_vsetq_R((n_rows_g - r0) * sizeof(float)); + v_d = Q6_V_vmux_QVV(q_valid, v_d, Q6_V_vzero()); + } + factx->vtcm_l_vec[i] = HVX_OP_ADD_F32(factx->vtcm_l_vec[i], hvx_vec_exp_f32(v_d)); + } + } + const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets; const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16)); const HVX_Vector one = hvx_vec_splat_f32(1.0f); @@ -1558,10 +1726,10 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, fa_softmax_args_t * sargs, size_t n_row_tiles, size_t n_row_tiles_g_br) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); - worker_callback_t softmax_fn = fa_softmax_thread; + work_queue_func_t softmax_fn = fa_softmax_thread; if (sargs->mask == NULL && !factx->has_softcap && !sargs->has_alibi) { softmax_fn = fa_softmax_thread_nomask; } else if (sargs->mask != NULL && factx->mask_broadcast && !factx->has_softcap && !sargs->has_alibi) { @@ -1572,13 +1740,12 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, } } - if (factx->n_threads > 1 && n_row_vec_cnt >= 2) { - uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt); + uint32_t n_use = 1; + if (factx->n_hvx > 1 && n_row_vec_cnt >= 2) { + n_use = (uint32_t) hex_smin((size_t) factx->n_hvx, n_row_vec_cnt); sargs->thread_div = init_fastdiv_values(n_use); - work_queue_run(wp, softmax_fn, sargs, n_use); - } else { - softmax_fn(1, 0, sargs); } + work_queue_run(wp, softmax_fn, sargs, n_use); } // ============================================================================ @@ -1596,7 +1763,7 @@ typedef struct { uint8_t * hmx_scales; } hmx_fa_qk_job_t; -static void hmx_fa_qk_dot_worker(void * data) { +static void hmx_fa_qk_dot_worker(const htp_thread_t * th, void * data) { hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_col_tiles = job->n_col_tiles; @@ -1638,7 +1805,7 @@ typedef struct { size_t DV; } hmx_fa_o_update_job_t; -static void hmx_fa_o_update_worker(void * data) { +static void hmx_fa_o_update_worker(const htp_thread_t * th, void * data) { hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_col_tiles = job->n_col_tiles; @@ -1683,7 +1850,7 @@ typedef struct { size_t DV; } hmx_fa_o_norm_job_t; -static void hmx_fa_o_norm_worker(void * data) { +static void hmx_fa_o_norm_worker(const htp_thread_t * th, void * data) { hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; @@ -1725,7 +1892,7 @@ static __attribute__((noinline)) void fa_compute_slopes( const float m0 = factx->m0; const float m1 = factx->m1; - __fp16 temp_slopes[512] __attribute__((aligned(128))); + __fp16 * temp_slopes = factx->ddr_slopes; if (G <= 32) { // Fast path: Compute G unique slope values in vector registers HVX_Vector v_val = hvx_alibi_slopes(kv_head, G, n_head_log2, m0, m1); @@ -1743,14 +1910,13 @@ static __attribute__((noinline)) void fa_compute_slopes( } } - // Allocate stack buffer to avoid scalar writes to VTCM (which generates L2 misses) - __fp16 local_slopes[n_rows_g] __attribute__((aligned(128))); - for (size_t r = 0; r < n_rows_g; ++r) { - local_slopes[r] = temp_slopes[fastmodulo(r, G, &factx->div_G)]; + // Replicate the G values in DDR to avoid scalar writes to VTCM (which generates L2 misses) + for (size_t r = G; r < n_rows_g; ++r) { + temp_slopes[r] = temp_slopes[r - G]; } // Copy to VTCM slopes using HVX block copy (both are aligned to 128 bytes) - hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)local_slopes, n_rows_g); + hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)temp_slopes, n_rows_g); } static void fa_push_mask_dma_gqa( @@ -1796,7 +1962,7 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor if (mask) { if (__builtin_expect(factx->mask_broadcast, true)) { const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16); - dma_cache_fa_push(dma_q, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q); + dma_cache_fa_push(dma_q, factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q); } else { fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx); } @@ -1809,7 +1975,6 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor int hmx_flash_attn_ext(struct htp_ops_context * octx) { struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0]; - struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS]; const struct htp_tensor * q = octx->src[0]; const struct htp_tensor * k = octx->src[1]; const struct htp_tensor * v = octx->src[2]; @@ -1854,7 +2019,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.sinks = octx->src[4]; // NULL if this op has no attention sinks factx.k = k; factx.v = v; - factx.n_threads = kparams->n_threads; + factx.n_hvx = kparams->n_hvx; factx.DK = DK; factx.DV = DV; factx.DK_pad = DK_pad; @@ -1869,8 +2034,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.Bc = kparams->Bc; factx.g_br = kparams->u.hmx.g_br; factx.n_kv_blocks = kparams->n_kv_blocks; - factx.is_q_fp32 = (kparams->is_q_fp32 != 0); - factx.is_dst_fp32 = (kparams->is_dst_fp32 != 0); + factx.is_q_fp32 = (q->type == HTP_TYPE_F32); + factx.is_dst_fp32 = (dst->type == HTP_TYPE_F32); factx.pipeline = (kparams->u.hmx.pipeline != 0); factx.mask_broadcast = (kparams->u.hmx.mask_broadcast != 0); if (mask) { @@ -1879,13 +2044,12 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { } factx.has_softcap = (kparams->logit_softcap != 0.0f); - if (!factx.has_softcap) { - factx.scale = (__fp16) (kparams->scale * EXP_LOG2E_F); // log2(e) - } else { - factx.scale = (__fp16) kparams->scale; - } + factx.scale = (__fp16) kparams->scale; factx.max_bias = kparams->max_bias; - factx.logit_softcap = factx.has_softcap ? (__fp16) (kparams->logit_softcap * EXP_LOG2E_F) : 0; + factx.logit_softcap = 0; + if (factx.has_softcap) { + factx.logit_softcap = (__fp16) kparams->logit_softcap; + } factx.n_head_log2 = kparams->n_head_log2; factx.m0 = kparams->m0; @@ -1895,25 +2059,61 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t Bc = factx.Bc; const uint32_t g_br = factx.g_br; const bool pipeline = factx.pipeline; - const uint32_t n_threads = factx.n_threads; + const uint32_t n_hvx = factx.n_hvx; const uint32_t G = factx.G; - // Multi-device: split Q blocks across devices + // Multi-device: prefer head-parallel partitioning (each core owns a disjoint head + // shard), falling back to Q-block (token) split when heads don't divide evenly. const uint32_t n_q_blocks = (neq1 + Br - 1) / Br; - uint32_t q_start_min = 0; - uint32_t q_start_max = neq1; + uint32_t q_start_min = 0; + uint32_t q_start_max = neq1; + uint32_t kv_head_min = 0; + uint32_t kv_head_max = n_kv_heads; if (octx->ctx->mdev.count > 1) { - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_q_blocks, htp_tensor_mdev_data_aligned(dst) ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); - const uint32_t block_start = range.start; - const uint32_t block_end = range.start + range.count; + const uint32_t mdev_count = octx->ctx->mdev.count; + const uint32_t mdev_idx = octx->ctx->mdev.idx; + const uint32_t dst_e_size = (dst->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(__fp16); + const bool can_split = htp_tensor_can_row_partition(dst, dst_e_size); + + if ((kparams->flags & HTP_FA_FLAG_TEAM_SPLIT) && can_split) { + // KV heads split across teams, Q-blocks split within a team + uint32_t n_teams, team_size; + htp_fa_mdev_teams(n_kv_heads, mdev_count, kparams->head_split != 0, &n_teams, &team_size); + + const uint32_t team = mdev_idx / team_size; + const uint32_t member = mdev_idx - team * team_size; + const uint32_t kv_per_team = n_kv_heads / n_teams; + kv_head_min = team * kv_per_team; + kv_head_max = kv_head_min + kv_per_team; + + const uint32_t base = n_q_blocks / team_size; + const uint32_t rem = n_q_blocks - base * team_size; + const uint32_t b0 = member * base + MIN(member, rem); + const uint32_t nblk = base + (member < rem ? 1 : 0); + + if (nblk == 0) { + return HTP_STATUS_OK; + } - if (block_start >= block_end) { - return HTP_STATUS_OK; - } + q_start_min = b0 * Br; + q_start_max = MIN((b0 + nblk) * Br, neq1); + } else if (kparams->head_split && can_split && n_kv_heads >= mdev_count && n_kv_heads % mdev_count == 0) { + const uint32_t kv_per_core = n_kv_heads / mdev_count; + kv_head_min = mdev_idx * kv_per_core; + kv_head_max = kv_head_min + kv_per_core; + } else { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_q_blocks, can_split ? 1 : 0, mdev_idx, mdev_count, &octx->ctx->mdev.count_div); + const uint32_t block_start = range.start; + const uint32_t block_end = range.start + range.count; + + if (block_start >= block_end) { + return HTP_STATUS_OK; + } - q_start_min = block_start * Br; - q_start_max = MIN(block_end * Br, neq1); + q_start_min = block_start * Br; + q_start_max = MIN(block_end * Br, neq1); + } } // ======== VTCM allocation (GQA-aware) ======== @@ -1929,7 +2129,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // Build the VTCM layout once (shared with the host estimator) and place every // scratch buffer at its computed offset. Padded head dims size the HMX tiles. struct hmx_fa_vtcm_layout L; - hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_threads, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads); + hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_hvx, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads); if (L.total_bytes > ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -1967,6 +2167,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.vtcm_mask_buf = VTCM_LAYOUT_PTR(__fp16, base, L.off_mask_buf); factx.mask_buf_row_stride = L.mask_buf_row_stride; factx.q_tile_bytes = L.q_tile_bytes; + factx.k_tile_bytes = L.k_tile_bytes; factx.o_tile_bytes = L.o_tile_bytes; factx.col_vec_bytes = L.col_vec_bytes; factx.d_tile_bytes = L.d_tile_bytes; @@ -1975,7 +2176,14 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop - dma_cache_fa_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE); + struct hmx_fa_ddr_spad * ddr = (struct hmx_fa_ddr_spad *) fa_get_ddr_spad(octx, sizeof(struct hmx_fa_ddr_spad) + factx.g_br * sizeof(__fp16)); + if (!ddr) { + return HTP_STATUS_INTERNAL_ERR; + } + + factx.ddr_slopes = ddr->slopes; + factx.m_cache = &octx->dma_cache_fa[0]; + dma_cache_fa_init(factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE); // Head-dim padding: the K/V DMA staging buffers and the flat-Q buffer are laid out // with padded row strides (size_{k,v,q}_row_padded, covering D_pad columns) but the @@ -1997,7 +2205,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // ======== Initialize HMX output scales ======== hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0 - hmx_init_column_scales(factx.vtcm_hmx_scales_qk, hvx_vec_splat_f16(factx.scale)); + { + union { __fp16 f; uint16_t u; } sc = { .f = factx.scale }; + hmx_init_column_scales(factx.vtcm_hmx_scales_qk, Q6_V_vsplat_R(sc.u)); + } // ======== DMA setup ======== dma_queue * const dma_q = ctx->dma[0]; @@ -2016,7 +2227,6 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const size_t q_src_stride = q_transposed ? q->nb[2] : q->nb[1]; const size_t q_row_bytes_untransposed = factx.G * factx.DK * qo_element_size; const size_t q_row_bytes_trans_factor = factx.DK * qo_element_size; - const uint32_t kv_rows0 = hex_smin(Bc, nek1); // ======== Reusable job descriptors for pipeline ======== hmx_fa_qk_job_t qk_job; @@ -2032,34 +2242,51 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS); const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS; - for (uint32_t kv_head = 0; kv_head < n_kv_heads; ++kv_head) { + uint16_t * active_blocks = ddr->active_blocks; + uint32_t n_active_blocks = factx.n_kv_blocks; + if (mask && factx.mask_broadcast && factx.n_kv_blocks > 1) { + const size_t scratch_bytes = factx.k_tile_bytes * (factx.pipeline ? 2 : 1); + n_active_blocks = fa_scan_active_blocks( + dma_q, mask, q_start, n_rows_q, 0, im3, nek1, Bc, + factx.n_kv_blocks, factx.vtcm_k_tiles[0], scratch_bytes, active_blocks); + } else { + #pragma clang loop vectorize(disable) + for (uint32_t b = 0; b < factx.n_kv_blocks; ++b) { + active_blocks[b] = (uint16_t) b; + } + } + const uint32_t b0 = (n_active_blocks > 0) ? active_blocks[0] : 0; + const uint32_t b0_start = b0 * Bc; + const uint32_t b0_rows = hex_smin(Bc, nek1 - b0_start); + + for (uint32_t kv_head = kv_head_min; kv_head < kv_head_max; ++kv_head) { const uint32_t ik2 = kv_head; const uint32_t ik3 = fastdiv(ib3, &kparams->broadcast_rk3); const uint32_t iv2 = kv_head; const uint32_t iv3 = fastdiv(ib3, &kparams->broadcast_rv3); - // 1. Push Q and KV DMAs for the very first iteration. - // Subsequent iterations are enqueued early at the end of the previous iteration. - if (ib3 == 0 && q_start == q_start_min && kv_head == 0) { + // 1. Push Q and KV DMAs for the first kv_head of this q_start block. + // Subsequent kv_head iterations are enqueued early at the end of the previous iteration. + if (kv_head == kv_head_min) { const dma_addr_t q_ptr = q->data + q_start * q->nb[1] + (kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3]; const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; const size_t n_rows = q_transposed ? factx.G : n_rows_q; dma_queue_push(dma_q, dma_make_data(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows); - if (factx.n_kv_blocks > 0) { - const dma_addr_t k_src = k->data + ik2 * k->nb[2] + ik3 * k->nb[3]; - dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0); + if (n_active_blocks > 0) { + const dma_addr_t k_src = k->data + b0_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, b0_rows); - const dma_addr_t v_src = v->data + iv2 * v->nb[2] + iv3 * v->nb[3]; - dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0); + const dma_addr_t v_src = v->data + b0_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, b0_rows); if (factx.pipeline && mask) { if (__builtin_expect(factx.mask_broadcast, true)) { - const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + 0; - dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q); + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16); + dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q); } else { - fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, 0, kv_head, G, m_line_bytes, kv_rows0, n_rows_q, &factx); + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx); } } } @@ -2091,27 +2318,28 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { hmx_fa_qk_job_t qk_job[2]; hmx_fa_o_update_job_t ou_job[2]; - // Prefetch block 1 early if there are multiple blocks - if (factx.n_kv_blocks > 1) { - fa_prefetch_block(dma_q, k, v, mask, 1, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, + // Prefetch second active block early if there are multiple active blocks + if (n_active_blocks > 1) { + fa_prefetch_block(dma_q, k, v, mask, active_blocks[1], Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, 1, &factx); } // Prep and start QK-dot(0) void * curr_k0 = (void *) dma_queue_pop(dma_q).dst; - fa_phase_k_interleave(&factx, kv_rows0, k_src_stride, curr_k0, 0, 0); + fa_phase_k_interleave(&factx, b0_rows, k_src_stride, curr_k0, b0_start, 0); qk_job[0].q_tiles = factx.vtcm_q_tiles; qk_job[0].k_tiles = factx.vtcm_k_tiles[0]; qk_job[0].s_tiles = factx.vtcm_s_tiles[0]; qk_job[0].n_row_tiles = n_row_tiles; - qk_job[0].n_col_tiles = hmx_ceil_div(kv_rows0, HMX_FP16_TILE_N_COLS); + qk_job[0].n_col_tiles = hmx_ceil_div(b0_rows, HMX_FP16_TILE_N_COLS); qk_job[0].n_dot_tiles = DK_pad / 32; qk_job[0].n_tiles_per_bc = n_tiles_per_bc; qk_job[0].hmx_scales = factx.vtcm_hmx_scales_qk; hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[0])); - for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) { + for (uint32_t step = 0; step < n_active_blocks; ++step) { + const uint32_t kv_blk = active_blocks[step]; const uint32_t kv_start = kv_blk * Bc; const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); @@ -2131,13 +2359,14 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { } } - // ---- 3. Start HMX O update for block kv_blk - 1 (reads P[1 - buf_idx], V[1 - buf_idx], D) ---- - // O update relys on the previous block's P and V tiles. + // ---- 3. Start HMX O update for step - 1 (reads P[1 - buf_idx], V[1 - buf_idx], D) ---- + // O update relies on the previous block's P and V tiles. // O update MUST be pushed before the next block's QK-dot: hmx_queue_pop() retires the // oldest descriptor, so push order alone decides which pop waits for which job. // If OU went in after QK(i+1), the pop below would retire QK(i+1) and leave // OU(i-1) in flight into the next iteration, where V-prep overwrites V[prev_buf]. - if (kv_blk > 0) { + if (step > 0) { + const uint32_t prev_blk = active_blocks[step - 1]; const size_t prev_buf = 1 - buf_idx; ou_job[prev_buf].o_curr = o_tile_curr; ou_job[prev_buf].o_prev = o_tile_prev; @@ -2147,16 +2376,17 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { ou_job[prev_buf].hmx_scales = factx.vtcm_hmx_scales_id; ou_job[prev_buf].n_row_tiles = n_row_tiles; ou_job[prev_buf].n_col_tiles = - hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS); + hmx_ceil_div(hex_smin(Bc, nek1 - prev_blk * Bc), HMX_FP16_TILE_N_COLS); ou_job[prev_buf].n_row_tiles_g_br = n_row_tiles_g_br; ou_job[prev_buf].n_tiles_per_bc = n_tiles_per_bc; ou_job[prev_buf].DV = DV_pad; hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[prev_buf])); } - // ---- 4. Pop and run K-prep for next block & push next QK-dot ---- - if (kv_blk + 1 < factx.n_kv_blocks) { - const uint32_t next_start = (kv_blk + 1) * Bc; + // ---- 4. Pop and run K-prep for next active block & push next QK-dot ---- + if (step + 1 < n_active_blocks) { + const uint32_t next_blk = active_blocks[step + 1]; + const uint32_t next_start = next_blk * Bc; const uint32_t next_rows = hex_smin(Bc, nek1 - next_start); const size_t next_buf = 1 - buf_idx; @@ -2203,15 +2433,15 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // Run Softmax on HVX (blocking call) fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); - // Wait for HMX O update for block kv_blk - 1 to finish - if (kv_blk > 0) { + // Wait for HMX O update for step - 1 to finish + if (step > 0) { hmx_queue_pop(hmx_q); hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); } - // Prefetch block kv_blk + 2 - if (kv_blk + 2 < factx.n_kv_blocks) { - fa_prefetch_block(dma_q, k, v, mask, kv_blk + 2, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, + // Prefetch block step + 2 + if (step + 2 < n_active_blocks) { + fa_prefetch_block(dma_q, k, v, mask, active_blocks[step + 2], Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, buf_idx, &factx); } @@ -2219,8 +2449,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { } // Epilogue - if (factx.n_kv_blocks > 0) { - const uint32_t last_blk = factx.n_kv_blocks - 1; + if (n_active_blocks > 0) { + const uint32_t last_blk = active_blocks[n_active_blocks - 1]; const size_t last_cols = hmx_ceil_div(hex_smin(Bc, nek1 - last_blk * Bc), HMX_FP16_TILE_N_COLS); ou_job[0].o_curr = o_tile_curr; ou_job[0].o_prev = o_tile_prev; @@ -2237,7 +2467,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // Overlapped: run HVX build diag inv L while HMX is busy executing the update htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); - fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br); + fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br, kv_head, n_rows_g); htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); hmx_queue_pop(hmx_q); @@ -2246,7 +2476,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { } else { // Fallback path - for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) { + for (uint32_t step = 0; step < n_active_blocks; ++step) { + const uint32_t kv_blk = active_blocks[step]; const uint32_t kv_start = kv_blk * Bc; const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); @@ -2254,14 +2485,15 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { if (mask) { if (__builtin_expect(factx.mask_broadcast, true)) { const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); - dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); + dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); } else { fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx); } } - if (kv_blk + 1 < factx.n_kv_blocks) { - const uint32_t prefetch_start = (kv_blk + 1) * Bc; + if (step + 1 < n_active_blocks) { + const uint32_t prefetch_blk = active_blocks[step + 1]; + const uint32_t prefetch_start = prefetch_blk * Bc; const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); const size_t prefetch_buf = 1 - buf_idx; const dma_addr_t k_prefetch_src = k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; @@ -2339,10 +2571,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { ou_job.DV = DV_pad; hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); - if (kv_blk + 1 == factx.n_kv_blocks) { + if (step + 1 == n_active_blocks) { // Overlapped: run HVX build diag inv L while HMX is busy executing the update htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); - fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br); + fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br, kv_head, n_rows_g); htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); } hmx_queue_pop(ctx->hmx_queue); @@ -2354,53 +2586,32 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { } } - // Enqueue DMAs for the next iteration early so they overlap with O-PROC - uint32_t next_kv_head = kv_head + 1; - uint32_t next_q_start = q_start; - uint32_t next_ib3 = ib3; - if (next_kv_head >= n_kv_heads) { - next_kv_head = 0; - next_q_start = q_start + Br; - if (next_q_start >= q_start_max) { - next_q_start = q_start_min; - next_ib3 = ib3 + 1; - } - } - bool has_next = (next_ib3 < neq3); + // Enqueue DMAs for the next kv_head early so they overlap with O-PROC + const uint32_t next_kv_head = kv_head + 1; + const bool has_next = (next_kv_head < kv_head_max); if (has_next) { - const uint32_t next_n_rows_q = hex_smin(Br, neq1 - next_q_start); - const dma_addr_t next_q_ptr = q->data + next_q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + next_ib3 * q->nb[3]; - const size_t next_q_row_bytes = q_transposed ? next_n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; - const size_t next_n_rows = q_transposed ? factx.G : next_n_rows_q; + const dma_addr_t next_q_ptr = q->data + q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3]; + const size_t next_q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; + const size_t next_n_rows = q_transposed ? factx.G : n_rows_q; dma_queue_push(dma_q, dma_make_data(factx.vtcm_q_dma, next_q_ptr), next_q_row_bytes, hex_smax(q_src_stride, next_q_row_bytes), next_q_row_bytes, next_n_rows); - if (factx.n_kv_blocks > 0) { + if (n_active_blocks > 0) { const uint32_t next_ik2 = next_kv_head; const uint32_t next_iv2 = next_kv_head; - uint32_t next_ik3 = ik3; - uint32_t next_iv3 = iv3; - if (next_ib3 != ib3) { - next_ik3 = fastdiv(next_ib3, &kparams->broadcast_rk3); - next_iv3 = fastdiv(next_ib3, &kparams->broadcast_rv3); - } - const dma_addr_t next_k_src = k->data + next_ik2 * k->nb[2] + next_ik3 * k->nb[3]; - dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0); + const dma_addr_t next_k_src = k->data + b0_start * k->nb[1] + next_ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, b0_rows); - const dma_addr_t next_v_src = v->data + next_iv2 * v->nb[2] + next_iv3 * v->nb[3]; - dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0); + const dma_addr_t next_v_src = v->data + b0_start * v->nb[1] + next_iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, b0_rows); if (factx.pipeline && mask) { - uint32_t next_im3 = im3; - if (next_ib3 != ib3) { - next_im3 = fastmodulo(next_ib3, mask->ne[3], &factx.src3_div3); - } if (__builtin_expect(factx.mask_broadcast, true)) { - const dma_addr_t ms_src = mask->data + next_q_start * mask->nb[1] + next_im3 * mask->nb[3] + 0; - dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q); + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16); + dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q); } else { - fa_push_mask_dma_gqa(dma_q, mask, next_q_start, next_im3, 0, next_kv_head, G, m_line_bytes, kv_rows0, next_n_rows_q, &factx); + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, next_kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx); } } } @@ -2448,11 +2659,11 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params; - if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED) { + if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED || kparams->n_kv_blocks > FA_MAX_KV_BLOCKS) { return HTP_STATUS_NO_SUPPORT; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -2478,7 +2689,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { factx.src3_div3 = kparams->src3_div3; } - factx.is_q_fp32 = (kparams->is_q_fp32 != 0); + factx.is_q_fp32 = (q->type == HTP_TYPE_F32); factx.size_q_row_padded = kparams->u.hvx.size_q_row_padded; factx.size_k_row_padded = kparams->u.hvx.size_k_row_padded; factx.size_v_row_padded = kparams->u.hvx.size_v_row_padded; @@ -2488,7 +2699,10 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { factx.scale = kparams->scale; factx.max_bias = kparams->max_bias; factx.has_softcap = (kparams->logit_softcap != 0.0f); - factx.logit_softcap = factx.has_softcap ? (__fp16) kparams->logit_softcap : 0; + factx.logit_softcap = 0; + if (factx.has_softcap) { + factx.logit_softcap = (__fp16) kparams->logit_softcap; + } factx.n_head_log2 = kparams->n_head_log2; factx.m0 = kparams->m0; @@ -2498,8 +2712,15 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { if (n_head > 512) { return HTP_STATUS_NO_SUPPORT; } + + factx.ddr = (struct hvx_fa_ddr_spad *) fa_get_ddr_spad(octx, sizeof(struct hvx_fa_ddr_spad)); + if (!factx.ddr) { + return HTP_STATUS_INTERNAL_ERR; + } + factx.m_cache = octx->dma_cache_dm; + for (uint32_t h = 0; h < n_head; ++h) { - factx.slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f); + factx.ddr->slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f); } // total rows in q @@ -2512,26 +2733,41 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { uint32_t qrows = total_qrows; if (octx->ctx->mdev.count > 1) { - const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_qrows, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); - qrow_start = range.start; - qrows = range.count; + const uint32_t mdev_count = octx->ctx->mdev.count; + const uint32_t mdev_idx = octx->ctx->mdev.idx; + const uint32_t n_kv_heads = k->ne[2]; + const uint32_t dst_e_size = (dst->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(__fp16); + const bool can_split = htp_tensor_can_row_partition(dst, dst_e_size); + + // head range is contiguous in flat row space only when neq3 == 1 + if (kparams->head_split && can_split && neq3 == 1 && n_kv_heads >= mdev_count && n_kv_heads % mdev_count == 0) { + const uint32_t G = kparams->G; + const uint32_t kv_per_core = n_kv_heads / mdev_count; + const uint32_t heads_per_core = kv_per_core * G; + const uint32_t head_start = mdev_idx * heads_per_core; + qrow_start = head_start * neq1; + qrows = heads_per_core * neq1; + } else { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_qrows, can_split ? 1 : 0, mdev_idx, mdev_count, &octx->ctx->mdev.count_div); + qrow_start = range.start; + qrows = range.count; + } } if (qrows == 0) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; factx.qrows = qrows; factx.qrow_start = qrow_start; - factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div); + factx.qrows_per_hvx = fastdiv(qrows + n_hvx - 1, &octx->n_hvx_div); const bool has_mask = (mask != NULL); const bool has_sinks = (octx->src[4] != NULL); struct hvx_fa_vtcm_layout L; - hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_threads); + hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_hvx); if (L.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -2545,12 +2781,21 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { uint8_t * const base = octx->ctx->vtcm_base; - factx.spad_q = VTCM_LAYOUT_PTR(uint8_t, base, L.off_q); - factx.spad_k = VTCM_LAYOUT_PTR(uint8_t, base, L.off_k); - factx.spad_v = VTCM_LAYOUT_PTR(uint8_t, base, L.off_v); - factx.spad_m = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_m, has_mask); - factx.spad_a = VTCM_LAYOUT_PTR(uint8_t, base, L.off_a); - factx.spad_sinks = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_sinks, has_sinks); + factx.spad_q = VTCM_LAYOUT_PTR(uint8_t, base, L.off_q); + factx.spad_k = VTCM_LAYOUT_PTR(uint8_t, base, L.off_k); + factx.spad_v = VTCM_LAYOUT_PTR(uint8_t, base, L.off_v); + factx.spad_m = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_m, has_mask); + factx.spad_a = VTCM_LAYOUT_PTR(uint8_t, base, L.off_a); + factx.spad_sinks = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_sinks, has_sinks); + factx.n_active_blocks = factx.n_blocks; + factx.shared_active_blocks = false; + + if (has_mask && neq1 == 1 && neq3 == 1 && mask->ne[2] == 1 && factx.n_blocks > 1 && factx.n_blocks <= FA_MAX_KV_BLOCKS) { + factx.n_active_blocks = fa_scan_active_blocks( + octx->ctx->dma[0], mask, 0, 1, 0, 0, k->ne[1], FLASH_ATTN_BLOCK_SIZE, + factx.n_blocks, factx.spad_k, factx.size_k_block * 2, factx.ddr->active_blocks); + factx.shared_active_blocks = true; + } if (has_sinks) { const struct htp_tensor * sinks = octx->src[4]; @@ -2559,7 +2804,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { dma_queue_pop(dma_q); } - work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads); + work_queue_run(octx->ctx->hvx_queue, flash_attn_ext_f16_thread, &factx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h index 22bb8c53d..70cf3e41a 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -32,11 +32,21 @@ enum htp_fa_kernel_type { HTP_FA_KERNEL_HMX }; +// Multicore: gcd(n_kv_heads, N) teams own the KV heads, team members share the Q-blocks +#define HTP_FA_FLAG_TEAM_SPLIT 0x01 + +static inline void htp_fa_mdev_teams(uint32_t n_kv_heads, uint32_t mdev_count, bool head_split, + uint32_t * n_teams, uint32_t * team_size) { + const uint32_t t = (head_split && n_kv_heads > 0 && mdev_count > 1) ? hex_gcd_u32(n_kv_heads, mdev_count) : 1; + *n_teams = t; + *team_size = mdev_count > 1 ? mdev_count / t : 1; +} + struct htp_fa_kernel_params { uint8_t kernel_type; // enum htp_fa_kernel_type - uint8_t is_q_fp32; // 1 = Q type is F32, 0 = F16 - uint8_t is_dst_fp32; // 1 = dst type is F32, 0 = F16 - uint8_t n_threads; // Number of threads to run + uint8_t head_split; // 1 = partition by KV heads in multicore, 0 = token partition + uint8_t flags; // HTP_FA_FLAG_* + uint8_t n_hvx; // Number of threads to run // Common parameters uint16_t Br; @@ -49,9 +59,6 @@ struct htp_fa_kernel_params { float logit_softcap; uint32_t vtcm_size; - uint32_t qrows; - uint32_t qrows_per_thread; - uint32_t qrow_start; float m0; float m1; uint32_t n_head_log2; @@ -67,8 +74,6 @@ struct htp_fa_kernel_params { union { struct { uint32_t g_br; - uint32_t row_buf_stride; - uint32_t mask_buf_row_stride; int32_t mask_broadcast; int32_t pipeline; struct fastdiv_values div_G; @@ -85,6 +90,8 @@ struct htp_fa_kernel_params { #if defined(__cplusplus) static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob"); #endif // VTCM region layout for the HMX flash-attention kernel. @@ -126,6 +133,7 @@ struct hmx_fa_vtcm_layout { // Region byte sizes reused by the device at runtime (not just for allocation). size_t q_tile_bytes; size_t o_tile_bytes; + size_t k_tile_bytes; size_t s_tile_bytes; // S and P tiles (same size) size_t d_tile_bytes; // d_tiles[0..1] + d_inv_l, allocated back to back size_t m_line_bytes; // one mask row @@ -141,11 +149,13 @@ struct hmx_fa_vtcm_layout { size_t total_bytes; }; +#define FA_MAX_KV_BLOCKS 1024 + // Build the VTCM layout. static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, size_t gqa_factor, size_t DK, size_t DV, - size_t Br, size_t Bc, size_t n_threads, + size_t Br, size_t Bc, size_t n_hvx, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS); @@ -197,7 +207,7 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_p_tiles[1], s_tile_size, pipeline); VTCM_LAYOUT_ALLOC(off_group_b, off_s_rowmax, col_vec_size); VTCM_LAYOUT_ALLOC(off_group_b, off_p_rowsum, col_vec_size); - VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_threads); + VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_hvx); const size_t group_b_size = off_group_b - off_group_b_c; @@ -224,6 +234,7 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, L->q_tile_bytes = q_tile_size; L->o_tile_bytes = o_tile_size; + L->k_tile_bytes = k_tile_size; L->col_vec_bytes = col_vec_size; L->s_tile_bytes = s_tile_size; // Measured from the actual offsets rather than assumed to be N * d_tile_size, so @@ -240,9 +251,9 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, } // Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration. -static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { +static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_hvx, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { struct hmx_fa_vtcm_layout L; - hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline, is_q_fp32, has_sinks, n_heads); + hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_hvx, pipeline, is_q_fp32, has_sinks, n_heads); return L.total_bytes; } @@ -271,7 +282,7 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, - size_t n_threads) { + size_t n_hvx) { const size_t size_q_row_padded = hex_round_up(DK * (is_q_fp32 ? 4 : 2), 128); const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128); const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128); @@ -285,11 +296,11 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_threads); - VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_threads); - VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_threads); - VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_threads, has_mask); - VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_threads); + VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_hvx, has_mask); + VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_hvx); VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, size_sinks, has_sinks); L->size_q_block = size_q_block; @@ -301,9 +312,9 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, L->total_bytes = off; } -static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_threads) { +static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_hvx) { struct hvx_fa_vtcm_layout L; - hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_threads); + hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_hvx); return L.total_bytes; } @@ -318,22 +329,27 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, size_t qo_len, size_t kv_len, size_t vtcm_budget, - size_t n_threads, + size_t n_hvx, bool is_q_fp32, bool has_sinks, - size_t n_heads) { + size_t n_heads, + size_t q_split) { const size_t T = HMX_FP16_TILE_N_ROWS; // 32 const size_t br_unit = hmx_ceil_div(T, gqa_factor); const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64 - const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_threads >= 2); + const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_hvx >= 2); // Br_max: largest Br aligned to br_unit that does not exceed qo_len. const size_t Br_max = qo_len >= br_unit ? hex_align_down(qo_len, br_unit) : br_unit; // Pipeline constraint: cap Bc so n_kv_blocks >= FA_MIN_KV_BLOCKS. // Only relax when kv_len is too short to form enough blocks. - const size_t Bc_limit = can_pipeline ? hex_align_down(kv_len / FA_MIN_KV_BLOCKS, bc_unit) : - (kv_len >= bc_unit ? hex_align_down(kv_len, bc_unit) : bc_unit); + size_t Bc_limit = can_pipeline ? hex_align_down(kv_len / FA_MIN_KV_BLOCKS, bc_unit) : + (kv_len >= bc_unit ? hex_align_down(kv_len, bc_unit) : bc_unit); + // For small-Q SWA-sized KV caches (qo_len <= 16, kv_len <= 2048), cap Bc at 256 for block-skipping granularity + if (qo_len <= 16 && kv_len <= 2048 && Bc_limit > 256) { + Bc_limit = 256; + } // Cost coefficients calibrated from profiling const size_t c_q_fixed = 800; // per-Q-block: q_load + epilogue o_update + o_norm + o_store const size_t c_iter_base = 200; // per-KV-iter base (HMX dot/update + DMA) @@ -345,19 +361,20 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) { // Try all Bc candidates from Bc_limit down to bc_unit for (size_t Bc = Bc_limit; Bc >= bc_unit; Bc -= bc_unit) { - size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline, is_q_fp32, has_sinks, n_heads); + size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_hvx, can_pipeline, is_q_fp32, has_sinks, n_heads); if (vtcm_needed <= vtcm_budget) { // This Bc fits for this Br! const size_t q_blocks = (qo_len + Br - 1) / Br; const size_t kv_blocks = (kv_len + Bc - 1) / Bc; - const size_t actual_threads = (kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1; + const size_t actual_hvx = (kv_blocks >= 3 && n_hvx >= 2) ? n_hvx : 1; const size_t n_rows_g = Br * gqa_factor; const size_t n_row_vec_cnt = (n_rows_g + 63) / 64; - const size_t n_use = n_row_vec_cnt < actual_threads ? n_row_vec_cnt : actual_threads; + const size_t n_use = n_row_vec_cnt < actual_hvx ? n_row_vec_cnt : actual_hvx; const size_t vecs_per_t = n_use > 0 ? (n_row_vec_cnt + n_use - 1) / n_use : 1; const size_t c_iter_actual = c_iter_base + c_softmax * vecs_per_t; - const size_t cost = q_blocks * (c_q_fixed + kv_blocks * c_iter_actual); + const size_t q_blocks_nsp = (q_blocks + q_split - 1) / (q_split ? q_split : 1); + const size_t cost = q_blocks_nsp * (c_q_fixed + kv_blocks * c_iter_actual); const size_t mn = Br * Bc; if (cost < best_cost || (cost == best_cost && mn > best_mn)) { diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c index 1dd828db7..0280f99a4 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c @@ -748,7 +748,7 @@ static inline void gdn_step_scalar_f32( } } -static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, void * data) { +static void gated_delta_net_f32_pp_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; struct htp_ops_context * octx = gctx->octx; const struct htp_gdn_kernel_params * kparams = gctx->kparams; @@ -777,10 +777,10 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo float * dst_base = (float *) (uintptr_t) dst->data; float * state_out_base = dst_cache ? (float *) (uintptr_t) dst_cache->data : (dst_base + S_v * H * n_tokens * n_seqs); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct htp_gdn_vtcm_layout * layout = &gctx->layout; float * s_work[2]; - s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_hvx * ith); s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); const struct fastdiv_values * fd_H = &kparams->div_H; @@ -817,7 +817,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; int curr_spad_idx = 0; for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { @@ -892,7 +892,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo dma_queue_flush(dma_q); } -static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, void * data) { +static void gated_delta_net_f32_tg_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; struct htp_ops_context * octx = gctx->octx; const struct htp_gdn_kernel_params * kparams = gctx->kparams; @@ -918,10 +918,10 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const float scale = kparams->scale; float * dst_base = (float *) (uintptr_t) dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct htp_gdn_vtcm_layout * layout = &gctx->layout; float * s_work[2]; - s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_hvx * ith); s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); const struct fastdiv_values * fd_H = &kparams->div_H; @@ -957,7 +957,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; int curr_spad_idx = 0; for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { @@ -1031,7 +1031,7 @@ struct htp_gdn_hmx_gemm_task { uint8_t * hmx_scales; }; -static void htp_gdn_hmx_gemm_worker(void * data) { +static void htp_gdn_hmx_gemm_worker(const htp_thread_t * th, void * data) { struct htp_gdn_hmx_gemm_task * task = (struct htp_gdn_hmx_gemm_task *) data; asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)task->hmx_scales)); @@ -1456,9 +1456,10 @@ static inline void gdn_dma_push_chunk_gb( const dma_addr_t g_dma = g->data + (uint64_t) iv3 * g->nb[3] + (uint64_t) t_chunk * g->nb[2] + (uint64_t) iv1 * g->nb[1]; const dma_addr_t beta_dma = beta->data + (uint64_t) iv3 * beta->nb[3] + (uint64_t) t_chunk * beta->nb[2] + (uint64_t) iv1 * beta->nb[1]; const uint32_t row_bytes = n_batch * sizeof(float); + const uint32_t dst_stride = (n_batch > 1) ? row_bytes + sizeof(float) : row_bytes; - dma_queue_push(dma_q, dma_make_data(vtcm_g_raw, g_dma), row_bytes, g->nb[2], row_bytes, chunk_size); - dma_queue_push(dma_q, dma_make_data(vtcm_b_raw, beta_dma), row_bytes, beta->nb[2], row_bytes, chunk_size); + dma_queue_push(dma_q, dma_make_data(vtcm_g_raw, g_dma), dst_stride, g->nb[2], row_bytes, chunk_size); + dma_queue_push(dma_q, dma_make_data(vtcm_b_raw, beta_dma), dst_stride, beta->nb[2], row_bytes, chunk_size); } static inline void gdn_pack_s_col_tiles( @@ -1622,7 +1623,7 @@ static inline void gdn_init_head_ptrs( head->rows_inv = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_inv + h * row_vecs_sz); head->rows_a = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_a + h * row_vecs_sz); - head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_thread_scratch + h * (64 * 128)); + head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_hvx_scratch + h * (64 * 128)); head->vtcm_tmp = head->vtcm_m + 32; head->iv1 = base_iv1 + h; @@ -1653,10 +1654,10 @@ struct htp_gdn_batch_context { const struct htp_gdn_kernel_params * kparams; }; -static void gdn_hvx_init_state_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_init_state_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); struct htp_gdn_head_ptrs * head = &bctx->heads[i]; @@ -1689,10 +1690,10 @@ static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) { return hvx_vec_repl4(Q6_V_vror_VR(v, 124)); } -static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase1a_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); @@ -1707,19 +1708,16 @@ static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) hvx_vmem(head->b_f32[curr_buf] + 0) = hvx_vmem(bctx->vtcm_b_raw + 0); hvx_vmem(head->b_f32[curr_buf] + 32) = hvx_vmem(bctx->vtcm_b_raw + 32); } else { - int32_t offsets[32] __attribute__((aligned(128))); - for (int k = 0; k < 32; ++k) { - offsets[k] = k * n_batch * sizeof(float); - } - HVX_Vector vv = *(const HVX_Vector *) offsets; + const uint32_t vtcm_stride = (n_batch + 1) * sizeof(float); + const HVX_Vector vv = hvx_vec_gather_offsets_w(vtcm_stride); const size_t rt_g = (size_t) ((const uint8_t *) bctx->vtcm_g_raw + i * sizeof(float)); const size_t rt_b = (size_t) ((const uint8_t *) bctx->vtcm_b_raw + i * sizeof(float)); - const size_t mu = 64 * n_batch * sizeof(float); + const size_t mu = 64 * vtcm_stride; Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 0), rt_g, mu, vv); - Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 32), rt_g + 32 * n_batch * sizeof(float), mu, vv); + Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 32), rt_g + 32 * vtcm_stride, mu, vv); Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 0), rt_b, mu, vv); - Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 32), rt_b + 32 * n_batch * sizeof(float), mu, vv); + Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 32), rt_b + 32 * vtcm_stride, mu, vv); } const uint32_t t_chunk = bctx->c * 64; @@ -1765,10 +1763,10 @@ static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); } -static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase1b_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); @@ -1862,10 +1860,10 @@ static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); } -static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase2_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); @@ -1886,10 +1884,10 @@ static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); } -static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase3_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); @@ -1930,10 +1928,10 @@ static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); } -static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase4_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); @@ -1961,10 +1959,10 @@ static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); } -static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase5_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); @@ -1986,10 +1984,10 @@ static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); } -static void gdn_hvx_phase6_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase6_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_STATE, info); @@ -2042,7 +2040,7 @@ static int gated_delta_net_f32_hmx_chunked( const uint32_t n_sv_tiles = S_v / 32; struct htp_gdn_hmx_vtcm_layout L; - htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_threads, kparams->pipeline != 0); + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_hvx, kparams->pipeline != 0); if (L.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -2064,7 +2062,7 @@ static int gated_delta_net_f32_hmx_chunked( hmx_queue_t hmx_q = octx->ctx->hmx_queue; dma_queue * dma_q = octx->ctx->dma[0]; - work_queue_t wp = octx->ctx->work_queue; + work_queue_t wp = octx->ctx->hvx_queue; struct htp_gdn_head_ptrs heads[8]; struct htp_gdn_hmx_gemm_task gemm_tasks[8][9]; @@ -2270,46 +2268,9 @@ static int gated_delta_net_f32_hmx_chunked( } int op_gated_delta_net(struct htp_ops_context * octx) { - const struct htp_tensor * q = octx->src[0]; - const struct htp_tensor * k = octx->src[1]; - const struct htp_tensor * v = octx->src[2]; - const struct htp_tensor * g = octx->src[3]; - const struct htp_tensor * beta = octx->src[4]; - const struct htp_tensor * state = octx->src[5]; - const struct htp_tensor * dst = octx->dst; + const struct htp_tensor * dst = octx->dst; - if (q->type != HTP_TYPE_F32 || k->type != HTP_TYPE_F32 || v->type != HTP_TYPE_F32 || - g->type != HTP_TYPE_F32 || beta->type != HTP_TYPE_F32 || state->type != HTP_TYPE_F32 || - dst->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; - } - - const uint32_t S_v = v->ne[0]; - const uint32_t H = v->ne[1]; - const uint32_t n_tokens = v->ne[2]; - const uint32_t n_seqs = v->ne[3]; - const uint32_t K = octx->op_params[0]; - - if (S_v == 0 || S_v > HTP_GDN_MAX_SV || H == 0 || n_tokens == 0 || n_seqs == 0) { - return HTP_STATUS_NO_SUPPORT; - } - if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) { - return HTP_STATUS_NO_SUPPORT; - } - if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] == 0 || k->ne[1] == 0 || - q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] == 0 || k->ne[3] == 0 || - (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) { - return HTP_STATUS_NO_SUPPORT; - } - // state holds s0 only: [S_v, S_v, H, n_seqs] - if (state->ne[0] != S_v || state->ne[1] != S_v || state->ne[2] != H || state->ne[3] != n_seqs) { - return HTP_STATUS_NO_SUPPORT; - } - if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K) { - return HTP_STATUS_NO_SUPPORT; - } - - for (int i = 0; i < 5; i++) { + for (int i = 0; i < 6; i++) { if (htp_tensor_is_extended(octx->src[i])) { return HTP_STATUS_NO_SUPPORT; } @@ -2325,68 +2286,12 @@ int op_gated_delta_net(struct htp_ops_context * octx) { } const struct htp_gdn_kernel_params * kparams = (const struct htp_gdn_kernel_params *) octx->kernel_params; - struct htp_gdn_kernel_params kparams_local; - if (!kparams || kparams->S_v == 0) { - const uint32_t rq3 = n_seqs / q->ne[3]; - const uint32_t rk3 = n_seqs / k->ne[3]; - const uint32_t total_rows = H * n_seqs; - uint32_t n_threads = (total_rows < octx->n_threads) ? total_rows : octx->n_threads; - if (n_threads == 0) { - n_threads = 1; - } - - memset(&kparams_local, 0, sizeof(kparams_local)); - kparams_local.n_threads = n_threads; - kparams_local.S_v = S_v; - kparams_local.H = H; - kparams_local.n_tokens = n_tokens; - kparams_local.n_seqs = n_seqs; - kparams_local.K = K; - kparams_local.total_rows = total_rows; - kparams_local.rows_per_thread = (total_rows + n_threads - 1) / n_threads; - const bool can_use_hmx = (octx->ctx->hmx_enabled) && - (S_v % 64 == 0) && - (n_tokens >= HTP_GDN_MIN_TOKENS) && - (g->ne[0] == 1) && - (K == 1); - - struct htp_gdn_hmx_vtcm_layout hmx_layout_local; - struct htp_gdn_vtcm_layout hvx_layout_local; - uint32_t n_heads_batch = 1; - - if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout_local, S_v, HTP_GDN_CHUNK_SIZE, total_rows, octx->ctx->vtcm_size, n_threads, true, &n_heads_batch)) { - kparams_local.kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED; - kparams_local.pipeline = hmx_layout_local.pipeline ? 1 : 0; - kparams_local.chunk_size = HTP_GDN_CHUNK_SIZE; - kparams_local.n_chunks = (n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE; - kparams_local.n_heads_batch = (uint16_t) n_heads_batch; - kparams_local.vtcm_size = (uint32_t) hmx_layout_local.total_bytes; - kparams_local.state_aligned = (uint32_t) hmx_layout_local.state_f32_bytes; - kparams_local.vtcm_per_thread = (uint32_t) (hmx_layout_local.total_bytes / (n_threads > 0 ? n_threads : 1)); - } else { - htp_gdn_vtcm_layout_build(&hvx_layout_local, S_v, n_threads); - kparams_local.kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT; - kparams_local.pipeline = 0; - kparams_local.n_heads_batch = 1; - kparams_local.state_aligned = (uint32_t) hvx_layout_local.state_aligned; - kparams_local.vtcm_per_thread = (uint32_t) hvx_layout_local.bytes_per_thread; - kparams_local.vtcm_size = (uint32_t) hvx_layout_local.total_bytes; - } - kparams_local.kda = (g->ne[0] == S_v) ? 1 : 0; - kparams_local.scale = 1.0f / sqrtf((float) S_v); - kparams_local.state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float)); - kparams_local.state_size_per_snap = S_v * S_v * H * n_seqs; - - kparams_local.div_H = init_fastdiv_values(H); - kparams_local.div_q1 = init_fastdiv_values(q->ne[1]); - kparams_local.div_k1 = init_fastdiv_values(k->ne[1]); - kparams_local.div_rq3 = init_fastdiv_values(rq3); - kparams_local.div_rk3 = init_fastdiv_values(rk3); - kparams_local.div_n_threads = init_fastdiv_values(n_threads); - - kparams = &kparams_local; + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; } + const uint32_t S_v = kparams->S_v; + const uint32_t n_tokens = kparams->n_tokens; const uint32_t total_rows = kparams->total_rows; uint32_t row_start = 0; uint32_t nrows = total_rows; @@ -2416,34 +2321,34 @@ int op_gated_delta_net(struct htp_ops_context * octx) { return gated_delta_net_f32_hmx_chunked(octx, kparams, row_start, nrows); } - const uint32_t n_threads = (nrows < kparams->n_threads) ? nrows : kparams->n_threads; + const uint32_t n_hvx = (nrows < octx->n_hvx) ? nrows : octx->n_hvx; struct htp_gdn_context gctx; gctx.octx = octx; gctx.kparams = kparams; gctx.row_start = row_start; gctx.nrows = nrows; - gctx.vtcm_base = octx->ctx->vtcm_base; - htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_threads); + htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_hvx); + gctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, gctx.layout.off_state); if (gctx.layout.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; } FARF(HIGH, "gated-delta-net-f32: q(%ux%ux%ux%u) k(%ux%ux%ux%u) v(%ux%ux%ux%u) state(%ux%ux%ux%u) -> (%ux%ux%ux%u) : " - "vtcm-size %zu n_threads %u\n", - q->ne[0], q->ne[1], q->ne[2], q->ne[3], - k->ne[0], k->ne[1], k->ne[2], k->ne[3], - v->ne[0], v->ne[1], v->ne[2], v->ne[3], - state->ne[0], state->ne[1], state->ne[2], state->ne[3], + "vtcm-size %zu n_hvx %u\n", + octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], + octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], + octx->src[2]->ne[0], octx->src[2]->ne[1], octx->src[2]->ne[2], octx->src[2]->ne[3], + octx->src[5]->ne[0], octx->src[5]->ne[1], octx->src[5]->ne[2], octx->src[5]->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - gctx.layout.total_bytes, n_threads); + gctx.layout.total_bytes, n_hvx); if (n_tokens == 1) { - work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, gated_delta_net_f32_tg_thread, &gctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, gated_delta_net_f32_pp_thread, &gctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h index 32fb7d24b..aff1eb645 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h @@ -29,7 +29,7 @@ struct htp_gdn_kernel_params { uint16_t n_chunks; uint16_t n_heads_batch; - uint32_t n_threads; + uint32_t n_hvx; uint32_t S_v; uint32_t H; uint32_t n_tokens; @@ -37,13 +37,7 @@ struct htp_gdn_kernel_params { uint32_t K; uint32_t total_rows; - uint32_t row_start; - uint32_t nrows; - uint32_t rows_per_thread; - uint32_t kda; - uint32_t state_aligned; - uint32_t vtcm_per_thread; uint32_t vtcm_size; uint32_t state_seq_stride; uint32_t state_size_per_snap; @@ -55,7 +49,6 @@ struct htp_gdn_kernel_params { struct fastdiv_values div_k1; struct fastdiv_values div_rq3; struct fastdiv_values div_rk3; - struct fastdiv_values div_n_threads; }; #if defined(__cplusplus) @@ -66,19 +59,22 @@ _Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_para struct htp_gdn_vtcm_layout { size_t state_aligned; - size_t bytes_per_thread; + size_t bytes_per_hvx; + size_t off_state; size_t total_bytes; }; static inline void htp_gdn_vtcm_layout_build( - struct htp_gdn_vtcm_layout * layout, + struct htp_gdn_vtcm_layout * L, uint32_t S_v, - uint32_t n_threads + uint32_t n_hvx ) { size_t state_bytes = (size_t) S_v * S_v * sizeof(float); - layout->state_aligned = hex_round_up(state_bytes, 128); - layout->bytes_per_thread = 2 * layout->state_aligned; - layout->total_bytes = layout->bytes_per_thread * n_threads; + L->state_aligned = hex_round_up(state_bytes, 128); + L->bytes_per_hvx = 2 * L->state_aligned; + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_hvx * n_hvx); + L->total_bytes = off; } struct htp_gdn_hmx_vtcm_layout { @@ -132,7 +128,7 @@ struct htp_gdn_hmx_vtcm_layout { size_t off_rows_inv; size_t off_rows_a; - size_t off_thread_scratch; + size_t off_hvx_scratch; size_t off_scales_1; size_t state_f32_bytes; @@ -144,7 +140,7 @@ struct htp_gdn_hmx_vtcm_layout { size_t tile_64x64_bytes; uint32_t n_heads_batch; - uint32_t n_threads; + uint32_t n_hvx; bool pipeline; size_t total_bytes; }; @@ -154,16 +150,16 @@ static inline void htp_gdn_hmx_vtcm_layout_build( uint32_t S_v, uint32_t chunk_size, uint32_t n_heads_batch, - uint32_t n_threads, + uint32_t n_hvx, bool pipeline ) { memset(L, 0, sizeof(*L)); L->n_heads_batch = n_heads_batch; - L->n_threads = n_threads; + L->n_hvx = n_hvx; L->pipeline = pipeline; const size_t bh = (size_t) n_heads_batch; - const size_t nth = (size_t) (n_threads > 0 ? n_threads : 1); + const size_t nth = (size_t) (n_hvx > 0 ? n_hvx : 1); const size_t state_f32_sz = hex_round_up(S_v * S_v * sizeof(float), 2048); const size_t state_f16_sz = hex_round_up(S_v * S_v * sizeof(__fp16), 2048); @@ -208,7 +204,7 @@ static inline void htp_gdn_hmx_vtcm_layout_build( VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_f32[1], bh * dma_scalar_sz, pipeline); VTCM_LAYOUT_ALLOC(off, off_b_f32[0], bh * dma_scalar_sz); VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_f32[1], bh * dma_scalar_sz, pipeline); - const size_t raw_gb_sz = hex_round_up(bh * chunk_size * sizeof(float), 128); + const size_t raw_gb_sz = hex_round_up((bh + 1) * chunk_size * sizeof(float), 128); VTCM_LAYOUT_ALLOC(off, off_g_raw[0], raw_gb_sz); VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_raw[1], raw_gb_sz, pipeline); VTCM_LAYOUT_ALLOC(off, off_b_raw[0], raw_gb_sz); @@ -252,9 +248,9 @@ static inline void htp_gdn_hmx_vtcm_layout_build( VTCM_LAYOUT_ALLOC(off, off_rows_inv, bh * row_vecs_sz); VTCM_LAYOUT_ALLOC(off, off_rows_a, bh * row_vecs_sz); - const size_t thread_scratch_sz = 64 * 128; + const size_t hvx_scratch_sz = 64 * 128; off = hex_align_up(off, HMX_FP16_TILE_SIZE); - VTCM_LAYOUT_ALLOC(off, off_thread_scratch, nth * thread_scratch_sz); + VTCM_LAYOUT_ALLOC(off, off_hvx_scratch, nth * hvx_scratch_sz); off = hex_align_up(off, HMX_FP16_TILE_SIZE); VTCM_LAYOUT_ALLOC(off, off_scales_1, HMX_FP16_TILE_SIZE); @@ -267,7 +263,7 @@ static inline bool htp_gdn_hmx_solve_layout( uint32_t chunk_size, uint32_t total_rows, size_t vtcm_budget, - uint32_t n_threads, + uint32_t n_hvx, bool pipeline, uint32_t * n_heads_batch_out ) { @@ -275,8 +271,8 @@ static inline bool htp_gdn_hmx_solve_layout( if (max_batch > total_rows) { max_batch = total_rows; } - if (max_batch > n_threads) { - max_batch = n_threads; + if (max_batch > n_hvx) { + max_batch = n_hvx; } static const uint32_t candidates[] = { 8, 6, 4, 2, 1 }; for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { @@ -285,7 +281,7 @@ static inline bool htp_gdn_hmx_solve_layout( continue; } struct htp_gdn_hmx_vtcm_layout L; - htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_threads, pipeline); + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_hvx, pipeline); if (L.total_bytes <= vtcm_budget) { *layout_out = L; *n_heads_batch_out = bh; @@ -293,7 +289,7 @@ static inline bool htp_gdn_hmx_solve_layout( } } if (pipeline) { - return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_threads, false, n_heads_batch_out); + return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_hvx, false, n_heads_batch_out); } return false; } diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.c new file mode 100644 index 000000000..02804db27 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gather-rows.c @@ -0,0 +1,403 @@ +#include "gather-rows.h" + +#include <assert.h> + +#include "hex-common.h" +#include "htp-trace.h" +#include "hvx-utils.h" +#include "dma-queue.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "work-queue.h" + +typedef void (*gather_rows_compute_fn_t)( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur); + +struct gather_rows_task { + struct htp_ops_context * octx; + const struct htp_tensor * a; + const struct htp_tensor * b; + dma_addr_t dst; + uint32_t r_start; + uint32_t rows; + uint32_t rows_per_hvx; + const struct htp_gather_rows_params * kparams; + gather_rows_compute_fn_t compute; +}; + +static inline __attribute__((always_inline)) void gather_rows_dma_in( + dma_queue * q, const struct gather_rows_task * task, + uint8_t * region, uint32_t r, uint32_t nr, uint32_t elem_size) { + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + const struct htp_gather_rows_params * kparams = task->kparams; + + const size_t a_size = (nr - 1) * a->nb[1] + kparams->na * elem_size; + dma_queue_push(q, dma_make_data(region, a->data + r * a->nb[1]), a_size, a_size, a_size, 1); + if (b) { + if (kparams->b_dense) { + const size_t b_size = (nr - 1) * b->nb[1] + kparams->nb * elem_size; + dma_queue_push(q, dma_make_data(region + kparams->v1, b->data + r * b->nb[1]), b_size, b_size, b_size, 1); + } else { + const size_t b_size = (nr - 1) * b->nb[1] + elem_size; + for (uint32_t k = 0; k < kparams->nb; k++) { + dma_queue_push(q, dma_make_data(region + kparams->v1 + k * kparams->span1, + b->data + k * b->nb[0] + r * b->nb[1]), b_size, b_size, b_size, 1); + } + } + } +} + +static inline void gather_rows_dma_out(dma_queue * q, const struct gather_rows_task * task, + uint8_t * out, uint32_t r, uint32_t nr, uint32_t elem_size) { + const struct htp_gather_rows_params * kparams = task->kparams; + const size_t out_size = nr * kparams->ne * elem_size; + dma_queue_push(q, dma_make_data(task->dst + r * kparams->ne * elem_size, out), out_size, out_size, out_size, 1); +} + +static __attribute__((noinline)) void gather_rows_init_tables_f32( + HVX_Vector * tab, HVX_Vector * inc, + const struct htp_gather_rows_params * kparams, + const struct htp_tensor * a, const struct htp_tensor * b) { + + const int32_t s_a = kparams->rg * a->nb[1]; + const int32_t s_b = b ? kparams->rg * b->nb[1] : 0; + const uint32_t b_nb1 = b ? b->nb[1] : 0; + uint32_t row = 0; + uint32_t col = 0; + int32_t t_buf[32] __attribute__((aligned(128))); + int32_t s_buf[32] __attribute__((aligned(128))); + for (uint32_t k = 0; k < kparams->vg; k++) { + for (uint32_t lane = 0; lane < 32; lane++) { + if (col < kparams->na) { + t_buf[lane] = row * a->nb[1] + col * 4; + s_buf[lane] = s_a; + } else { + t_buf[lane] = kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1; + s_buf[lane] = s_b; + } + if (++col == kparams->ne) { + col = 0; + row++; + } + } + tab[k] = *(const HVX_Vector *) t_buf; + inc[k] = *(const HVX_Vector *) s_buf; + } +} + +static __attribute__((noinline)) void gather_rows_init_tables_f16( + HVX_Vector * tab, HVX_Vector * inc, + const struct htp_gather_rows_params * kparams, + const struct htp_tensor * a, const struct htp_tensor * b) { + + const int16_t s_a = (int16_t) (kparams->rg * a->nb[1]); + const int16_t s_b = (int16_t) (b ? kparams->rg * b->nb[1] : 0); + const uint32_t b_nb1 = b ? b->nb[1] : 0; + uint32_t row = 0; + uint32_t col = 0; + int16_t t_buf[64] __attribute__((aligned(128))); + int16_t s_buf[64] __attribute__((aligned(128))); + for (uint32_t k = 0; k < kparams->vg; k++) { + for (uint32_t lane = 0; lane < 64; lane++) { + if (col < kparams->na) { + t_buf[lane] = (int16_t) (row * a->nb[1] + col * 2); + s_buf[lane] = s_a; + } else { + t_buf[lane] = (int16_t) (kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1); + s_buf[lane] = s_b; + } + if (++col == kparams->ne) { + col = 0; + row++; + } + } + tab[k] = *(const HVX_Vector *) t_buf; + inc[k] = *(const HVX_Vector *) s_buf; + } +} + +static void gather_rows_compute_f32_vg1( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + (void) cur; + const uint32_t region_size = kparams->region_size; + HVX_Vector vcur = tab[0]; + const HVX_Vector vinc = inc[0]; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur); + vcur = Q6_Vw_vadd_VwVw(vcur, vinc); + } +} + +static void gather_rows_compute_f32_vgn( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + const uint32_t vg = kparams->vg; + const uint32_t region_size = kparams->region_size; + for (uint32_t k = 0; k < vg; k++) { + cur[k] = tab[k]; + } + uint32_t k = 0; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]); + cur[k] = Q6_Vw_vadd_VwVw(cur[k], inc[k]); + if (++k == vg) { + k = 0; + } + } +} + +static void gather_rows_compute_f16_vg1( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + (void) cur; + const uint32_t region_size = kparams->region_size; + HVX_Vector vcur = tab[0]; + const HVX_Vector vinc = inc[0]; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur); + vcur = Q6_Vh_vadd_VhVh(vcur, vinc); + } +} + +static void gather_rows_compute_f16_vgn( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + const uint32_t vg = kparams->vg; + const uint32_t region_size = kparams->region_size; + for (uint32_t k = 0; k < vg; k++) { + cur[k] = tab[k]; + } + uint32_t k = 0; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]); + cur[k] = Q6_Vh_vadd_VhVh(cur[k], inc[k]); + if (++k == vg) { + k = 0; + } + } +} + +static void gather_rows_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct gather_rows_task * task = (const struct gather_rows_task *) data; + const struct htp_gather_rows_params * kparams = task->kparams; + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + + const uint32_t r_offset = MIN(ith * task->rows_per_hvx, task->rows); + const uint32_t r_beg = task->r_start + r_offset; + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_hvx, task->rows); + if (r_beg >= r_end) { + return; + } + + struct htp_thread_trace * tr = th->trace; + + struct htp_gather_rows_vtcm_layout layout; + htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 4); + + uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice); + HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab); + HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc); + HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur); + + uint8_t * region[2] = { + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0), + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1), + }; + uint8_t * out[2] = { + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0), + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1), + }; + + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + gather_rows_init_tables_f32(tab, inc, kparams, a, b); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + + dma_queue * q = th->dma_q; + const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); + const uint32_t nr_max = kparams->nr_max; + const uint32_t nr0 = MIN(nr_max, r_end - r_beg); + + dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[0], r_beg, nr0, 4); + + const uint32_t r1 = r_beg + nr_max; + if (r1 < r_end) { + const uint32_t nr1 = MIN(nr_max, r_end - r1); + dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[1], r1, nr1, 4); + } + + uint32_t buf = 0; + for (uint32_t r = r_beg; r < r_end; r += nr_max) { + const uint32_t nr = MIN(nr_max, r_end - r); + const uint32_t next_buf = buf ^ 1; + + dma_queue_pop(q); // complete out + for (uint32_t i = 0; i < n_in; i++) { + dma_queue_pop(q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + const uint32_t nvec = (nr * kparams->ne + 31) >> 5; + task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur); + hvx_gather_sync(out[buf]); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + + gather_rows_dma_out(q, task, out[buf], r, nr, 4); + + const uint32_t r_prefetch = r + 2 * nr_max; + if (r_prefetch < r_end) { + const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch); + gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 4); + } + + buf = next_buf; + } + dma_queue_flush(q); +} + +static void gather_rows_thread_f16(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct gather_rows_task * task = (const struct gather_rows_task *) data; + const struct htp_gather_rows_params * kparams = task->kparams; + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + + const uint32_t r_offset = MIN(ith * task->rows_per_hvx, task->rows); + const uint32_t r_beg = task->r_start + r_offset; + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_hvx, task->rows); + if (r_beg >= r_end) { + return; + } + + struct htp_thread_trace * tr = th->trace; + + struct htp_gather_rows_vtcm_layout layout; + htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 2); + + uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice); + HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab); + HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc); + HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur); + + uint8_t * region[2] = { + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0), + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1), + }; + uint8_t * out[2] = { + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0), + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1), + }; + + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + gather_rows_init_tables_f16(tab, inc, kparams, a, b); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + + dma_queue * q = th->dma_q; + const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); + + const uint32_t nr_max = kparams->nr_max; + const uint32_t nr0 = MIN(nr_max, r_end - r_beg); + dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[0], r_beg, nr0, 2); + + const uint32_t r1 = r_beg + nr_max; + if (r1 < r_end) { + const uint32_t nr1 = MIN(nr_max, r_end - r1); + dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[1], r1, nr1, 2); + } + + uint32_t buf = 0; + for (uint32_t r = r_beg; r < r_end; r += nr_max) { + const uint32_t nr = MIN(nr_max, r_end - r); + const uint32_t next_buf = buf ^ 1; + + dma_queue_pop(q); // complete out + for (uint32_t i = 0; i < n_in; i++) { + dma_queue_pop(q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + const uint32_t nvec = (nr * kparams->ne + 63) >> 6; + task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur); + hvx_gather_sync(out[buf]); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + + gather_rows_dma_out(q, task, out[buf], r, nr, 2); + + const uint32_t r_prefetch = r + 2 * nr_max; + if (r_prefetch < r_end) { + const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch); + gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 2); + } + + buf = next_buf; + } + dma_queue_flush(q); +} + +int htp_gather_rows( + struct htp_ops_context * octx, + const struct htp_tensor * a, + const struct htp_tensor * b, + const struct htp_gather_rows_params * kparams, + uint32_t n_hvx) { + + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + assert(kparams->slice * octx->n_hvx <= octx->ctx->vtcm_size); + + const uint32_t total_rows = kparams->total_rows; + uint32_t r_start = 0; + uint32_t r_count = total_rows; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, kparams->rg, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + r_start = range.start; + r_count = range.count; + } + + if (r_count == 0) { + return HTP_STATUS_OK; + } + + const uint32_t elem_size = (a->type == HTP_TYPE_F16) ? 2 : 4; + + struct gather_rows_task task; + task.octx = octx; + task.a = a; + task.b = b; + task.dst = octx->dst->data; + task.r_start = r_start; + task.rows = r_count; + task.kparams = kparams; + task.rows_per_hvx = kparams->rows_per_hvx; + + if (elem_size == 2) { + task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn; + } else { + task.compute = (kparams->vg == 1) ? gather_rows_compute_f32_vg1 : gather_rows_compute_f32_vgn; + } + + work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32; + work_queue_run(octx->ctx->hvx_queue, worker_func, &task, octx->n_hvx); + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.h new file mode 100644 index 000000000..9ca215ac3 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gather-rows.h @@ -0,0 +1,189 @@ +#ifndef HTP_GATHER_ROWS_H +#define HTP_GATHER_ROWS_H + +#include "hex-common.h" +#include "htp-vtcm.h" +#include <stdint.h> +#include <stdbool.h> + +#ifndef GGML_MAX_DIMS +#define GGML_MAX_DIMS 4 +#endif + +struct htp_gather_rows_params { + uint32_t na; + uint32_t nb; + uint32_t ne; + uint32_t rg; + uint32_t vg; + uint32_t nr_max; + uint32_t v1; + uint32_t span1; + uint32_t region_size; + uint32_t slice; + uint32_t total_rows; + uint32_t rows_per_hvx; + uint8_t b_dense; + uint8_t pad[3]; +}; + +struct htp_gather_rows_vtcm_layout { + size_t off_tab; + size_t off_inc; + size_t off_cur; + size_t off_region0; + size_t off_out0; + size_t off_region1; + size_t off_out1; + size_t total_per_hvx; +}; + +static inline void htp_gather_rows_vtcm_layout_build( + struct htp_gather_rows_vtcm_layout * L, + uint32_t vg, + uint32_t region_size, + uint32_t nr_max, + uint32_t ne, + uint32_t elem_size) { + + const size_t vec_bytes = (size_t) vg * 128; + const size_t region_aligned = hex_round_up(region_size, 128); + const size_t out_aligned = hex_round_up(region_aligned + (size_t) nr_max * ne * elem_size, 128) - region_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_tab, vec_bytes); + VTCM_LAYOUT_ALLOC(off, off_inc, vec_bytes); + VTCM_LAYOUT_ALLOC(off, off_cur, vec_bytes); + VTCM_LAYOUT_ALLOC(off, off_region0, region_aligned); + VTCM_LAYOUT_ALLOC(off, off_out0, out_aligned); + VTCM_LAYOUT_ALLOC(off, off_region1, region_aligned); + VTCM_LAYOUT_ALLOC(off, off_out1, out_aligned); + L->total_per_hvx = off; +} + +static inline bool htp_gather_rows_solve_layout( + struct htp_gather_rows_params * kparams, + uint32_t na, uint32_t a_nb1, + uint32_t nb, uint32_t b_nb1, bool b_dense, + uint32_t elem_size, + uint32_t total_rows, + uint32_t n_hvx, + uint32_t mdev_count, + uint32_t vtcm_size) { + + const uint32_t max_ne0 = 128 / elem_size; + const uint32_t ne = na + nb; + if (ne == 0 || ne > max_ne0) { + return false; + } + + if (elem_size != 4 && elem_size != 2) { + return false; + } + + const uint32_t v_lanes = 128 / elem_size; + const uint32_t g = hex_gcd_u32(ne, v_lanes); + const uint32_t rg = v_lanes / g; + const uint32_t vg = ne / g; + + const uint32_t slice = (vtcm_size / n_hvx) & ~127u; + const uint32_t total_chunks = total_rows / rg; + const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count; + const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count) + ? (chunks_per_dev * rg + (total_rows % rg)) + : total_rows; + const uint32_t rows_per_hvx = hex_round_up((max_rows_per_dev + n_hvx - 1) / n_hvx, rg); + + const uint32_t fixed = 3 * vg * 128; + const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size; + if (slice <= fixed + 2 * per_row * rg) { + return false; + } + + const uint32_t avail_per_buf = (slice - fixed) / 2; + uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_hvx); + if (elem_size == 2) { + const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0); + if (per_row_region > 0) { + nr_max = MIN(nr_max, (32768u - 256u) / per_row_region); + } + } + nr_max = (nr_max / rg) * rg; + + uint32_t v1 = 0; + uint32_t span1 = 0; + uint32_t region_size = 0; + + while (nr_max > 0) { + v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128); + if (nb > 0) { + if (b_dense) { + span1 = elem_size; + region_size = v1 + (nr_max - 1) * b_nb1 + nb * elem_size; + } else { + span1 = (nr_max - 1) * b_nb1 + elem_size; + region_size = v1 + nb * span1; + } + } else { + span1 = 0; + region_size = v1; + } + + struct htp_gather_rows_vtcm_layout layout; + htp_gather_rows_vtcm_layout_build(&layout, vg, region_size, nr_max, ne, elem_size); + + if ((elem_size != 2 || region_size <= 32768) && layout.total_per_hvx <= slice) { + break; + } + + nr_max -= rg; + } + + if (nr_max == 0) { + return false; + } + + kparams->na = na; + kparams->nb = nb; + kparams->ne = ne; + kparams->rg = rg; + kparams->vg = vg; + kparams->nr_max = nr_max; + kparams->v1 = v1; + kparams->span1 = span1; + kparams->region_size = region_size; + kparams->slice = slice; + kparams->total_rows = total_rows; + kparams->rows_per_hvx = rows_per_hvx; + kparams->b_dense = b_dense ? 1 : 0; + kparams->pad[0] = 0; + kparams->pad[1] = 0; + kparams->pad[2] = 0; + return true; +} + +#ifdef __cplusplus +struct ggml_tensor; +static inline bool htp_gather_rows_tensor_outer_contiguous(const struct ggml_tensor * t) { + size_t next_nb = ((const size_t *) t->nb)[1] * ((const int64_t *) t->ne)[1]; + for (int i = 2; i < GGML_MAX_DIMS; i++) { + if (((const int64_t *) t->ne)[i] != 1 && ((const size_t *) t->nb)[i] != next_nb) { + return false; + } + next_nb *= ((const int64_t *) t->ne)[i]; + } + return true; +} +#else +struct htp_ops_context; +struct htp_tensor; + +int htp_gather_rows( + struct htp_ops_context * octx, + const struct htp_tensor * a, + const struct htp_tensor * b, + const struct htp_gather_rows_params * kparams, + uint32_t n_hvx); +#endif + +#endif // HTP_GATHER_ROWS_H diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index f354b813a..83e85f005 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -17,21 +17,32 @@ #include "htp-tensor.h" #include "hvx-utils.h" #include "hvx-quant.h" -#include "matmul-ops.h" #include "get-rows-ops.h" #include "work-queue.h" +struct get_rows_context; + +typedef uint32_t (*get_rows_read_index_fn_t)(const void * src1, uint32_t idx, int idx_type); +typedef void (*get_rows_flat_compute_fn_t)(float * dst_spad, const void * src_spad, uint32_t cur_elems); +typedef void (*get_rows_tiled_compute_fn_t)(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row); + struct get_rows_context { struct htp_ops_context * octx; const struct htp_get_rows_kernel_params * kparams; struct htp_get_rows_vtcm_layout vtcm_layout; - uint8_t * vtcm_base; + uint8_t * vtcm_base; + const void * src1_data; + get_rows_read_index_fn_t read_index; + get_rows_flat_compute_fn_t compute_flat; + get_rows_tiled_compute_fn_t compute_tiled; uint32_t task_start; uint32_t tasks; - uint32_t tasks_per_thread; + uint32_t tasks_per_hvx; uint32_t tile_size; uint32_t tile_stride; - bool index_i32; + uint32_t src0_row_size; + uint32_t dst_row_size; + int idx_type; }; #define get_rows_preamble \ @@ -64,195 +75,169 @@ struct get_rows_context { \ const uint32_t nr = ne10 * ne11 * ne12; -#define GET_ROWS_THREAD_ST_FN(IDX_TYPE) \ -static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ - struct get_rows_context * grctx = (struct get_rows_context *)data; \ - struct htp_ops_context * octx = grctx->octx; \ - const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ - get_rows_preamble; \ - const uint32_t dr = grctx->tasks_per_thread; \ - const uint32_t ir0 = grctx->task_start + dr * ith; \ - if (ir0 >= grctx->task_start + grctx->tasks) { \ - return; \ - } \ - const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ - const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - for (uint32_t i = ir0; i < ir1; ++i) { \ - const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); \ - const uint32_t rem = i - i12 * ne11 * ne10; \ - const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ - const uint32_t i10 = rem - i11 * ne10; \ - const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ - const uint32_t i01 = (uint32_t)*src1_ptr; \ - assert(i01 < ne01); \ - const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ - const uint32_t i02 = i11 - q02 * ne02; \ - const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ - const uint32_t i03 = i12 - q03 * ne03; \ - const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; \ - const dma_addr_t dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; \ - while (!dma_queue_push(dma_q, dma_make_data(dst_data, src0_data), nb1, nb01, \ - row_size_bytes, 1)) { \ - dma_queue_pop(dma_q); \ - } \ - } \ - dma_queue_flush(dma_q); \ +static uint32_t get_rows_read_index_ddr( + const void * src1, + uint32_t idx, + int idx_type +) { + const uint32_t off = idx * ((idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t)); + return *(const uint32_t *) ((const uint8_t *) src1 + off); } -GET_ROWS_THREAD_ST_FN(int32_t) -GET_ROWS_THREAD_ST_FN(int64_t) - -#define GET_ROWS_THREAD_DT_FN(TYPE_NAME, SRC0_SIZE_EXPR, IDX_TYPE, COMPUTE_EXPR) \ -static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ - struct get_rows_context * grctx = (struct get_rows_context *)data; \ - struct htp_ops_context * octx = grctx->octx; \ - const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ - get_rows_preamble; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = grctx->tasks_per_thread; \ - const uint32_t ir0 = grctx->task_start + dr * ith; \ - if (ir0 >= grctx->task_start + grctx->tasks) { \ - return; \ - } \ - const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ - const uint32_t chunks_per_row = kparams->chunks_per_row; \ - const uint32_t chunk_size = kparams->chunk_size; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; \ - uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ - uint8_t * vtcm_dst = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ - for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { \ - const uint32_t i = ir0 + step; \ - const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ - const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ - const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ - const uint32_t rem = row_idx - i12 * ne11 * ne10; \ - const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ - const uint32_t i10 = rem - i11 * ne10; \ - const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ - const uint32_t i01 = (uint32_t)*src1_ptr; \ - assert(i01 < ne01); \ - const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ - const uint32_t i02 = i11 - q02 * ne02; \ - const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ - const uint32_t i03 = i12 - q03 * ne03; \ - const uint32_t offset = chunk_idx * chunk_size; \ - const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ - const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems); \ - const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ - const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset); \ - dma_queue_push(dma_q, \ - dma_make_data(octx->dst->data, \ - vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ - cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0); \ - dma_queue_push(dma_q, \ - dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, src0_data), \ - vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1); \ - } \ - for (uint32_t step = 0; step < ir1 - ir0; ++step) { \ - const uint32_t i = ir0 + step; \ - void * dst_spad = (void *) dma_queue_pop(dma_q).src; \ - void * src_spad = (void *) dma_queue_pop(dma_q).dst; \ - const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ - const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ - const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ - const uint32_t rem = row_idx - i12 * ne11 * ne10; \ - const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ - const uint32_t i10 = rem - i11 * ne10; \ - const uint32_t offset = chunk_idx * chunk_size; \ - const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ - const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i); \ - COMPUTE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i); \ - const dma_addr_t dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); \ - dma_queue_push(dma_q, \ - dma_make_data(dst_data, dst_spad), \ - cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1); \ - const uint32_t next_step = step + 2; \ - if (next_step < ir1 - ir0) { \ - const uint32_t pi = ir0 + next_step; \ - const uint32_t prow_idx = fastdiv(pi, &kparams->div_chunks_per_row); \ - const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row; \ - const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11); \ - const uint32_t prem = prow_idx - pi12 * ne11 * ne10; \ - const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10); \ - const uint32_t pi10 = prem - pi11 * ne10; \ - const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \ - const uint32_t pi01 = (uint32_t)*psrc1_ptr; \ - assert(pi01 < ne01); \ - const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02); \ - const uint32_t pi02 = pi11 - pq02 * ne02; \ - const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03); \ - const uint32_t pi03 = pi12 - pq03 * ne03; \ - const uint32_t poffset = pchunk_idx * chunk_size; \ - const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0; \ - const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems); \ - const dma_addr_t psrc0_data = \ - octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset); \ - dma_queue_push(dma_q, \ - dma_make_data(src_spad, psrc0_data), \ - vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1); \ - } \ - } \ - dma_queue_flush(dma_q); \ +static __attribute__((noinline)) uint32_t get_rows_read_index_vtcm( + const void * src1, + uint32_t idx, + int idx_type +) { + const uint32_t off = idx * ((idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t)); + const HVX_Vector v = hvx_vmem((const uint8_t *) src1 + off); // off & ~127u in HW + // vextract is normally slow, but here it avoids an HVX->L2 store and L1D miss + return (uint32_t) Q6_R_vextract_VR(v, off); } -#define F16_BYTES(n) ((n) * sizeof(__fp16)) -#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0)) +static void get_rows_thread_sametype(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct get_rows_context * grctx = (struct get_rows_context *) data; + struct htp_ops_context * octx = grctx->octx; + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; + get_rows_preamble; + + const uint32_t dr = grctx->tasks_per_hvx; + const uint32_t ir0 = grctx->task_start + dr * ith; + if (ir0 >= grctx->task_start + grctx->tasks) { + return; + } + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); + const uint32_t row_size_bytes = grctx->src0_row_size; + dma_queue * dma_q = th->dma_q; + + for (uint32_t i = ir0; i < ir1; ++i) { + const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); + const uint32_t rem = i - i12 * ne11 * ne10; + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); + const uint32_t i10 = rem - i11 * ne10; + + const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type); + assert(i01 < ne01); + + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); + const uint32_t i02 = i11 - q02 * ne02; + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); + const uint32_t i03 = i12 - q03 * ne03; -static __attribute__((noinline)) void compute_get_rows_f16(float * dst_spad, const void * src_spad, uint32_t cur_elems) { - hvx_dequantize_row_f16_f32(dst_spad, src_spad, cur_elems); + const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; + const dma_addr_t dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; + + while (!dma_queue_push(dma_q, dma_make_data(dst_data, src0_data), nb1, nb01, row_size_bytes, 1)) { + dma_queue_pop(dma_q); + } + } + dma_queue_flush(dma_q); } -static __attribute__((noinline)) void compute_get_rows_q8_0(float * dst_spad, const void * src_spad, uint32_t cur_elems) { - hvx_dequantize_row_q8_0_f32(dst_spad, src_spad, cur_elems); + + +struct get_rows_flat_task { + dma_addr_t src0_data; + dma_addr_t dst_data; +}; + +static inline struct get_rows_flat_task get_rows_flat_calc_task( + const struct htp_ops_context * octx, + const struct get_rows_context * grctx, + uint32_t i, + bool need_src0 +) { + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; + get_rows_preamble; + + const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); + const uint32_t rem = i - i12 * ne11 * ne10; + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); + const uint32_t i10 = rem - i11 * ne10; + + struct get_rows_flat_task task; + task.src0_data = 0; + task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; + + if (need_src0) { + const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type); + assert(i01 < ne01); + + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); + const uint32_t i02 = i11 - q02 * ne02; + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); + const uint32_t i03 = i12 - q03 * ne03; + + task.src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; + } + + return task; } -GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); }) -GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); }) - -GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); }) -GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); }) - - -static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4) { - const HVX_VectorPred first2 = Q6_Q_vsetq_R(2); - const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); - HVX_Vector vq = Q6_V_vzero(); - if (q4) { - const HVX_VectorPred first1 = Q6_Q_vsetq_R(1); - const HVX_VectorPred first3 = Q6_Q_vsetq_R(3); - for (int group = 3; group >= 0; --group) { - const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), row); - // Four planes contribute bytes at 0, 32, 64 and 96 after rotation. - HVX_Vector packed = Q6_V_vmux_QVV(first1, v, Q6_V_vror_VR(v, 31)); - packed = Q6_V_vmux_QVV(first2, packed, Q6_V_vror_VR(v, 62)); - packed = Q6_V_vmux_QVV(first3, packed, Q6_V_vror_VR(v, 93)); - vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); - } - const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); - const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4); - vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); - vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8)); - } else { - for (int group = 7; group >= 0; --group) { - const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row); - // Two planes contribute halfwords at 0 and 64 after rotation. - const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62)); - vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); +static void get_rows_thread_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct get_rows_context * grctx = (struct get_rows_context *) data; + struct htp_ops_context * octx = grctx->octx; + get_rows_preamble; + + const uint32_t dr = grctx->tasks_per_hvx; + const uint32_t ir0 = grctx->task_start + dr * ith; + if (ir0 >= grctx->task_start + grctx->tasks) { + return; + } + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); + + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; + uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_hvx; + uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_hvx; + const uint32_t src0_bytes = grctx->src0_row_size; + const uint32_t dst_bytes = grctx->dst_row_size; + get_rows_flat_compute_fn_t compute = grctx->compute_flat; + + for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { + const uint32_t i = ir0 + step; + struct get_rows_flat_task task = get_rows_flat_calc_task(octx, grctx, i, true); + + dma_queue_push(dma_q, + dma_make_data(octx->dst->data, + vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), + dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0); + dma_queue_push(dma_q, + dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, task.src0_data), + vtcm_layout->src0_spad_half_size, src0_bytes, src0_bytes, 1); + } + + for (uint32_t step = 0; step < ir1 - ir0; ++step) { + const uint32_t i = ir0 + step; + void * dst_spad = (void *) dma_queue_pop(dma_q).src; + void * src_spad = (void *) dma_queue_pop(dma_q).dst; + + struct get_rows_flat_task task = get_rows_flat_calc_task(octx, grctx, i, false); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + compute((float *) dst_spad, src_spad, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + + dma_queue_push(dma_q, + dma_make_data(task.dst_data, dst_spad), + dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1); + + const uint32_t next_step = step + 2; + if (next_step < ir1 - ir0) { + const uint32_t pi = ir0 + next_step; + struct get_rows_flat_task next_task = get_rows_flat_calc_task(octx, grctx, pi, true); + dma_queue_push(dma_q, + dma_make_data(src_spad, next_task.src0_data), + vtcm_layout->src0_spad_half_size, src0_bytes, src0_bytes, 1); } } - const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024)); - const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); - const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); - const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); - const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); - const HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); - *(HVX_Vector *) dst = values; + + dma_queue_flush(dma_q); } + + struct get_rows_tiled_task { dma_addr_t tile_src_base; dma_addr_t dst_data; @@ -273,16 +258,15 @@ static inline struct get_rows_tiled_task get_rows_tiled_calc_task( const uint32_t rem = i - i12 * ne11 * ne10; const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); const uint32_t i10 = rem - i11 * ne10; - const dma_addr_t src1_data = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; - const uint32_t i01 = grctx->index_i32 ? *(const int32_t *)(uintptr_t) src1_data : (uint32_t) *(const int64_t *)(uintptr_t) src1_data; + const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type); assert(i01 < ne01); const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); const uint32_t i02 = i11 - q02 * ne02; const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); const uint32_t i03 = i12 - q03 * ne03; - const uint32_t column_tile = i01 / HTP_MM_HMX_TILE_N_ROWS; - const uint32_t row = i01 % HTP_MM_HMX_TILE_N_ROWS; + const uint32_t column_tile = i01 / HTP_TENSOR_TILE_N_ROWS; + const uint32_t row = i01 % HTP_TENSOR_TILE_N_ROWS; const dma_addr_t matrix = octx->src[0]->data + i02*nb02 + i03*nb03; struct get_rows_tiled_task task; @@ -292,30 +276,29 @@ static inline struct get_rows_tiled_task get_rows_tiled_calc_task( return task; } -static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) { +static void get_rows_thread_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct get_rows_context * grctx = (struct get_rows_context *) data; struct htp_ops_context * octx = grctx->octx; - const struct htp_get_rows_kernel_params * kparams = grctx->kparams; get_rows_preamble; - const uint32_t dr = grctx->tasks_per_thread; + const uint32_t dr = grctx->tasks_per_hvx; const uint32_t ir0 = grctx->task_start + dr * ith; if (ir0 >= grctx->task_start + grctx->tasks) { return; } const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); - const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS; + const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type); const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; - uint8_t * src_spad_base = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; - uint8_t * dst_spad_base = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_hvx; + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_hvx; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const uint32_t tile_size = grctx->tile_size; const uint32_t tile_stride = grctx->tile_stride; - const uint32_t dst_bytes = ne00 * sizeof(float); - const bool is_q4 = (octx->src[0]->type == HTP_TYPE_Q4_0); + const uint32_t dst_bytes = grctx->dst_row_size; + get_rows_tiled_compute_fn_t compute_tiled = grctx->compute_tiled; for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) { const uint32_t i = ir0 + step; @@ -340,11 +323,7 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); - for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { - const uint8_t * tile = src_spad + k_tile * tile_stride; - float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS; - compute_get_rows_tiled(dst_block, tile, task.row, is_q4); - } + compute_tiled(dst_spad, src_spad, n_k_tiles, tile_stride, task.row); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); // Real writeback of dst_spad @@ -365,14 +344,40 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat dma_queue_flush(dma_q); } +static __attribute__((noinline)) void get_rows_dma_src1( + struct htp_ops_context * octx, + uint8_t * vtcm_src1, + uint32_t elem_size +) { + const uint32_t ne10 = octx->src[1]->ne[0]; + const uint32_t ne11 = octx->src[1]->ne[1]; + const uint32_t ne12 = octx->src[1]->ne[2]; + const uint32_t ne13 = octx->src[1]->ne[3]; + const uint32_t nb11 = octx->src[1]->nb[1]; + const uint32_t nb12 = octx->src[1]->nb[2]; + const uint32_t nb13 = octx->src[1]->nb[3]; + + dma_queue * dma_q = octx->ctx->dma[0]; + uint8_t * cur_dst = vtcm_src1; + for (uint32_t i13 = 0; i13 < ne13; ++i13) { + for (uint32_t i12 = 0; i12 < ne12; ++i12) { + const dma_addr_t src_plane = octx->src[1]->data + i12 * nb12 + i13 * nb13; + while (!dma_queue_push(dma_q, dma_make_data(cur_dst, src_plane), + ne10 * elem_size, nb11, ne10 * elem_size, ne11)) { + dma_queue_pop(dma_q); + } + cur_dst += ne10 * ne11 * elem_size; + } + } + dma_queue_flush(dma_q); +} + int op_get_rows(struct htp_ops_context * octx) { const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params; - if (octx->src[0]->type != HTP_TYPE_F32 && - octx->src[0]->type != HTP_TYPE_F16 && - octx->src[0]->type != HTP_TYPE_Q4_0 && - octx->src[0]->type != HTP_TYPE_Q8_0 && - octx->src[0]->type != HTP_TYPE_I32) { + const uint32_t ne00 = octx->src[0]->ne[0]; + const uint32_t src0_row_size = htp_tensor_get_row_size(octx->src[0]->type, ne00); + if (src0_row_size == 0) { return HTP_STATUS_NO_SUPPORT; } @@ -390,20 +395,16 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (htp_tensor_is_extended(octx->src[1])) { - return HTP_STATUS_NO_SUPPORT; - } - const struct htp_tensor * dst = octx->dst; const uint32_t total_tasks = kparams->total_tasks; - const size_t dst_row_size = htp_tensor_get_row_size(dst->type, dst->ne[0]); + const uint32_t dst_row_size = (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) ? src0_row_size : ne00 * sizeof(float); uint32_t task_start = 0; uint32_t tasks = total_tasks; if (octx->ctx->mdev.count > 1) { uint32_t tasks_per_chunk = 1; - htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], (uint32_t) dst_row_size, &tasks_per_chunk); + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst->nb[0], dst_row_size, &tasks_per_chunk); const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); task_start = range.start; tasks = range.count; @@ -413,25 +414,28 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct get_rows_context grctx; grctx.octx = octx; grctx.kparams = kparams; grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; + grctx.compute_flat = NULL; + grctx.compute_tiled = NULL; grctx.task_start = task_start; grctx.tasks = tasks; - grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div); - grctx.tile_size = octx->src[0]->type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0; - grctx.tile_stride = (grctx.tile_size + 127) & ~127; - grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32; + grctx.tasks_per_hvx = fastdiv(tasks + n_hvx - 1, &octx->n_hvx_div); + grctx.tile_size = 0; + grctx.tile_stride = 0; + grctx.src0_row_size = src0_row_size; + grctx.dst_row_size = dst_row_size; + grctx.idx_type = octx->src[1]->type; - const uint32_t ne00 = octx->src[0]->ne[0]; - htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, n_threads); + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_hvx); if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) { FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n", @@ -439,35 +443,68 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_INVAL_PARAMS; } - const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); - work_queue_func_t q_func = NULL; switch (kparams->kernel_type) { case HTP_GET_ROWS_KERNEL_SAMETYPE: - q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); + q_func = get_rows_thread_sametype; break; case HTP_GET_ROWS_KERNEL_TILED: + switch (octx->src[0]->type) { + case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break; + case HTP_TYPE_Q1_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q1_0_f32; break; + case HTP_TYPE_Q2_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q2_0_f32; break; + case HTP_TYPE_Q8_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q8_0_f32; break; + case HTP_TYPE_Q4_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_0_f32; break; + case HTP_TYPE_Q4_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_k_f32; break; + case HTP_TYPE_Q5_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q5_k_f32; break; + case HTP_TYPE_Q6_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q6_k_f32; break; + default: return HTP_STATUS_NO_SUPPORT; + } + grctx.tile_size = htp_tensor_get_tile_size(octx->src[0]->type); + grctx.tile_stride = htp_tensor_get_tile_padded_size(octx->src[0]->type); q_func = get_rows_thread_tiled; break; case HTP_GET_ROWS_KERNEL_FLAT: switch (octx->src[0]->type) { - case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break; - case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break; + case HTP_TYPE_F16: grctx.compute_flat = hvx_dequantize_row_flat_f16_f32; break; + case HTP_TYPE_BF16: grctx.compute_flat = hvx_dequantize_row_flat_bf16_f32; break; + case HTP_TYPE_Q1_0: grctx.compute_flat = hvx_dequantize_row_flat_q1_0_f32; break; + case HTP_TYPE_Q2_0: grctx.compute_flat = hvx_dequantize_row_flat_q2_0_f32; break; + case HTP_TYPE_Q8_0: grctx.compute_flat = hvx_dequantize_row_flat_q8_0_f32; break; + case HTP_TYPE_Q4_0: grctx.compute_flat = hvx_dequantize_row_flat_q4_0_f32; break; + case HTP_TYPE_Q4_K: grctx.compute_flat = hvx_dequantize_row_flat_q4_k_f32; break; + case HTP_TYPE_Q5_K: grctx.compute_flat = hvx_dequantize_row_flat_q5_k_f32; break; + case HTP_TYPE_Q6_K: grctx.compute_flat = hvx_dequantize_row_flat_q6_k_f32; break; default: return HTP_STATUS_NO_SUPPORT; } + q_func = get_rows_thread_flat; break; default: return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n", + const uint32_t elem_size = (grctx.idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t); + const size_t src1_bytes = (size_t) total_tasks * elem_size; + + if (!htp_tensor_is_extended(octx->src[1]) && src1_bytes <= 128 && + htp_tensor_is_contiguous(octx->src[1], elem_size)) { + grctx.src1_data = (const void *) (uintptr_t) octx->src[1]->data; + grctx.read_index = get_rows_read_index_ddr; + } else { + uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, grctx.vtcm_base, grctx.vtcm_layout.off_src1); + get_rows_dma_src1(octx, vtcm_src1, elem_size); + grctx.src1_data = vtcm_src1; + grctx.read_index = get_rows_read_index_vtcm; + } + + FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-hvx %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - grctx.vtcm_layout.src0_bytes_per_thread * n_threads, - grctx.vtcm_layout.dst_bytes_per_thread * n_threads, - kparams->kernel_type, n_threads); + grctx.vtcm_layout.src0_bytes_per_hvx * n_hvx, + grctx.vtcm_layout.dst_bytes_per_hvx * n_hvx, + kparams->kernel_type, n_hvx); - work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, q_func, &grctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h index 06ca1ea74..986966f11 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h @@ -5,7 +5,8 @@ #include <string.h> #include "hex-fastdiv.h" -#include "matmul-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" enum htp_get_rows_kernel_type { HTP_GET_ROWS_KERNEL_SAMETYPE = 0, @@ -14,86 +15,71 @@ enum htp_get_rows_kernel_type { }; struct htp_get_rows_kernel_params { - int32_t n_threads; - int32_t kernel_type; - int32_t chunks_per_row; - int32_t chunk_size; - int32_t total_tasks; - int32_t tasks_per_thread; - int32_t vtcm_size; + uint8_t kernel_type; + uint32_t n_hvx; + uint32_t total_tasks; + uint32_t vtcm_size; // Fastdiv helpers struct fastdiv_values div_ne10; struct fastdiv_values div_ne10_ne11; - struct fastdiv_values div_chunks_per_row; struct fastdiv_values div_ne02; struct fastdiv_values div_ne03; }; struct htp_get_rows_vtcm_layout { size_t total_bytes; + size_t off_src1; + size_t src1_bytes; size_t off_src0; size_t off_dst; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; }; static inline void htp_get_rows_vtcm_layout_build( - struct htp_get_rows_vtcm_layout * vtcm_layout, + struct htp_get_rows_vtcm_layout * L, int kernel_type, int type, uint32_t ne00, - uint32_t n_threads) { - - if (kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) { - memset(vtcm_layout, 0, sizeof(*vtcm_layout)); - return; - } - - if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) { - const size_t tile_size = type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0; - const size_t tile_stride = (tile_size + 127) & ~127; - const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS; - const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride; - vtcm_layout->src0_spad_half_size = (row_tiles_size + 255) & ~255; - vtcm_layout->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255; - vtcm_layout->src0_bytes_per_thread = 2 * vtcm_layout->src0_spad_half_size; - vtcm_layout->dst_bytes_per_thread = 2 * vtcm_layout->dst_spad_half_size; - vtcm_layout->off_src0 = 0; - vtcm_layout->off_dst = vtcm_layout->src0_bytes_per_thread * n_threads; - vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads; - return; - } - - uint32_t src0_row_size = 0; - switch (type) { - case HTP_TYPE_F16: - src0_row_size = ne00 * 2; - break; - case HTP_TYPE_Q8_0: - src0_row_size = (ne00 / 32) * 34; - break; - default: - src0_row_size = 0; - break; + uint32_t total_tasks, + int idx_type, + uint32_t n_hvx) { + + memset(L, 0, sizeof(*L)); + + const size_t elem_size = (idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t); + const size_t src1_raw_bytes = total_tasks * elem_size; + const size_t src1_bytes = ((src1_raw_bytes + 127) & ~127) + 128; + + L->src1_bytes = src1_bytes; + const size_t src1_alloc_bytes = (src1_bytes + 255) & ~255; + + if (kernel_type != HTP_GET_ROWS_KERNEL_SAMETYPE) { + size_t src0_row_size = 0; + if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) { + const size_t tile_stride = htp_tensor_get_tile_padded_size(type); + const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(type); + src0_row_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride; + } else { + src0_row_size = htp_tensor_get_row_size(type, ne00); + } + + L->src0_spad_half_size = (src0_row_size + 255) & ~255; + L->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255; + L->src0_bytes_per_hvx = 2 * L->src0_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; } - size_t src0_row_size_aligned = (src0_row_size + 255) & ~255; - size_t dst_row_size_aligned = (ne00 * sizeof(float) + 255) & ~255; - - vtcm_layout->src0_spad_half_size = src0_row_size_aligned; - vtcm_layout->dst_spad_half_size = dst_row_size_aligned; - - vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2; - vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2; - - vtcm_layout->off_src0 = 0; - vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads; - vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads; + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src1, src1_alloc_bytes); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + L->total_bytes = off; } #if defined(__cplusplus) diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.c similarity index 65% rename from ggml/src/ggml-hexagon/htp/act-ops.c rename to ggml/src/ggml-hexagon/htp/glu-ops.c index 59a0575c4..223c1829f 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/glu-ops.c @@ -17,11 +17,12 @@ #include "hex-common.h" #include "htp-tensor.h" #include "htp-vtcm.h" +#include "glu-ops.h" -#define htp_act_preamble \ - const struct htp_tensor * src0 = actx->octx->src[0]; \ - const struct htp_tensor * src1 = actx->octx->src[1]; \ - const struct htp_tensor * dst = actx->octx->dst; \ +#define htp_glu_preamble \ + const struct htp_tensor * src0 = gctx->octx->src[0]; \ + const struct htp_tensor * src1 = gctx->octx->src[1]; \ + const struct htp_tensor * dst = gctx->octx->dst; \ \ const uint32_t ne00 = src0->ne[0]; \ const uint32_t ne01 = src0->ne[1]; \ @@ -53,21 +54,21 @@ const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; -struct htp_act_context; +struct htp_glu_context; typedef void (*glu_compute_f32_fn_t)(const float * restrict src0, const float * restrict src1, float * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx); + const struct htp_glu_context * gctx); typedef void (*glu_compute_f16_fn_t)(const _Float16 * restrict src0, const _Float16 * restrict src1, _Float16 * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx); + const struct htp_glu_context * gctx); -struct htp_act_context { +struct htp_glu_context { struct htp_ops_context * octx; void * compute; @@ -95,7 +96,7 @@ struct htp_act_context { uint32_t block; uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; int nc; @@ -103,58 +104,23 @@ struct htp_act_context { uint8_t * vtcm_src1; uint8_t * vtcm_dst; - size_t vtcm_src0_size_per_thread; - size_t vtcm_src1_size_per_thread; - size_t vtcm_dst_size_per_thread; + size_t vtcm_src0_size_per_hvx; + size_t vtcm_src1_size_per_hvx; + size_t vtcm_dst_size_per_hvx; }; -struct htp_act_vtcm_layout { - size_t total_bytes; - size_t off_src0; - size_t off_src1; - size_t off_dst; - - size_t src0_bytes_per_thread; - size_t src1_bytes_per_thread; - size_t dst_bytes_per_thread; - - uint32_t vtcm_row_per_thread; -}; - -static inline void htp_act_vtcm_layout_build(struct htp_act_vtcm_layout * L, - size_t src0_row_size_aligned, - size_t src1_row_size_aligned, - size_t dst_row_size_aligned, - uint32_t n_threads, - size_t vtcm_size) { - const size_t spad_size_per_row = src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned; - const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * spad_size_per_row)); - - L->vtcm_row_per_thread = vtcm_row_per_thread; - - L->src0_bytes_per_thread = src0_row_size_aligned * vtcm_row_per_thread; - L->src1_bytes_per_thread = src1_row_size_aligned * vtcm_row_per_thread; - L->dst_bytes_per_thread = dst_row_size_aligned * vtcm_row_per_thread; - - L->off_src0 = 0; - L->off_src1 = L->off_src0 + L->src0_bytes_per_thread * n_threads; - L->off_dst = L->off_src1 + L->src1_bytes_per_thread * n_threads; - - L->total_bytes = L->off_dst + L->dst_bytes_per_thread * n_threads; -} - #define htp_glu_op_preamble \ - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ - const int nc = actx->nc; + const size_t src0_row_size_aligned = gctx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \ + const int nc = gctx->nc; // swiglu(x) = x1 * sigmoid(x0) static void swiglu_f32(const float * restrict src0, const float * restrict src1, float * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; for (uint32_t ib = 0; ib < num_rows; ib++) { @@ -172,10 +138,10 @@ static void swiglu_oai_f32(const float * restrict src0, const float * restrict src1, float * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; - const float alpha = ((const float *) (actx->octx->op_params))[2]; - const float limit = ((const float *) (actx->octx->op_params))[3]; + const float alpha = ((const float *) (gctx->octx->op_params))[2]; + const float limit = ((const float *) (gctx->octx->op_params))[3]; for (uint32_t ib = 0; ib < num_rows; ib++) { const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); @@ -201,9 +167,9 @@ static void swiglu_clamp_f32(const float * restrict src0, const float * restrict src1, float * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; - const float limit = ((const float *) (actx->octx->op_params))[3]; + const float limit = ((const float *) (gctx->octx->op_params))[3]; for (uint32_t ib = 0; ib < num_rows; ib++) { const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); @@ -220,53 +186,6 @@ static void swiglu_clamp_f32(const float * restrict src0, static const float GELU_COEF_A = 0.044715f; static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; -static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) { - v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); - v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3)); - - HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); - HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); - HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); - - HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); - v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); - - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); - v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); - v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); - - HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); - v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); - - HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); - HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); - - // Newton-Raphson with 2 iterations - HVX_Vector two_sf = hvx_vec_splat_f32(2.0f); - HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); - HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( - i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); - r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( - r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); - HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf); - - res = Q6_Vqf32_vmpy_VsfVsf(v3, res); - - return Q6_Vsf_equals_Vqf32(res); -} - -static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v, - HVX_Vector one, - HVX_Vector max_exp, - HVX_Vector min_exp) { - const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v); - const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp); - - HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v); - out = Q6_V_vmux_QVV(pred_max, out, one); - return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); -} - static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src0 % 128 == 0); @@ -455,7 +374,7 @@ static void geglu_f32(const float * restrict src0, const float * restrict src1, float * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; for (uint32_t ib = 0; ib < num_rows; ib++) { @@ -472,7 +391,7 @@ static void geglu_quick_f32(const float * restrict src0, const float * restrict src1, float * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; for (uint32_t ib = 0; ib < num_rows; ib++) { @@ -489,7 +408,7 @@ static void geglu_erf_f32(const float * restrict src0, const float * restrict src1, float * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; for (uint32_t ib = 0; ib < num_rows; ib++) { @@ -501,12 +420,124 @@ static void geglu_erf_f32(const float * restrict src0, } } +static inline void hvx_reglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_zero = Q6_V_vzero(); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = Q6_Vsf_vmax_VsfVsf(vsrc0[i], v_zero); + vdst[i] = hvx_vec_mul_f32_f32(x, vsrc1[i]); + } + + if (nloe) { + HVX_Vector x = Q6_Vsf_vmax_VsfVsf(vsrc0[i], v_zero); + HVX_Vector res = hvx_vec_mul_f32_f32(x, vsrc1[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res); + } +} + +// reglu(x, g) = relu(x) * g +static void reglu_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_reglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +static __attribute__((noinline)) void hvx_sigglu_f32_vec(HVX_Vector * restrict vdst, + const HVX_Vector * restrict vsrc0, + const HVX_Vector * restrict vsrc1, + uint32_t nvec) { + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(vsrc0[i], v_one, v_max_exp, v_min_exp); + vdst[i] = hvx_vec_mul_f32_f32(sig, vsrc1[i]); + } +} + +static __attribute__((noinline)) void hvx_sigglu_f32_tail(HVX_Vector * restrict vdst, + const HVX_Vector * restrict vsrc0, + const HVX_Vector * restrict vsrc1, + uint32_t nloe) { + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(vsrc0[0], v_one, v_max_exp, v_min_exp); + HVX_Vector res = hvx_vec_mul_f32_f32(sig, vsrc1[0]); + hvx_vec_store_a((void *) vdst, nloe * sizeof(float), res); +} + +static inline void hvx_sigglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + if (nvec) { + hvx_sigglu_f32_vec(vdst, vsrc0, vsrc1, nvec); + } + if (nloe) { + hvx_sigglu_f32_tail(vdst + nvec, vsrc0 + nvec, vsrc1 + nvec, nloe); + } +} + +// sigglu(x, g) = sigmoid(x) * g +static void sigglu_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_sigglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + // swiglu(x) = x1 * sigmoid(x0) static void swiglu_f16(const _Float16 * restrict src0, const _Float16 * restrict src1, _Float16 * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; for (uint32_t ib = 0; ib < num_rows; ib++) { @@ -524,10 +555,10 @@ static void swiglu_oai_f16(const _Float16 * restrict src0, const _Float16 * restrict src1, _Float16 * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; - const _Float16 alpha = (_Float16) ((const float *) (actx->octx->op_params))[2]; - const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3]; + const _Float16 alpha = (_Float16) ((const float *) (gctx->octx->op_params))[2]; + const _Float16 limit = (_Float16) ((const float *) (gctx->octx->op_params))[3]; for (uint32_t ib = 0; ib < num_rows; ib++) { const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); @@ -553,9 +584,9 @@ static void swiglu_clamp_f16(const _Float16 * restrict src0, const _Float16 * restrict src1, _Float16 * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; - const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3]; + const _Float16 limit = (_Float16) ((const float *) (gctx->octx->op_params))[3]; for (uint32_t ib = 0; ib < num_rows; ib++) { const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); @@ -574,7 +605,7 @@ static void geglu_f16(const _Float16 * restrict src0, const _Float16 * restrict src1, _Float16 * restrict dst, const uint32_t num_rows, - const struct htp_act_context * actx) { + const struct htp_glu_context * gctx) { htp_glu_op_preamble; for (uint32_t ib = 0; ib < num_rows; ib++) { @@ -586,56 +617,155 @@ static void geglu_f16(const _Float16 * restrict src0, } } +static inline void hvx_reglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(_Float16); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_zero = Q6_V_vzero(); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = Q6_Vhf_vmax_VhfVhf(vsrc0[i], v_zero); + vdst[i] = hvx_vec_mul_f16_f16(x, vsrc1[i]); + } + + if (nloe) { + HVX_Vector x = Q6_Vhf_vmax_VhfVhf(vsrc0[i], v_zero); + HVX_Vector res = hvx_vec_mul_f16_f16(x, vsrc1[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res); + } +} + +// reglu(x, g) = relu(x) * g +static void reglu_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_reglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +static __attribute__((noinline)) void hvx_sigglu_f16_vec(HVX_Vector * restrict vdst, + const HVX_Vector * restrict vsrc0, + const HVX_Vector * restrict vsrc1, + uint32_t nvec) { + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector sig = hvx_vec_fast_sigmoid_f16(vsrc0[i]); + vdst[i] = hvx_vec_mul_f16_f16(sig, vsrc1[i]); + } +} + +static __attribute__((noinline)) void hvx_sigglu_f16_tail(HVX_Vector * restrict vdst, + const HVX_Vector * restrict vsrc0, + const HVX_Vector * restrict vsrc1, + uint32_t nloe) { + HVX_Vector sig = hvx_vec_fast_sigmoid_f16(vsrc0[0]); + HVX_Vector res = hvx_vec_mul_f16_f16(sig, vsrc1[0]); + hvx_vec_store_a((void *) vdst, nloe * sizeof(_Float16), res); +} + +static inline void hvx_sigglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(_Float16); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + if (nvec) { + hvx_sigglu_f16_vec(vdst, vsrc0, vsrc1, nvec); + } + if (nloe) { + hvx_sigglu_f16_tail(vdst + nvec, vsrc0 + nvec, vsrc1 + nvec, nloe); + } +} + +// sigglu(x, g) = sigmoid(x) * g +static void sigglu_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_sigglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + #define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \ - static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_act_context * actx = (struct htp_act_context *) data; \ - htp_act_preamble; \ + static void glu_##SUFFIX##_per_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ + struct htp_glu_context * gctx = (struct htp_glu_context *) data; \ + htp_glu_preamble; \ \ - struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = th->trace; \ \ - size_t src0_row_size = actx->src0_row_size; \ - size_t src1_row_size = actx->src1_row_size; \ - size_t dst_row_size = actx->dst_row_size; \ + size_t src0_row_size = gctx->src0_row_size; \ + size_t src1_row_size = gctx->src1_row_size; \ + size_t dst_row_size = gctx->dst_row_size; \ \ - size_t src0_row_stride = actx->src0_row_stride; \ - size_t src1_row_stride = actx->src1_row_stride; \ + size_t src0_row_stride = gctx->src0_row_stride; \ + size_t src1_row_stride = gctx->src1_row_stride; \ \ - const uint32_t src0_nrows = actx->src0_nrows; \ - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \ + const uint32_t src0_nrows = gctx->src0_nrows; \ + const uint32_t src0_nrows_per_hvx = gctx->src0_nrows_per_hvx; \ \ - const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \ + const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_hvx * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, gctx->row_start + src0_nrows); \ \ /* no work for this thread */ \ if (src0_start_row >= src0_end_row) { \ return; \ } \ \ - const dma_addr_t data_src0 = actx->data_src0; \ - const dma_addr_t data_src1 = actx->data_src1; \ - const dma_addr_t data_dst = actx->data_dst; \ + const dma_addr_t data_src0 = gctx->data_src0; \ + const dma_addr_t data_src1 = gctx->data_src1; \ + const dma_addr_t data_dst = gctx->data_dst; \ \ - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ + const size_t src0_row_size_aligned = gctx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \ \ - uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \ - uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \ - uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \ + uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_hvx); \ + uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_hvx); \ + uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_hvx); \ \ - size_t src0_spad_half_size = actx->src0_spad_half_size; \ - size_t src1_spad_half_size = actx->src1_spad_half_size; \ - size_t dst_spad_half_size = actx->dst_spad_half_size; \ + size_t src0_spad_half_size = gctx->src0_spad_half_size; \ + size_t src1_spad_half_size = gctx->src1_spad_half_size; \ + size_t dst_spad_half_size = gctx->dst_spad_half_size; \ \ - const int BLOCK = actx->block; \ - if (BLOCK == 0) { \ - FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", \ - actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ - return; \ - } \ + const int BLOCK = gctx->block; \ \ - dma_queue * dma_q = actx->octx->ctx->dma[ith]; \ - glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) actx->compute; \ + dma_queue * dma_q = th->dma_q; \ + glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) gctx->compute; \ \ for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ @@ -662,7 +792,7 @@ static void geglu_f16(const _Float16 * restrict src0, TYPE * src1_spad = (TYPE *) dma_queue_pop(dma_q).dst; \ \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - compute(src0_spad, src1_spad, dst_spad, block_size, actx); \ + compute(src0_spad, src1_spad, dst_spad, block_size, gctx); \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ \ dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), \ @@ -685,16 +815,11 @@ static void geglu_f16(const _Float16 * restrict src0, DEFINE_GLU_PER_THREAD(float, f32) DEFINE_GLU_PER_THREAD(_Float16, f16) -static int execute_op_activations_f32(struct htp_ops_context * octx) { +static int execute_op_glu_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - if ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1]) { - FARF(ERROR, "Non-contiguous dst is not supported at this time \n"); - return HTP_STATUS_NO_SUPPORT; - } - glu_compute_f32_fn_t compute_fn = NULL; const char * op_type = NULL; @@ -729,11 +854,27 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { op_type = "geglu-erf-f32"; break; + case HTP_OP_GLU_REGLU: + compute_fn = reglu_f32; + op_type = "reglu-f32"; + break; + + case HTP_OP_GLU_SIGMOID: + compute_fn = sigglu_f32; + op_type = "sigglu-f32"; + break; + default: - FARF(ERROR, "Unsupported activations Op %u\n", octx->op); + FARF(ERROR, "Unsupported GLU Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; } + const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const size_t dst_row_size = dst->ne[0] * SIZEOF_FP32; @@ -752,7 +893,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). @@ -762,85 +903,79 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { const size_t src0_row_stride = src0->nb[1]; const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); + struct htp_glu_vtcm_layout L; + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, kparams->vtcm_row_per_hvx); - struct htp_act_vtcm_layout L; - htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads, - octx->ctx->vtcm_size); + const size_t src0_row_size_aligned = L.row_size_aligned; + const size_t src1_row_size_aligned = L.row_size_aligned; + const size_t dst_row_size_aligned = L.row_size_aligned; // Make sure the reserved vtcm size is sufficient - if (L.vtcm_row_per_thread == 0) { - FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, - (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + if (kparams->vtcm_row_per_hvx == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { + FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_hvx); return HTP_STATUS_VTCM_TOO_SMALL; } if (src1) { FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, - L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_hvx * n_hvx, + L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } else { FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + L.src0_bytes_per_hvx * n_hvx, L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } // Prepare context - struct htp_act_context actx; - actx.octx = octx; - actx.compute = (void *) compute_fn; - actx.op_str = op_type; + struct htp_glu_context gctx; + gctx.octx = octx; + gctx.compute = (void *) compute_fn; + gctx.op_str = op_type; - actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + gctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); - actx.src0_row_size = src0_row_size; - actx.src1_row_size = src1_row_size; - actx.dst_row_size = dst_row_size; + gctx.src0_row_size = src0_row_size; + gctx.src1_row_size = src1_row_size; + gctx.dst_row_size = dst_row_size; - actx.src0_row_size_aligned = src0_row_size_aligned; - actx.src1_row_size_aligned = src1_row_size_aligned; - actx.dst_row_size_aligned = dst_row_size_aligned; + gctx.src0_row_size_aligned = src0_row_size_aligned; + gctx.src1_row_size_aligned = src1_row_size_aligned; + gctx.dst_row_size_aligned = dst_row_size_aligned; - actx.src0_row_stride = src0_row_stride; - actx.src1_row_stride = src1_row_stride; + gctx.src0_row_stride = src0_row_stride; + gctx.src1_row_stride = src1_row_stride; uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); - actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); - actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + gctx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread; - actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread; - actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread; + gctx.vtcm_src0_size_per_hvx = (uint32_t) L.src0_bytes_per_hvx; + gctx.vtcm_src1_size_per_hvx = (uint32_t) L.src1_bytes_per_hvx; + gctx.vtcm_dst_size_per_hvx = (uint32_t) L.dst_bytes_per_hvx; - actx.src0_spad_half_size = L.src0_bytes_per_thread / 2; - actx.src1_spad_half_size = L.src1_bytes_per_thread / 2; - actx.dst_spad_half_size = L.dst_bytes_per_thread / 2; + gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_hvx >> 1; + gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_hvx >> 1; + gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_hvx >> 1; - actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned; - actx.src0_nrows = nrows; - actx.row_start = row_start; + gctx.block = L.block; + gctx.src0_nrows = nrows; + gctx.row_start = row_start; - actx.nc = dst->ne[0]; + gctx.nc = dst->ne[0]; // Addresses and GLU logic dma_addr_t data_src0 = src0->data; dma_addr_t data_src1 = src1 ? src1->data : 0; - if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || - octx->op == HTP_OP_GLU_SWIGLU_OAI || - octx->op == HTP_OP_GLU_SWIGLU_CLAMP || - octx->op == HTP_OP_GLU_GEGLU || - octx->op == HTP_OP_GLU_GEGLU_QUICK || - octx->op == HTP_OP_GLU_GEGLU_ERF)) { + if (!src1) { const int32_t swapped = octx->op_params[1]; data_src1 = data_src0; - actx.src1_row_size = actx.src0_row_size; + gctx.src1_row_size = gctx.src0_row_size; - size_t nc_in_bytes = actx.nc * SIZEOF_FP32; + size_t nc_in_bytes = gctx.nc * SIZEOF_FP32; if (swapped) { data_src0 += nc_in_bytes; } else { @@ -848,24 +983,19 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { } } - actx.data_src0 = data_src0; - actx.data_src1 = data_src1; - actx.data_dst = dst->data; + gctx.data_src0 = data_src0; + gctx.data_src1 = data_src1; + gctx.data_dst = dst->data; - work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, (work_queue_func_t)glu_f32_per_hvx, &gctx, n_hvx); return HTP_STATUS_OK; } -static int execute_op_activations_f16(struct htp_ops_context * octx) { +static int execute_op_glu_f16(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - if ((dst->ne[0] * SIZEOF_FP16) != dst->nb[1]) { - FARF(ERROR, "Non-contiguous dst is not supported at this time \n"); - return HTP_STATUS_NO_SUPPORT; - } - glu_compute_f16_fn_t compute_fn = NULL; const char * op_type = NULL; @@ -890,11 +1020,27 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) { op_type = "geglu-f16"; break; + case HTP_OP_GLU_REGLU: + compute_fn = reglu_f16; + op_type = "reglu-f16"; + break; + + case HTP_OP_GLU_SIGMOID: + compute_fn = sigglu_f16; + op_type = "sigglu-f16"; + break; + default: - FARF(ERROR, "Unsupported activations Op %u\n", octx->op); + FARF(ERROR, "Unsupported GLU Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; } + const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const size_t dst_row_size = dst->ne[0] * SIZEOF_FP16; @@ -913,7 +1059,7 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). @@ -923,83 +1069,79 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) { const size_t src0_row_stride = src0->nb[1]; const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); + struct htp_glu_vtcm_layout L; + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, kparams->vtcm_row_per_hvx); - struct htp_act_vtcm_layout L; - htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads, - octx->ctx->vtcm_size); + const size_t src0_row_size_aligned = L.row_size_aligned; + const size_t src1_row_size_aligned = L.row_size_aligned; + const size_t dst_row_size_aligned = L.row_size_aligned; // Make sure the reserved vtcm size is sufficient - if (L.vtcm_row_per_thread == 0) { - FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, - (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + if (kparams->vtcm_row_per_hvx == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { + FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_hvx); return HTP_STATUS_VTCM_TOO_SMALL; } if (src1) { FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, - L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_hvx * n_hvx, + L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } else { FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + L.src0_bytes_per_hvx * n_hvx, L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } // Prepare context - struct htp_act_context actx; - actx.octx = octx; - actx.compute = (void *) compute_fn; - actx.op_str = op_type; + struct htp_glu_context gctx; + gctx.octx = octx; + gctx.compute = (void *) compute_fn; + gctx.op_str = op_type; - actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + gctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); - actx.src0_row_size = src0_row_size; - actx.src1_row_size = src1_row_size; - actx.dst_row_size = dst_row_size; + gctx.src0_row_size = src0_row_size; + gctx.src1_row_size = src1_row_size; + gctx.dst_row_size = dst_row_size; - actx.src0_row_size_aligned = src0_row_size_aligned; - actx.src1_row_size_aligned = src1_row_size_aligned; - actx.dst_row_size_aligned = dst_row_size_aligned; + gctx.src0_row_size_aligned = src0_row_size_aligned; + gctx.src1_row_size_aligned = src1_row_size_aligned; + gctx.dst_row_size_aligned = dst_row_size_aligned; - actx.src0_row_stride = src0_row_stride; - actx.src1_row_stride = src1_row_stride; + gctx.src0_row_stride = src0_row_stride; + gctx.src1_row_stride = src1_row_stride; uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); - actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); - actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + gctx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread; - actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread; - actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread; + gctx.vtcm_src0_size_per_hvx = (uint32_t) L.src0_bytes_per_hvx; + gctx.vtcm_src1_size_per_hvx = (uint32_t) L.src1_bytes_per_hvx; + gctx.vtcm_dst_size_per_hvx = (uint32_t) L.dst_bytes_per_hvx; - actx.src0_spad_half_size = L.src0_bytes_per_thread / 2; - actx.src1_spad_half_size = L.src1_bytes_per_thread / 2; - actx.dst_spad_half_size = L.dst_bytes_per_thread / 2; + gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_hvx >> 1; + gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_hvx >> 1; + gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_hvx >> 1; - actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned; - actx.src0_nrows = nrows; - actx.row_start = row_start; + gctx.block = L.block; + gctx.src0_nrows = nrows; + gctx.row_start = row_start; - actx.nc = dst->ne[0]; + gctx.nc = dst->ne[0]; // Addresses and GLU logic dma_addr_t data_src0 = src0->data; dma_addr_t data_src1 = src1 ? src1->data : 0; - if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || - octx->op == HTP_OP_GLU_SWIGLU_OAI || - octx->op == HTP_OP_GLU_SWIGLU_CLAMP || - octx->op == HTP_OP_GLU_GEGLU)) { + if (!src1) { const int32_t swapped = octx->op_params[1]; data_src1 = data_src0; - actx.src1_row_size = actx.src0_row_size; + gctx.src1_row_size = gctx.src0_row_size; - size_t nc_in_bytes = actx.nc * SIZEOF_FP16; + size_t nc_in_bytes = gctx.nc * SIZEOF_FP16; if (swapped) { data_src0 += nc_in_bytes; } else { @@ -1007,22 +1149,22 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) { } } - actx.data_src0 = data_src0; - actx.data_src1 = data_src1; - actx.data_dst = dst->data; + gctx.data_src0 = data_src0; + gctx.data_src1 = data_src1; + gctx.data_dst = dst->data; - work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, (work_queue_func_t)glu_f16_per_hvx, &gctx, n_hvx); return HTP_STATUS_OK; } -int op_activations(struct htp_ops_context * octx) { +int op_glu(struct htp_ops_context * octx) { switch (octx->src[0]->type) { case HTP_TYPE_F32: - return execute_op_activations_f32(octx); + return execute_op_glu_f32(octx); case HTP_TYPE_F16: - return execute_op_activations_f16(octx); + return execute_op_glu_f16(octx); default: return HTP_STATUS_NO_SUPPORT; diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.h b/ggml/src/ggml-hexagon/htp/glu-ops.h new file mode 100644 index 000000000..5e248f7d4 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/glu-ops.h @@ -0,0 +1,76 @@ +#ifndef HTP_GLU_OPS_H +#define HTP_GLU_OPS_H + +#include "hex-common.h" +#include "htp-ops.h" +#include "htp-vtcm.h" +#include <stdint.h> + +struct htp_glu_kernel_params { + uint32_t n_hvx; + uint32_t vtcm_row_per_hvx; + uint32_t vtcm_size; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob"); +#endif + +static inline bool htp_op_is_glu(uint32_t opcode) { + switch (opcode) { + case HTP_OP_GLU_SWIGLU: + case HTP_OP_GLU_SWIGLU_OAI: + case HTP_OP_GLU_SWIGLU_CLAMP: + case HTP_OP_GLU_GEGLU: + case HTP_OP_GLU_GEGLU_QUICK: + case HTP_OP_GLU_GEGLU_ERF: + case HTP_OP_GLU_REGLU: + case HTP_OP_GLU_SIGMOID: + return true; + default: + return false; + } +} + +struct htp_glu_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_src1; + size_t off_dst; + + size_t row_size_aligned; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; + + uint32_t vtcm_row_per_hvx; + uint32_t block; +}; + +static inline void htp_glu_vtcm_layout_build( + struct htp_glu_vtcm_layout * L, + size_t dst_row_size, + uint32_t n_hvx, + uint32_t vtcm_row_per_hvx +) { + const size_t row_size_aligned = hex_round_up(dst_row_size, 128); + + L->row_size_aligned = row_size_aligned; + L->vtcm_row_per_hvx = vtcm_row_per_hvx; + L->block = vtcm_row_per_hvx >> 1; + + L->src0_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; + L->src1_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; + L->dst_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + + L->total_bytes = off; +} + +#endif // HTP_GLU_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/hex-common.h b/ggml/src/ggml-hexagon/htp/hex-common.h index e6a52540d..b943453a3 100644 --- a/ggml/src/ggml-hexagon/htp/hex-common.h +++ b/ggml/src/ggml-hexagon/htp/hex-common.h @@ -17,6 +17,10 @@ #define MIN(a, b) ((a) < (b) ? (a) : (b)) #endif +#ifndef HEX_L2_LINE_SIZE +#define HEX_L2_LINE_SIZE 128 +#endif + static inline uint32_t hex_ceil_pow2(uint32_t x) { if (x <= 1) { return 1; } int p = 2; diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.h deleted file mode 100644 index a26961fc9..000000000 --- a/ggml/src/ggml-hexagon/htp/hex-profile.h +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef HEX_PROFILE_H -#define HEX_PROFILE_H - -#include <stdbool.h> -#include <stdint.h> -#include <qurt.h> - -#include "hex-utils.h" -#include "htp-ops.h" - -#define HTP_TRACE_EVT_START 0 -#define HTP_TRACE_EVT_STOP 1 - -#ifndef HEX_NUM_PMU_COUNTERS -#define HEX_NUM_PMU_COUNTERS 8 -#endif - -static inline void hex_get_pmu(uint32_t counters[]) { -#if __HVX_ARCH__ >= 79 - asm volatile("%0 = upmucnt0" : "=r"(counters[0])); - asm volatile("%0 = upmucnt1" : "=r"(counters[1])); - asm volatile("%0 = upmucnt2" : "=r"(counters[2])); - asm volatile("%0 = upmucnt3" : "=r"(counters[3])); - asm volatile("%0 = upmucnt4" : "=r"(counters[4])); - asm volatile("%0 = upmucnt5" : "=r"(counters[5])); - asm volatile("%0 = upmucnt6" : "=r"(counters[6])); - asm volatile("%0 = upmucnt7" : "=r"(counters[7])); -#else - counters[0] = qurt_pmu_get(QURT_PMUCNT0); - counters[1] = qurt_pmu_get(QURT_PMUCNT1); - counters[2] = qurt_pmu_get(QURT_PMUCNT2); - counters[3] = qurt_pmu_get(QURT_PMUCNT3); - counters[4] = qurt_pmu_get(QURT_PMUCNT4); - counters[5] = qurt_pmu_get(QURT_PMUCNT5); - counters[6] = qurt_pmu_get(QURT_PMUCNT6); - counters[7] = qurt_pmu_get(QURT_PMUCNT7); -#endif -} - -struct htp_thread_trace { - uint32_t count; - uint32_t max_events; - struct htp_trace_desc * events; -}; - -static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { - if (tr->count < tr->max_events) { - uint32_t i = tr->count; - tr->events[i].id = id; - tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); - tr->events[i].cycles = (uint32_t) hex_get_cycles(); - tr->count++; - } -} - -static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { - htp_trace_event(tr, id, info, HTP_TRACE_EVT_START); -} - -static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { - htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP); -} - -#endif /* HEX_PROFILE_H */ diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index 853f1c1b2..cb3093dec 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -25,6 +25,32 @@ static inline uint64_t hex_get_pktcnt() { return pktcnt; } +#ifndef HEX_NUM_PMU_COUNTERS +#define HEX_NUM_PMU_COUNTERS 8 +#endif + +static inline void hex_get_pmu(uint32_t counters[]) { +#if __HVX_ARCH__ >= 79 + asm volatile("%0 = upmucnt0" : "=r"(counters[0])); + asm volatile("%0 = upmucnt1" : "=r"(counters[1])); + asm volatile("%0 = upmucnt2" : "=r"(counters[2])); + asm volatile("%0 = upmucnt3" : "=r"(counters[3])); + asm volatile("%0 = upmucnt4" : "=r"(counters[4])); + asm volatile("%0 = upmucnt5" : "=r"(counters[5])); + asm volatile("%0 = upmucnt6" : "=r"(counters[6])); + asm volatile("%0 = upmucnt7" : "=r"(counters[7])); +#else + counters[0] = qurt_pmu_get(QURT_PMUCNT0); + counters[1] = qurt_pmu_get(QURT_PMUCNT1); + counters[2] = qurt_pmu_get(QURT_PMUCNT2); + counters[3] = qurt_pmu_get(QURT_PMUCNT3); + counters[4] = qurt_pmu_get(QURT_PMUCNT4); + counters[5] = qurt_pmu_get(QURT_PMUCNT5); + counters[6] = qurt_pmu_get(QURT_PMUCNT6); + counters[7] = qurt_pmu_get(QURT_PMUCNT7); +#endif +} + static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, uint32_t height) { const uint64_t control = Q6_P_combine_RR(stride, Q6_R_combine_RlRl(width, height)); Q6_l2fetch_AP((void *) p, control); @@ -37,22 +63,11 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) { hex_l2fetch(addr, width, width, height); } -#define HEX_L2_LINE_SIZE 128 -#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration) -#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) -#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) - static inline void hex_l2flush(void * addr, size_t size) { + if (size == 0) return; const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1); const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1); - const uint32_t eb = s + ((e - s) & ~(HEX_L2_BLOCK_SIZE - 1)); - for (uint32_t i = s; i < eb; i += HEX_L2_BLOCK_SIZE) { - Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 0)); - Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 1)); - Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 2)); - Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 3)); - } - for (uint32_t i = eb; i < e; i += HEX_L2_LINE_SIZE) { + for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE) { Q6_dccleaninva_A((void *) i); } } diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 7d7455d04..a828012dd 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -14,14 +14,17 @@ static const __fp16 iq4_nl_to_fp16_lut[64] __attribute__((aligned(VLEN))) = { // --- tiled format dequantizers --- -typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; +typedef struct tiled_dequantize_state tiled_dequantize_state_t; +typedef void (*tiled_dequant_task_fn_t)(const tiled_dequantize_state_t * state, uint32_t start_tile, uint32_t end_tile); + +struct tiled_dequantize_state { __fp16 * dst; const uint8_t * src; + tiled_dequant_task_fn_t task_fn; struct fastdiv_values n_k_tiles_div; uint32_t n_k_tiles; + uint32_t n_weight_tiles; uint32_t n_tot_tiles; uint32_t n_tiles_per_task; uint32_t tile_size; @@ -31,7 +34,40 @@ typedef struct { uint32_t k_block; size_t row_stride; uint32_t weight_type; -} tiled_dequantize_state_t; +}; + +static void dequantize_tiled_weight_to_fp16_task_q1_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + const HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + 4 * t * HTP_MM_HMX_TILE_N_ELMS; + + const HVX_Vector v_sc = hvx_vmem(tile_src + 512); + const HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + for (uint32_t q = 0; q < 4; q++) { + const HVX_Vector v_plane = hvx_vmem(tile_src + q * 128); + __fp16 * quarter_dst = dst_ptr + q * HTP_MM_HMX_TILE_N_ELMS; + + #pragma unroll + for (int g = 0; g < 8; g++) { + const HVX_Vector v_q = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + const HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + const HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(quarter_dst + (2 * g + 0) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); + hvx_vmem(quarter_dst + (2 * g + 1) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + } + } + } +} // Dequantize a single tile from tiled weight data (already in VTCM) to tile-major FP16. static void dequantize_tiled_weight_to_fp16_task_q4_0( @@ -611,7 +647,107 @@ static void dequantize_tiled_weight_to_fp16_task_q5_k( } } -// Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q6_K. +static void dequantize_tiled_weight_to_fp16_task_q5_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = hvx_vmem(tile_src + 640); + HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + // Load all 4 groups in parallel + HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128); + HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128); + HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128); + HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128); + + // Nibble extraction + HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4); + HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4); + HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4); + HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4); + HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4); + HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4); + HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4); + HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4); + + // Q5_0: OR 0xF0 where the plane flags a clear 5th bit (q - 16) + HVX_Vector v_plane = hvx_vmem(tile_src + 512); + v_lo0 = hvx_q5_0_or_neg(v_lo0, v_plane, 0); + v_hi0 = hvx_q5_0_or_neg(v_hi0, v_plane, 1); + v_lo1 = hvx_q5_0_or_neg(v_lo1, v_plane, 2); + v_hi1 = hvx_q5_0_or_neg(v_hi1, v_plane, 3); + v_lo2 = hvx_q5_0_or_neg(v_lo2, v_plane, 4); + v_hi2 = hvx_q5_0_or_neg(v_hi2, v_plane, 5); + v_lo3 = hvx_q5_0_or_neg(v_lo3, v_plane, 6); + v_hi3 = hvx_q5_0_or_neg(v_hi3, v_plane, 7); + + // Shuffling + HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1); + HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1); + HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1); + HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1); + + // Unpack to 16-bit + HVX_VectorPair vp_int16_lo0 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf0)); + HVX_VectorPair vp_int16_hi0 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf0)); + HVX_VectorPair vp_int16_lo1 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf1)); + HVX_VectorPair vp_int16_hi1 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf1)); + HVX_VectorPair vp_int16_lo2 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf2)); + HVX_VectorPair vp_int16_hi2 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf2)); + HVX_VectorPair vp_int16_lo3 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf3)); + HVX_VectorPair vp_int16_hi3 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf3)); + + // Convert and scale multiplication + HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo0)), v_scale_duplicated)); + HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo0)), v_scale_duplicated)); + HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi0)), v_scale_duplicated)); + HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi0)), v_scale_duplicated)); + + HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo1)), v_scale_duplicated)); + HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo1)), v_scale_duplicated)); + HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi1)), v_scale_duplicated)); + HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi1)), v_scale_duplicated)); + + HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo2)), v_scale_duplicated)); + HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo2)), v_scale_duplicated)); + HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi2)), v_scale_duplicated)); + HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi2)), v_scale_duplicated)); + + HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo3)), v_scale_duplicated)); + HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo3)), v_scale_duplicated)); + HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi3)), v_scale_duplicated)); + HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi3)), v_scale_duplicated)); + + // Parallel Stores + hvx_vmem(dst_ptr + 0 * 64) = v_grp0_0; + hvx_vmem(dst_ptr + 1 * 64) = v_grp0_1; + hvx_vmem(dst_ptr + 2 * 64) = v_grp0_2; + hvx_vmem(dst_ptr + 3 * 64) = v_grp0_3; + + hvx_vmem(dst_ptr + 4 * 64) = v_grp1_0; + hvx_vmem(dst_ptr + 5 * 64) = v_grp1_1; + hvx_vmem(dst_ptr + 6 * 64) = v_grp1_2; + hvx_vmem(dst_ptr + 7 * 64) = v_grp1_3; + + hvx_vmem(dst_ptr + 8 * 64) = v_grp2_0; + hvx_vmem(dst_ptr + 9 * 64) = v_grp2_1; + hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2; + hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3; + + hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0; + hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1; + hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2; + hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3; + } +} + +// Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_TENSOR_TILE_SIZE_Q6_K. // A k-group holds 4 k per row, the HMX tile holds 2, so each group is dealt into two tiles. static void dequantize_tiled_weight_to_fp16_task_q6_k( const tiled_dequantize_state_t *state, @@ -631,22 +767,45 @@ static void dequantize_tiled_weight_to_fp16_task_q6_k( HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2)); #pragma unroll - for (int g = 0; g < 8; g++) { + for (int g = 0; g < 8; g += 4) { const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16; - HVX_Vector v_q = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32); - HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); - HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); - - hvx_vmem(dst_ptr + (2 * g + 0) * 64) = - Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); - hvx_vmem(dst_ptr + (2 * g + 1) * 64) = - Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + HVX_Vector v_q0 = unpack_q6_k_group(vptr, g + 0, mask_0f, mask_03, i32); + HVX_Vector v_q1 = unpack_q6_k_group(vptr, g + 1, mask_0f, mask_03, i32); + HVX_Vector v_q2 = unpack_q6_k_group(vptr, g + 2, mask_0f, mask_03, i32); + HVX_Vector v_q3 = unpack_q6_k_group(vptr, g + 3, mask_0f, mask_03, i32); + + HVX_VectorPair vp16_0 = Q6_Wh_vunpack_Vb(v_q0); + HVX_VectorPair vp16_1 = Q6_Wh_vunpack_Vb(v_q1); + HVX_VectorPair vp16_2 = Q6_Wh_vunpack_Vb(v_q2); + HVX_VectorPair vp16_3 = Q6_Wh_vunpack_Vb(v_q3); + HVX_VectorPair vp_k0 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_0), Q6_V_lo_W(vp16_0), -4); + HVX_VectorPair vp_k1 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_1), Q6_V_lo_W(vp16_1), -4); + HVX_VectorPair vp_k2 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_2), Q6_V_lo_W(vp16_2), -4); + HVX_VectorPair vp_k3 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_3), Q6_V_lo_W(vp16_3), -4); + + HVX_Vector v_out00 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k0)), v_scale)); + HVX_Vector v_out01 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k0)), v_scale)); + HVX_Vector v_out10 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k1)), v_scale)); + HVX_Vector v_out11 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k1)), v_scale)); + HVX_Vector v_out20 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k2)), v_scale)); + HVX_Vector v_out21 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k2)), v_scale)); + HVX_Vector v_out30 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k3)), v_scale)); + HVX_Vector v_out31 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k3)), v_scale)); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = v_out00; + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = v_out01; + hvx_vmem(dst_ptr + (2 * g + 2) * 64) = v_out10; + hvx_vmem(dst_ptr + (2 * g + 3) * 64) = v_out11; + hvx_vmem(dst_ptr + (2 * g + 4) * 64) = v_out20; + hvx_vmem(dst_ptr + (2 * g + 5) * 64) = v_out21; + hvx_vmem(dst_ptr + (2 * g + 6) * 64) = v_out30; + hvx_vmem(dst_ptr + (2 * g + 7) * 64) = v_out31; } } } -// Q3_K stores 3-bit weights and one fp16 scale per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q3_K. +// Q3_K stores 3-bit weights and one fp16 scale per 16 k, see HTP_TENSOR_TILE_SIZE_Q3_K. static void dequantize_tiled_weight_to_fp16_task_q3_k( const tiled_dequantize_state_t *state, uint32_t start_tile, uint32_t end_tile) { @@ -678,20 +837,29 @@ static void dequantize_tiled_weight_to_fp16_task_q3_k( } } -// Q2_K stores 2-bit weights and one fp16 scale and offset per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q2_K. +// Q2_K stores 2-bit weights, the ggml scale bytes and d / -dmin per super-block, see HTP_TENSOR_TILE_SIZE_Q2_K. static void dequantize_tiled_weight_to_fp16_task_q2_k( const tiled_dequantize_state_t *state, uint32_t start_tile, uint32_t end_tile) { const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + HVX_Vector v_d = Q6_V_vzero(); + HVX_Vector v_dm = Q6_V_vzero(); + for (uint32_t t = start_tile; t < end_tile; t++) { - const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size); + const uint8_t * tile = state->src + t * state->aligned_tile_size; + const HVX_Vector * vptr = (const HVX_Vector *) tile; __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; - HVX_Vector v_sc = vptr[2]; + // rows hold all k tiles and n_k_tiles is a multiple of 8, so t % 8 is the tile within its super-block + if (t == start_tile || (t & 7) == 0) { + q2_k_load_dmin(tile - (t & 7) * state->aligned_tile_size, &v_d, &v_dm); + } + + HVX_Vector v_sc, v_m; + q2_k_unpack_scales(tile, v_d, v_dm, &v_sc, &v_m); HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64); - HVX_Vector v_m = vptr[3]; HVX_Vector v_m_k16 = Q6_V_vror_VR(v_m, 64); HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2)); @@ -715,6 +883,40 @@ static void dequantize_tiled_weight_to_fp16_task_q2_k( } } +// Q2_0 stores codes 0..3 (value code - 1) and one fp16 scale per row, see HTP_MM_WEIGHT_TILE_SIZE_Q2_0. +// One packed 64-k tile expands into two consecutive 32-k fp16 tiles. +static void dequantize_tiled_weight_to_fp16_task_q2_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector i1 = Q6_Vb_vsplat_R(1); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size); + __fp16 * dst_base = state->dst + 2 * t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = vptr[4]; + HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + for (int half = 0; half < 2; half++) { + __fp16 * dst_ptr = dst_base + half * HTP_MM_HMX_TILE_N_ELMS; + + #pragma unroll + for (int g = 0; g < 8; g++) { + HVX_Vector v_q = Q6_Vb_vsub_VbVb(unpack_q3_k_low2(vptr + 2 * half, g, mask_03), i1); + HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + } + } + } +} + static __attribute__((noinline)) void convert_f16_weight_to_fp16_tiles_task( const tiled_dequantize_state_t *state, @@ -745,7 +947,7 @@ void convert_f16_weight_to_fp16_tiles_task( const uint8_t *r0 = state->src + row0 * state->row_stride; const uint8_t *r1 = state->src + row1 * state->row_stride; - HVX_Vector v0 = hvx_vmemu((const __fp16 *)(r0 + byte_off)); + HVX_Vector v0 = (row0 < state->n_cols) ? hvx_vmemu((const __fp16 *)(r0 + byte_off)) : Q6_V_vzero(); HVX_Vector v1 = (row1 < state->n_cols) ? hvx_vmemu((const __fp16 *)(r1 + byte_off)) : Q6_V_vzero(); Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v0); @@ -788,7 +990,7 @@ void quantize_f32_weight_to_fp16_tiles_task( const uint8_t *r0 = state->src + row0 * state->row_stride; const uint8_t *r1 = state->src + row1 * state->row_stride; - HVX_Vector v0_f32 = hvx_vmem((const float *)(r0 + byte_off)); + HVX_Vector v0_f32 = (row0 < state->n_cols) ? hvx_vmem((const float *)(r0 + byte_off)) : Q6_V_vzero(); HVX_Vector v1_f32 = (row1 < state->n_cols) ? hvx_vmem((const float *)(r1 + byte_off)) : Q6_V_vzero(); HVX_Vector v_out = hvx_vec_f32_to_f16(v0_f32, v1_f32); @@ -805,6 +1007,56 @@ void quantize_f32_weight_to_fp16_tiles_task( } } +static __attribute__((noinline)) +void convert_bf16_weight_to_fp16_tiles_task( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const uint32_t n_k_tiles = state->n_k_tiles; + const struct fastdiv_values n_k_tiles_div = state->n_k_tiles_div; + + const HVX_Vector v_scat_base = hvx_vmem(hmx_transpose_scatter_offsets); + const HVX_Vector v_scat_step = Q6_V_vsplat_R(4); + const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64); + + unsigned ct = fastdiv((unsigned)start_tile, &n_k_tiles_div); + unsigned kt = fastmodulo((unsigned)start_tile, n_k_tiles, &n_k_tiles_div); + + for (unsigned t = start_tile; t < (unsigned)end_tile; ) { + if (kt >= (unsigned)n_k_tiles) { kt = 0; ct++; } + + __fp16 *tile_base = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + { + uint32_t byte_off = kt * 32 * sizeof(uint16_t); + + HVX_Vector v_off = v_scat_base; + for (uint32_t r = 0; r < HTP_MM_HMX_TILE_N_ROWS; r += 2) { + uint32_t row0 = ct * HTP_MM_HMX_TILE_N_COLS + r; + uint32_t row1 = row0 + 1; + + const uint8_t *r0 = state->src + row0 * state->row_stride; + const uint8_t *r1 = state->src + row1 * state->row_stride; + + HVX_Vector v0_raw = (row0 < state->n_cols) ? hvx_vmemu((const void *)(r0 + byte_off)) : Q6_V_vzero(); + HVX_Vector v1_raw = (row1 < state->n_cols) ? hvx_vmemu((const void *)(r1 + byte_off)) : Q6_V_vzero(); + + HVX_Vector v0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(Q6_Ww_vunpack_Vh(v0_raw)), 16); + HVX_Vector v1_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(Q6_Ww_vunpack_Vh(v1_raw)), 16); + + HVX_Vector v_out = hvx_vec_f32_to_f16(v0_f32, v1_f32); + + Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out); + v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); + + HVX_Vector v_out_hi = Q6_V_vror_VR(v_out, 64); + Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out_hi); + v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); + } + } + ++t; ++kt; + } +} + // --- End tiled dequantizers --- // dot-chunk functions require external HMX lock @@ -988,9 +1240,7 @@ static void transfer_output_chunk_fp16_to_fp32_col_chunk( uint32_t src2_stride, uint32_t dst_cols ) { - assert(c_len % HTP_MM_HMX_TILE_N_COLS == 0); - assert(total_n_cols % HTP_MM_HMX_TILE_N_COLS == 0); - const size_t tile_row_stride = (total_n_cols / HTP_MM_HMX_TILE_N_COLS) * HTP_MM_HMX_TILE_N_ELMS; + const size_t tile_row_stride = hmx_ceil_div(total_n_cols, HTP_MM_HMX_TILE_N_COLS) * HTP_MM_HMX_TILE_N_ELMS; const HVX_Vector one = hvx_vec_splat_f16(1.0); @@ -1083,7 +1333,6 @@ typedef struct { uint32_t dst_stride; // DDR row stride uint32_t src2_stride; // DDR row stride for residual uint32_t dst_cols; // Actual output columns - struct htp_thread_trace * traces; } output_transfer_task_state_t; // activations : fp32 -> fp16 @@ -1137,6 +1386,73 @@ static void transfer_activation_row_pair_fp32_to_fp16( } } +// F16-input variant of transfer_activation_row_pair_fp32_to_fp16, for F16 activation +// (src1). Same shape as the F16 Q-prep in hmx-fa-kernels.h: one 128-byte load carries 64 f16 +// = two tile columns, and Q6_W_vshuff_VVR interleaves the two rows straight into the HMX tile +// layout, so no F32 round-trip is needed. Rows are only 64-byte aligned when k_block is an odd +// multiple of the tile width, hence the unaligned load type. +static void transfer_activation_row_pair_f16_to_f16(__fp16 * restrict vtcm_dst, + const __fp16 * restrict row0, + const __fp16 * restrict row1, + uint32_t r, + uint32_t k_block, + uint32_t k_valid, + bool row0_valid, + bool row1_valid) { + uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + const uint32_t n_tile_cols = k_block / HTP_MM_HMX_TILE_N_COLS; + __fp16 * restrict tile_row = vtcm_dst + (size_t) r0 * n_tile_cols * HTP_MM_HMX_TILE_N_ELMS; + + const HVX_UVector * pv0 = row0_valid ? (const HVX_UVector *) row0 : NULL; + const HVX_UVector * pv1 = row1_valid ? (const HVX_UVector *) row1 : NULL; + + uint32_t c = 0; + for (; c + 64 <= k_valid; c += 64) { + HVX_Vector v0 = pv0 ? pv0[c / 64] : Q6_V_vzero(); + HVX_Vector v1 = pv1 ? pv1[c / 64] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + + HVX_Vector * tile0 = (HVX_Vector *) (tile_row + (size_t) c0 * HTP_MM_HMX_TILE_N_ELMS); + HVX_Vector * tile1 = (HVX_Vector *) (tile_row + (size_t) (c0 + 1) * HTP_MM_HMX_TILE_N_ELMS); + + tile0[r1 / 2] = Q6_V_lo_W(vp); + tile1[r1 / 2] = Q6_V_hi_W(vp); + } + // Tail: fewer than 64 valid columns left, plus the k_valid..k_block padding that HMX will + // still multiply, so it has to be written as zeros. + for (; c < k_block; c += 64) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + + if (c < k_valid) { + uint32_t rem = k_valid - c; // 1..63 valid f16 lanes + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem * sizeof(__fp16)); + if (pv0) { + v0 = Q6_V_vmux_QVV(mask, pv0[c / 64], Q6_V_vzero()); + } + if (pv1) { + v1 = Q6_V_vmux_QVV(mask, pv1[c / 64], Q6_V_vzero()); + } + } + + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + + HVX_Vector * tile0 = (HVX_Vector *) (tile_row + (size_t) c0 * HTP_MM_HMX_TILE_N_ELMS); + tile0[r1 / 2] = Q6_V_lo_W(vp); + + if (c0 + 1 < n_tile_cols) { + HVX_Vector * tile1 = (HVX_Vector *) (tile_row + (size_t) (c0 + 1) * HTP_MM_HMX_TILE_N_ELMS); + tile1[r1 / 2] = Q6_V_hi_W(vp); + } + } +} + static void transfer_activation_row_pair_fp32_to_fp16_col_chunk( __fp16 *restrict vtcm_dst, const float *restrict row0, // offset by c_first diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.c b/ggml/src/ggml-hexagon/htp/hmx-queue.c index c369d3dd2..e40a28bed 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.c +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.c @@ -46,9 +46,9 @@ static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { case HMX_QUEUE_WAKEUP: hmx_lock(q); break; default: hmx_lock(q); - htp_trace_event_start(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); - d->func(d->data); - htp_trace_event_stop(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); + htp_trace_event_start(q->th.trace, HTP_TRACE_EVT_HMX_COMP, ir); + d->func(&q->th, d->data); + htp_trace_event_stop(q->th.trace, HTP_TRACE_EVT_HMX_COMP, ir); break; } @@ -63,7 +63,7 @@ static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { static void hmx_queue_thread(void * arg) { hmx_queue_t q = (hmx_queue_t) arg; - FARF(HIGH, "hmx-queue-thread: started"); + FARF(HIGH, "hmx-queue-thread (tid %u): started", q->th.tid); bool killed = false; @@ -90,7 +90,7 @@ static void hmx_queue_thread(void * arg) { hmx_queue_process(q, &killed); } - FARF(HIGH, "hmx-queue-thread: stopped"); + FARF(HIGH, "hmx-queue-thread (tid %u): stopped", q->th.tid); } size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) { @@ -105,7 +105,7 @@ size_t hmx_queue_alignof(void) { return HEX_L2_LINE_SIZE; } -hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) { +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, dma_queue_t * dma, struct htp_thread_trace * trace) { capacity = hex_ceil_pow2(capacity); size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); @@ -126,7 +126,7 @@ hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uin q->stack = block; memset(q->stack, 0, stack_size); - q->trace = trace; + htp_thread_init(&q->th, dma, trace); // Match caller thread priority (same pattern as worker-pool.c). int prio = qurt_thread_get_priority(qurt_thread_get_id()); diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.h b/ggml/src/ggml-hexagon/htp/hmx-queue.h index c2b1859a2..efefb1455 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.h +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.h @@ -11,7 +11,8 @@ #include <HAP_farf.h> #include "hex-utils.h" -#include "hex-profile.h" +#include "htp-trace.h" +#include "htp-thread.h" #ifdef __cplusplus extern "C" { @@ -23,7 +24,7 @@ extern "C" { #define HMX_QUEUE_POLL_COUNT 1 #endif -typedef void (*hmx_queue_func)(void *); +typedef void (*hmx_queue_func)(const htp_thread_t * th, void *); // Dummy funcs used as signals enum hmx_queue_signal { @@ -49,10 +50,10 @@ struct hmx_queue_s { atomic_uint seqn; // incremented for all pushes, used with futex qurt_thread_t thread; + htp_thread_t th; void * stack; uint32_t hap_rctx; bool hmx_locked; - struct htp_thread_trace * trace; bool external_mem; // memory owned externally }; @@ -60,7 +61,7 @@ typedef struct hmx_queue_s * hmx_queue_t; size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size); size_t hmx_queue_alignof(void); -hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace); +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, dma_queue_t * dma, struct htp_thread_trace * trace); void hmx_queue_free(hmx_queue_t q); static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) { diff --git a/ggml/src/ggml-hexagon/htp/hmx-utils.h b/ggml/src/ggml-hexagon/htp/hmx-utils.h index ad295cb7d..1952aaa2c 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-utils.h +++ b/ggml/src/ggml-hexagon/htp/hmx-utils.h @@ -73,19 +73,20 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, for (uint32_t r = start_row; r < end_row; r += 2) { const uint32_t ct = r / HMX_FP16_TILE_N_ROWS; const uint32_t local_r = r % HMX_FP16_TILE_N_ROWS; + const bool row0_valid = r < n_cols; const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_cols; const HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4)); const HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step); __fp16 * tile_base = vtcm_dst + (size_t) ct * n_k_tiles * HMX_FP16_TILE_N_ELMS; - const uint8_t * p0 = (const uint8_t *) (vtcm_src + r * src_stride); + const uint8_t * p0 = row0_valid ? (const uint8_t *) (vtcm_src + r * src_stride) : NULL; const uint8_t * p1 = next_row_valid ? (const uint8_t *) (vtcm_src + (r + 1) * src_stride) : NULL; - assert(hex_is_aligned(p0, 128)); - assert(hex_is_aligned(p1, 128)); + assert(!p0 || hex_is_aligned(p0, 128)); + assert(!p1 || hex_is_aligned(p1, 128)); assert(c_byte_step % 128 == 0); - if (p1) { + if (p0 && p1) { for (uint32_t i = 0; i < n_c_iters; ++i) { HVX_Vector v0 = hvx_vmem(p0); p0 += c_byte_step; HVX_Vector v1 = hvx_vmem(p1); p1 += c_byte_step; @@ -96,9 +97,12 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, } else { const HVX_Vector vzero = Q6_V_vzero(); for (uint32_t i = 0; i < n_c_iters; ++i) { - HVX_Vector v0 = hvx_vmem(p0); p0 += c_byte_step; + HVX_Vector v0 = p0 ? hvx_vmem(p0) : vzero; + if (p0) p0 += c_byte_step; + HVX_Vector v1 = p1 ? hvx_vmem(p1) : vzero; + if (p1) p1 += c_byte_step; Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off0, v0); - Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off1, vzero); + Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off1, v1); tile_base += dst_step; } } @@ -113,15 +117,16 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, for (uint32_t r = start_row; r < end_row; r += 2) { const uint32_t ct = r / HMX_FP16_TILE_N_ROWS; const uint32_t local_r = r % HMX_FP16_TILE_N_ROWS; + const bool row0_valid = r < n_cols; const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_cols; const HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4)); const HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step); __fp16 * tile_base = vtcm_dst + (size_t) ct * n_k_tiles * HMX_FP16_TILE_N_ELMS; - const uint8_t * p0 = (const uint8_t *) (vtcm_src + r * src_stride); + const uint8_t * p0 = row0_valid ? (const uint8_t *) (vtcm_src + r * src_stride) : NULL; const uint8_t * p1 = next_row_valid ? (const uint8_t *) (vtcm_src + (r + 1) * src_stride) : NULL; - if (p1) { + if (p0 && p1) { for (uint32_t i = 0; i < n_c_iters; ++i) { HVX_Vector v0 = hvx_vmemu(p0); p0 += c_byte_step; HVX_Vector v1 = hvx_vmemu(p1); p1 += c_byte_step; @@ -132,9 +137,12 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, } else { const HVX_Vector vzero = Q6_V_vzero(); for (uint32_t i = 0; i < n_c_iters; ++i) { - HVX_Vector v0 = hvx_vmemu(p0); p0 += c_byte_step; + HVX_Vector v0 = p0 ? hvx_vmemu(p0) : vzero; + if (p0) p0 += c_byte_step; + HVX_Vector v1 = p1 ? hvx_vmemu(p1) : vzero; + if (p1) p1 += c_byte_step; Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off0, v0); - Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off1, vzero); + Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off1, v1); tile_base += dst_step; } } diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index f9f682be4..721fa2162 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -3,23 +3,21 @@ #include "dma-queue.h" #include "hmx-queue.h" +#include "htp-dcache.h" #include "htp-ops.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "work-queue.h" #include "hex-fastdiv.h" #include <assert.h> #include <dspqueue.h> #include <stdatomic.h> +#include <stddef.h> #include <stdint.h> #include <stdbool.h> +#include <string.h> -#ifndef HTP_MAX_NTHREADS -#define HTP_MAX_NTHREADS 10 -#endif - -#define HTP_MAX_MMAPS 64 -#define HTP_MAX_DIRTY_RANGES 64 +#define HTP_MAX_MMAPS 64 // Memory mapping struct htp_mmap { @@ -29,18 +27,13 @@ struct htp_mmap { uint32_t flags; }; -struct htp_dirty_range { - uint32_t start; - uint32_t end; -}; - // Scratchpad state struct htp_spad { const struct htp_tensor * src; // original src of the data (for reuse) uint8_t * data; // pointer to an area in vtcm uint32_t stride; // stride used inside this spad uint32_t size; // total size - uint32_t size_per_thread; // size per thread + uint32_t size_per_hvx; // size per hvx }; struct htp_mdev_group { @@ -71,14 +64,18 @@ struct htp_ops_context { // TODO convert these to an array struct htp_spad src0_spad; struct htp_spad src1_spad; - struct htp_spad src2_spad; - struct htp_spad src3_spad; struct htp_spad dst_spad; uint32_t flags; - uint32_t n_threads; - struct fastdiv_values n_threads_div; + uint32_t n_hvx; + struct fastdiv_values n_hvx_div; int status; + + // Must stay last: not cleared by htp_ops_context_reset(), ops init the caches they use + union { + dma_cache_dm dma_cache_dm[HTP_MAX_NHVX]; + dma_cache_fa dma_cache_fa[HTP_MAX_NHVX]; + } __attribute__((aligned(128))); }; // Main context for htp DSP backend @@ -87,12 +84,13 @@ struct htp_context { struct htp_mmap mmap[HTP_MAX_MMAPS]; dma_queue_t dma[HTP_MAX_NTHREADS]; - struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; - work_queue_t work_queue; + struct htp_thread_trace trace[HTP_MAX_NTHREADS]; + work_queue_t hvx_queue; hmx_queue_t hmx_queue; uint32_t n_threads; - struct fastdiv_values n_threads_div; + uint32_t n_hvx; + struct fastdiv_values n_hvx_div; int thread_id; int thread_prio; @@ -123,21 +121,41 @@ struct htp_context { size_t footprint; }; -static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { - if (n_threads == 0 || n_threads > octx->ctx->n_threads) { +static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct htp_context * ctx) { + memset(octx, 0, offsetof(struct htp_ops_context, dma_cache_dm)); + octx->ctx = ctx; + octx->n_hvx = ctx->n_hvx; + octx->n_hvx_div = ctx->n_hvx_div; +} + +static inline bool htp_ops_context_set_n_hvx_div( + struct htp_ops_context * octx, + uint32_t n_hvx, + struct fastdiv_values n_hvx_div) { + if (n_hvx == 0 || n_hvx > octx->ctx->n_hvx) { return false; } - if (n_threads != octx->n_threads) { - octx->n_threads = n_threads; - octx->n_threads_div = n_threads == octx->ctx->n_threads - ? octx->ctx->n_threads_div - : init_fastdiv_values(n_threads); - } + octx->n_hvx = n_hvx; + octx->n_hvx_div = n_hvx_div; return true; } +static inline bool htp_ops_context_set_n_hvx(struct htp_ops_context * octx, uint32_t n_hvx) { + if (n_hvx == 0 || n_hvx > octx->ctx->n_hvx) { + return false; + } + if (n_hvx == octx->n_hvx) { + return true; + } + + const struct fastdiv_values n_hvx_div = n_hvx == octx->ctx->n_hvx + ? octx->ctx->n_hvx_div + : init_fastdiv_values(n_hvx); + return htp_ops_context_set_n_hvx_div(octx, n_hvx, n_hvx_div); +} + static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) { if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) { octx->status = status; @@ -153,7 +171,7 @@ int op_unary(struct htp_ops_context * octx); int op_sum(struct htp_ops_context * octx); int op_sum_rows(struct htp_ops_context * octx); int op_argmax(struct htp_ops_context * octx); -int op_activations(struct htp_ops_context * octx); +int op_glu(struct htp_ops_context * octx); int op_softmax(struct htp_ops_context * octx); int op_add_id(struct htp_ops_context * octx); int op_rope(struct htp_ops_context * octx); @@ -166,6 +184,7 @@ int op_argsort(struct htp_ops_context * octx); int op_top_k(struct htp_ops_context * octx); int op_ssm_conv(struct htp_ops_context * octx); int op_cumsum(struct htp_ops_context * octx); +int op_top_k_mask(struct htp_ops_context * octx); int op_fill(struct htp_ops_context * octx); int op_concat(struct htp_ops_context * octx); int op_diag(struct htp_ops_context * octx); @@ -173,7 +192,14 @@ int op_solve_tri(struct htp_ops_context * octx); int op_gated_delta_net(struct htp_ops_context * octx); int op_pad(struct htp_ops_context * octx); int op_im2col(struct htp_ops_context * octx); +int op_im2col_3d(struct htp_ops_context * octx); +int op_conv_2d(struct htp_ops_context * octx); +int op_conv_2d_dw(struct htp_ops_context * octx); +int op_pool_2d(struct htp_ops_context * octx); +int op_pool_1d(struct htp_ops_context * octx); int op_allreduce(struct htp_ops_context * octx); int op_roll(struct htp_ops_context * octx); +int op_upscale(struct htp_ops_context * octx); +int op_lightning_indexer(struct htp_ops_context * octx); #endif /* HTP_CTX_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.c b/ggml/src/ggml-hexagon/htp/htp-dcache.c similarity index 83% rename from ggml/src/ggml-hexagon/htp/htp-tensor.c rename to ggml/src/ggml-hexagon/htp/htp-dcache.c index 03b0070d7..52fc8e403 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.c +++ b/ggml/src/ggml-hexagon/htp/htp-dcache.c @@ -1,4 +1,4 @@ -#include "htp-tensor.h" +#include "htp-dcache.h" #include <qurt.h> #include <qurt_memory.h> @@ -7,7 +7,7 @@ #include "hex-common.h" #include "hex-utils.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "work-queue.h" @@ -15,15 +15,14 @@ struct l2flush_range { uint32_t start; // line-aligned start address uint32_t end; // line-aligned end address uint32_t block_first; // global block index of this range's first block - uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial) + uint32_t n_blocks; // number of HTP_DCACHE_BLOCK_SIZE chunks (last may be partial) }; struct l2flush_multi_task { - struct htp_thread_trace * trace; struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES]; uint32_t n_ranges; uint32_t total_blocks; - uint32_t blocks_per_thread; + uint32_t blocks_per_hvx; }; static void flush_all_dcache(struct htp_context * ctx) { @@ -35,12 +34,12 @@ static void flush_all_dcache(struct htp_context * ctx) { memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); } -static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { +static void l2flush_multi_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct l2flush_multi_task * task = (struct l2flush_multi_task *) data; (void) n; - const uint32_t gb_first = i * task->blocks_per_thread; - uint32_t gb_last = gb_first + task->blocks_per_thread; + const uint32_t gb_first = i * task->blocks_per_hvx; + uint32_t gb_last = gb_first + task->blocks_per_hvx; if (gb_last > task->total_blocks) { gb_last = task->total_blocks; } @@ -48,7 +47,7 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &task->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); for (uint32_t r = 0; r < task->n_ranges; r++) { @@ -62,8 +61,8 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { continue; } - const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE; - uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE; + const uint32_t s = rg->start + (lo - rb_first) * HTP_DCACHE_BLOCK_SIZE; + uint32_t e = rg->start + (hi - rb_first) * HTP_DCACHE_BLOCK_SIZE; if (e > rg->end) { e = rg->end; } @@ -94,7 +93,7 @@ static void merge_dirty_ranges(struct htp_context * ctx) { } } -void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { +void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { const struct htp_tensor * pending[HTP_OP_MAX_OUTPUTS]; uint32_t n_pending = 0; @@ -163,7 +162,7 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co total_evict_size += r->end - r->start; } - if (total_evict_size > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_evict_size > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); for (uint32_t i = 0; i < n_pending; i++) { struct htp_dirty_range * r = &ctx->dirty_ranges[i]; @@ -174,9 +173,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co return; } - if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { + if (total_evict_size > HTP_DCACHE_FLUSH_WQ_THRESHOLD && ctx->n_hvx > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { struct l2flush_multi_task task; - task.trace = ctx->trace; task.n_ranges = n_evict; uint32_t block_acc = 0; @@ -188,14 +186,14 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + rg->n_blocks = (rg->end - rg->start + HTP_DCACHE_BLOCK_SIZE - 1) / HTP_DCACHE_BLOCK_SIZE; block_acc += rg->n_blocks; } - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + task.total_blocks = block_acc; + task.blocks_per_hvx = fastdiv(block_acc + ctx->n_hvx - 1, &ctx->n_hvx_div); - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + work_queue_run(ctx->hvx_queue, l2flush_multi_worker, &task, ctx->n_hvx); } else { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); @@ -270,9 +268,8 @@ static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_te } static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) { - if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + if (total_dirty >= HTP_DCACHE_FLUSH_WQ_THRESHOLD && ctx->n_hvx > 1) { struct l2flush_multi_task task; - task.trace = ctx->trace; task.n_ranges = n_ranges; uint32_t block_acc = 0; @@ -282,14 +279,14 @@ static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_ rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + rg->n_blocks = (rg->end - rg->start + HTP_DCACHE_BLOCK_SIZE - 1) / HTP_DCACHE_BLOCK_SIZE; block_acc += rg->n_blocks; } - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + task.total_blocks = block_acc; + task.blocks_per_hvx = fastdiv(block_acc + ctx->n_hvx - 1, &ctx->n_hvx_div); - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + work_queue_run(ctx->hvx_queue, l2flush_multi_worker, &task, ctx->n_hvx); } else { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); @@ -301,7 +298,7 @@ static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_ } } -void htp_flush_dirty_ranges(struct htp_context * ctx) { +void htp_dcache_flush_dirty_ranges(struct htp_context * ctx) { struct htp_dirty_range ranges[HTP_MAX_DIRTY_RANGES]; uint32_t n_ranges = 0; uint64_t total_dirty = 0; @@ -319,7 +316,7 @@ void htp_flush_dirty_ranges(struct htp_context * ctx) { return; } - if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_dirty > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); return; } @@ -328,7 +325,7 @@ void htp_flush_dirty_ranges(struct htp_context * ctx) { memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); } -void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { +void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS]; struct htp_dirty_range ranges[HTP_OP_MAX_INPUTS]; uint32_t n_dirty = 0; @@ -348,7 +345,7 @@ void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * co return; } - if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_dirty > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); return; } diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.h b/ggml/src/ggml-hexagon/htp/htp-dcache.h new file mode 100644 index 000000000..c1c37573c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-dcache.h @@ -0,0 +1,24 @@ +#ifndef HTP_DCACHE_H +#define HTP_DCACHE_H + +#include <stdint.h> +#include "hex-common.h" +#include "htp-ops.h" + +#define HTP_MAX_DIRTY_RANGES 64 +#define HTP_DCACHE_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread) +#define HTP_DCACHE_FLUSH_WQ_THRESHOLD (4 * 1024) +#define HTP_DCACHE_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) + +struct htp_dirty_range { + uint32_t start; + uint32_t end; +}; + +struct htp_context; + +void htp_dcache_flush_dirty_ranges(struct htp_context * ctx); +void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); +void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); + +#endif // HTP_DCACHE_H diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index a6a3bb85a..c28b8d61c 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -21,6 +21,8 @@ enum htp_data_type { HTP_TYPE_F16 = 1, HTP_TYPE_Q4_0 = 2, HTP_TYPE_Q4_1 = 3, + HTP_TYPE_Q5_0 = 6, + HTP_TYPE_Q5_1 = 7, HTP_TYPE_Q8_0 = 8, HTP_TYPE_Q2_K = 10, HTP_TYPE_Q3_K = 11, @@ -30,24 +32,14 @@ enum htp_data_type { HTP_TYPE_IQ4_NL = 20, HTP_TYPE_I32 = 26, HTP_TYPE_I64 = 27, + HTP_TYPE_BF16 = 30, HTP_TYPE_MXFP4 = 39, - - // types used internally for repack, dyn.quant, etc - HTP_TYPE_Q4_0_TILED = 200, - HTP_TYPE_Q4_1_TILED, - HTP_TYPE_Q8_0_TILED, - HTP_TYPE_MXFP4_TILED, + HTP_TYPE_Q1_0 = 41, + HTP_TYPE_Q2_0 = 42, HTP_TYPE_INVALID }; -// Constats for internal types -#define QK_Q4_0_TILED 256 // 32x32 Q4_0 tiled layout -#define QK_Q8_0_TILED 128 // 32x32 Q8_0 tiled layout -#define QK_MXFP4_TILED 256 // 32x32 MXFP4 tiled layout - - - // Do not reorder first 4 (used as an index) enum htp_op_code { HTP_OP_MUL = 0, @@ -61,6 +53,7 @@ enum htp_op_code { HTP_OP_MUL_MAT_ADD, HTP_OP_RMS_NORM, HTP_OP_RMS_NORM_MUL, + HTP_OP_RMS_NORM_MUL_ADD, HTP_OP_UNARY_SILU, HTP_OP_UNARY_GELU, HTP_OP_UNARY_SIGMOID, @@ -106,6 +99,8 @@ enum htp_op_code { HTP_OP_CLAMP, HTP_OP_LEAKY_RELU, HTP_OP_IM2COL, + HTP_OP_CONV_2D, + HTP_OP_CONV_2D_DW, HTP_OP_FENCE, HTP_OP_ALLREDUCE, HTP_OP_ALLREDUCE_ADD, @@ -115,18 +110,36 @@ enum htp_op_code { HTP_OP_ARGMAX, HTP_OP_UNARY_GELU_ERF, HTP_OP_GLU_GEGLU_ERF, + HTP_OP_POOL_2D, + HTP_OP_POOL_1D, + HTP_OP_UNARY_GELU_QUICK, + HTP_OP_IM2COL_3D, + HTP_OP_HADAMARD, + HTP_OP_GLU_REGLU, + HTP_OP_GLU_SIGMOID, + HTP_OP_UNARY_CEIL, + HTP_OP_UNARY_SGN, + HTP_OP_UPSCALE, + HTP_OP_CONV_3D, + HTP_OP_DSV4_HC_PRE, + HTP_OP_DSV4_HC_POST, + HTP_OP_TOP_K_MASK, + HTP_OP_LIGHTNING_INDEXER, HTP_OP_INVALID }; -#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS -#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS -#define HTP_OP_MAX_OUTPUTS 4 -#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS +#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS +#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS +#define HTP_OP_MAX_OUTPUTS 4 +#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS #define HTP_OP_MAX_KERN_PARAMS 32 -#define HTP_OP_MAX_BUFS 16 -#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16) +#define HTP_OP_MAX_BUFS 64 +#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16) + +#define HTP_OP_MAX_NTHREADS 10 +#define HTP_OP_MAX_NHVX 8 #define HTP_FENCE_TIMEOUT (1000000000ULL) #define HTP_FENCE_SLOT_SIZE 128 @@ -135,10 +148,13 @@ enum htp_op_code { #define HTP_MMAP_MAX_VMEM (2147483648u) +#define HTP_DDR_SPAD_SIZE (512 * 1024) + enum htp_tensor_flags { - HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute) - HTP_TENSOR_REPACK = (1U << 1), // Tensor is in repacked tiled format - HTP_TENSOR_FENCE = (1U << 2) // Tensor is synchronization fence (explicitly managed) + HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute) + HTP_TENSOR_TILED = (1U << 1), // Tensor is in tiled format + HTP_TENSOR_FENCE = (1U << 2), // Tensor is synchronization fence (explicitly managed) + HTP_TENSOR_FUSEABLE = (1U << 3), // Tensor can be fused (host-only) }; enum htp_buf_flags { @@ -180,8 +196,20 @@ struct htp_op_desc { uint16_t pad[2]; // padding to align to 64 bits }; -#ifndef HTP_MAX_NTHREADS +#ifdef __HVX_ARCH__ +#if __HVX_ARCH__ <= 73 +#define HTP_MAX_NTHREADS 6 +#define HTP_MAX_NHVX 4 +#elif __HVX_ARCH__ <= 75 +#define HTP_MAX_NTHREADS 8 +#define HTP_MAX_NHVX 6 +#else #define HTP_MAX_NTHREADS 10 +#define HTP_MAX_NHVX 8 +#endif + +_Static_assert(HTP_MAX_NTHREADS <= HTP_OP_MAX_NTHREADS, "HTP_MAX_NTHREADS exceeds HTP_OP_MAX_NTHREADS"); +_Static_assert(HTP_MAX_NHVX <= HTP_OP_MAX_NHVX, "HTP_MAX_NHVX exceeds HTP_OP_MAX_NHVX"); #endif #define HTP_TRACE_MAX_EVENTS 256 @@ -193,6 +221,20 @@ enum htp_profiler_mode { HTP_PROF_TRACE = 3, }; +// DCVS/HMX power profile requested for the session. +enum htp_power_mode { + HTP_POWER_MODE_LOW_POWER_SAVER = 0, + HTP_POWER_MODE_POWER_SAVER = 1, + HTP_POWER_MODE_HIGH_POWER_SAVER = 2, + HTP_POWER_MODE_LOW_BALANCED = 3, + HTP_POWER_MODE_BALANCED = 4, + HTP_POWER_MODE_HIGH_PERFORMANCE = 5, + HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE = 6, + HTP_POWER_MODE_BURST = 7, + + HTP_POWER_MODE_COUNT +}; + enum htp_trace_event_id { HTP_TRACE_EVT_DMA = 0, HTP_TRACE_EVT_L2FLUSH = 1, @@ -260,7 +302,7 @@ struct htp_opbatch_rsp { uint32_t n_tensors; // Number of tensors uint32_t n_ops; // Number of op profile descriptors uint32_t usecs; // Number of usec - uint32_t n_traces[HTP_MAX_NTHREADS + 1]; + uint32_t n_traces[HTP_OP_MAX_NTHREADS]; // struct htp_prof_desc profs[]; -- dspqueue buf 0 }; diff --git a/ggml/src/ggml-hexagon/htp/htp-swiv.py b/ggml/src/ggml-hexagon/htp/htp-swiv.py new file mode 100644 index 000000000..15d7bef27 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-swiv.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import os +import shutil +import struct +import sys +import tempfile +import zlib + +ELF_MAGIC = b'\x7fELF' +ELF_CLASS_INDEX = 4 +ELFCLASS32 = 1 +ELFCLASS64 = 2 +ELF_VERSION_INDEX = 6 +EV_CURRENT = 1 +MAX_PHDR_COUNT = 100 + +LOAD_TYPE = 0x1 + +SHT_NOBITS = 8 + +SWIV_MAGIC = b'SWIV' +SWIV_SEG_SIZE_MIN = 16 +# SHT_LOUSER plus ASCII "SWIV". +SWIV_SECTION_TYPE = 0xD3574956 + +logger = logging.getLogger("htp-swiv") + + +class Elf_Ehdr_common: + s = struct.Struct('16sHHI') + + def __init__(self, data): + unpacked_data = (Elf_Ehdr_common.s).unpack(data) + self.e_ident = unpacked_data[0] + self.e_type = unpacked_data[1] + self.e_machine = unpacked_data[2] + self.e_version = unpacked_data[3] + + +class Elf32_Ehdr: + s = struct.Struct('16sHHIIIIIHHHHHH') + + def __init__(self, data): + unpacked_data = (Elf32_Ehdr.s).unpack(data) + self.e_ident = unpacked_data[0] + self.e_type = unpacked_data[1] + self.e_machine = unpacked_data[2] + self.e_version = unpacked_data[3] + self.e_entry = unpacked_data[4] + self.e_phoff = unpacked_data[5] + self.e_shoff = unpacked_data[6] + self.e_flags = unpacked_data[7] + self.e_ehsize = unpacked_data[8] + self.e_phentsize = unpacked_data[9] + self.e_phnum = unpacked_data[10] + self.e_shentsize = unpacked_data[11] + self.e_shnum = unpacked_data[12] + self.e_shstrndx = unpacked_data[13] + + def get_packed_data(self): + values = [self.e_ident, + self.e_type, + self.e_machine, + self.e_version, + self.e_entry, + self.e_phoff, + self.e_shoff, + self.e_flags, + self.e_ehsize, + self.e_phentsize, + self.e_phnum, + self.e_shentsize, + self.e_shnum, + self.e_shstrndx + ] + + return (Elf32_Ehdr.s).pack(*values) + + +class Elf32_Phdr: + s = struct.Struct('I' * 8) + + def __init__(self, data): + unpacked_data = (Elf32_Phdr.s).unpack(data) + self.p_type = unpacked_data[0] + self.p_offset = unpacked_data[1] + self.p_vaddr = unpacked_data[2] + self.p_paddr = unpacked_data[3] + self.p_filesz = unpacked_data[4] + self.p_memsz = unpacked_data[5] + self.p_flags = unpacked_data[6] + self.p_align = unpacked_data[7] + + def get_packed_data(self): + values = [self.p_type, + self.p_offset, + self.p_vaddr, + self.p_paddr, + self.p_filesz, + self.p_memsz, + self.p_flags, + self.p_align + ] + + return (Elf32_Phdr.s).pack(*values) + + +class Elf32_Shdr: + s = struct.Struct('I' * 10) + + def __init__(self, data): + unpacked_data = (Elf32_Shdr.s).unpack(data) + self.sh_name = unpacked_data[0] + self.sh_type = unpacked_data[1] + self.sh_flags = unpacked_data[2] + self.sh_addr = unpacked_data[3] + self.sh_offset = unpacked_data[4] + self.sh_size = unpacked_data[5] + self.sh_link = unpacked_data[6] + self.sh_info = unpacked_data[7] + self.sh_addralign = unpacked_data[8] + self.sh_entsize = unpacked_data[9] + self.sh_name_str = "" + + def get_packed_data(self): + values = [self.sh_name, + self.sh_type, + self.sh_flags, + self.sh_addr, + self.sh_offset, + self.sh_size, + self.sh_link, + self.sh_info, + self.sh_addralign, + self.sh_entsize] + + return (Elf32_Shdr.s).pack(*values) + + +class Elf64_Ehdr: + s = struct.Struct('16sHHIQQQIHHHHHH') + + def __init__(self, data): + unpacked_data = (Elf64_Ehdr.s).unpack(data) + self.e_ident = unpacked_data[0] + self.e_type = unpacked_data[1] + self.e_machine = unpacked_data[2] + self.e_version = unpacked_data[3] + self.e_entry = unpacked_data[4] + self.e_phoff = unpacked_data[5] + self.e_shoff = unpacked_data[6] + self.e_flags = unpacked_data[7] + self.e_ehsize = unpacked_data[8] + self.e_phentsize = unpacked_data[9] + self.e_phnum = unpacked_data[10] + self.e_shentsize = unpacked_data[11] + self.e_shnum = unpacked_data[12] + self.e_shstrndx = unpacked_data[13] + + def get_packed_data(self): + values = [self.e_ident, + self.e_type, + self.e_machine, + self.e_version, + self.e_entry, + self.e_phoff, + self.e_shoff, + self.e_flags, + self.e_ehsize, + self.e_phentsize, + self.e_phnum, + self.e_shentsize, + self.e_shnum, + self.e_shstrndx + ] + + return (Elf64_Ehdr.s).pack(*values) + + +class Elf64_Phdr: + s = struct.Struct('IIQQQQQQ') + + def __init__(self, data): + unpacked_data = (Elf64_Phdr.s).unpack(data) + self.p_type = unpacked_data[0] + self.p_flags = unpacked_data[1] + self.p_offset = unpacked_data[2] + self.p_vaddr = unpacked_data[3] + self.p_paddr = unpacked_data[4] + self.p_filesz = unpacked_data[5] + self.p_memsz = unpacked_data[6] + self.p_align = unpacked_data[7] + + def get_packed_data(self): + values = [self.p_type, + self.p_flags, + self.p_offset, + self.p_vaddr, + self.p_paddr, + self.p_filesz, + self.p_memsz, + self.p_align + ] + + return (Elf64_Phdr.s).pack(*values) + + +class Elf64_Shdr: + s = struct.Struct('IIQQQQIIQQ') + + def __init__(self, data): + unpacked_data = (Elf64_Shdr.s).unpack(data) + self.sh_name = unpacked_data[0] + self.sh_type = unpacked_data[1] + self.sh_flags = unpacked_data[2] + self.sh_addr = unpacked_data[3] + self.sh_offset = unpacked_data[4] + self.sh_size = unpacked_data[5] + self.sh_link = unpacked_data[6] + self.sh_info = unpacked_data[7] + self.sh_addralign = unpacked_data[8] + self.sh_entsize = unpacked_data[9] + self.sh_name_str = "" + + def get_packed_data(self): + values = [self.sh_name, + self.sh_type, + self.sh_flags, + self.sh_addr, + self.sh_offset, + self.sh_size, + self.sh_link, + self.sh_info, + self.sh_addralign, + self.sh_entsize] + + return (Elf64_Shdr.s).pack(*values) + + +class SWIV_Seg: + s = struct.Struct('4sIQ') + + def __init__(self): + self.magic = SWIV_MAGIC + self.crc = 0 + self.padding = 0 + + def get_packed_data(self): + values = [self.magic, + self.crc, + self.padding + ] + + return (SWIV_Seg.s).pack(*values) + + +def is_elf64(elf_header): + return elf_header.e_ident[ELF_CLASS_INDEX] == ELFCLASS64 + + +def verify_elf_header(elf_header): + ident = elf_header.e_ident + return (ident.startswith(ELF_MAGIC) + and ident[ELF_CLASS_INDEX] in (ELFCLASS32, ELFCLASS64) + and ident[ELF_VERSION_INDEX] == EV_CURRENT) + + +def file_copy_offset(in_fp, in_off, out_fp, out_off, num_bytes): + in_fp.seek(in_off) + read_in = in_fp.read(num_bytes) + out_fp.seek(out_off) + out_fp.write(read_in) + + return num_bytes + + +def preprocess_elf_file(elf_file_name): + with open(elf_file_name, 'rb') as elf_fp: + elf_header = Elf_Ehdr_common(elf_fp.read(Elf_Ehdr_common.s.size)) + + if verify_elf_header(elf_header) is False: + raise RuntimeError("Input ELF verification failed : " + elf_file_name) + + elf_fp.seek(0) + + if is_elf64(elf_header): + elf_header = Elf64_Ehdr(elf_fp.read(Elf64_Ehdr.s.size)) + else: + elf_header = Elf32_Ehdr(elf_fp.read(Elf32_Ehdr.s.size)) + + phdr_table = [] + + phdr_size = elf_header.e_phentsize + + file_offset = elf_header.e_phoff + elf_fp.seek(file_offset) + + for _ in range(elf_header.e_phnum): + if is_elf64(elf_header): + phdr_table.append(Elf64_Phdr(elf_fp.read(phdr_size))) + else: + phdr_table.append(Elf32_Phdr(elf_fp.read(phdr_size))) + + return [elf_header, phdr_table] + + +def process_section_header(elf_file_name): + with open(elf_file_name, 'rb') as elf_fp: + elf_header = Elf_Ehdr_common(elf_fp.read(Elf_Ehdr_common.s.size)) + + if verify_elf_header(elf_header) is False: + raise RuntimeError("Input ELF verification failed : " + elf_file_name) + + elf_fp.seek(0) + + if is_elf64(elf_header): + elf_header = Elf64_Ehdr(elf_fp.read(Elf64_Ehdr.s.size)) + else: + elf_header = Elf32_Ehdr(elf_fp.read(Elf32_Ehdr.s.size)) + + shdr_table = [] + + shdr_size = elf_header.e_shentsize + elf_fp.seek(elf_header.e_shoff) + + for _ in range(elf_header.e_shnum): + if is_elf64(elf_header): + shdr_table.append(Elf64_Shdr(elf_fp.read(shdr_size))) + else: + shdr_table.append(Elf32_Shdr(elf_fp.read(shdr_size))) + + return shdr_table + + +def crc32_update_file(elf_fp, offset, size, crc): + elf_fp.seek(offset) + remaining = size + while remaining > 0: + chunk = elf_fp.read(min(1024 * 1024, remaining)) + if not chunk: + raise RuntimeError("Unexpected EOF while calculating CRC") + crc = zlib.crc32(chunk, crc) + remaining -= len(chunk) + return crc + + +def crc32_update_zeros(size, crc): + zero_chunk = b'\x00' * min(1024 * 1024, size) + remaining = size + while remaining > 0: + chunk = zero_chunk[:min(len(zero_chunk), remaining)] + crc = zlib.crc32(chunk, crc) + remaining -= len(chunk) + return crc + + +def crc32_calculate(elf_fp, phdr_table): + crc = 0 + + # SWIV validates loadable segment contents, including BSS as zeros. + for curr_phdr in phdr_table: + if curr_phdr.p_type != LOAD_TYPE: + continue + + crc = crc32_update_file(elf_fp, curr_phdr.p_offset, curr_phdr.p_filesz, crc) + + zero_size = curr_phdr.p_memsz - curr_phdr.p_filesz + if zero_size > 0: + crc = crc32_update_zeros(zero_size, crc) + + crc &= 0xffffffff + logger.info("calculated CRC: %#x", crc) + + return crc + + +def verify_swiv_seg(elf_in_fp, offset): + + elf_in_fp.seek(offset) + swiv_seg = SWIV_Seg() + unpacked_data = (SWIV_Seg.s).unpack(elf_in_fp.read(SWIV_SEG_SIZE_MIN)) + swiv_seg.magic = unpacked_data[0] + + return swiv_seg.magic == SWIV_MAGIC + + +def same_output_file(input_elf, output_elf): + try: + return os.path.exists(output_elf) and os.path.samefile(input_elf, output_elf) + except FileNotFoundError: + return False + + +def make_output_target(input_elf, output_elf): + if not same_output_file(input_elf, output_elf): + return output_elf, None + + output_dir = os.path.dirname(os.path.abspath(output_elf)) + output_base = os.path.basename(output_elf) + fd, temp_output = tempfile.mkstemp(prefix=output_base + ".", suffix=".tmp", dir=output_dir) + os.close(fd) + shutil.copymode(input_elf, temp_output) + return temp_output, temp_output + + +def swiv_generate(input_elf, output_elf): + output_target, temp_output = make_output_target(input_elf, output_elf) + + try: + with open(input_elf, "rb") as elf_in_fp, open(output_target, "wb+") as elf_out_fp: + [elf_header, phdr_table] = preprocess_elf_file(input_elf) + + num_phdrs = elf_header.e_phnum + num_shdrs = elf_header.e_shnum + phdr_size = elf_header.e_phentsize + + if num_phdrs == 0 or num_phdrs > MAX_PHDR_COUNT: + raise RuntimeError("Input ELF has invalid number of program headers.") + + if elf_header.e_shoff == 0 or num_shdrs == 0: + raise RuntimeError("Input ELF does not contain a section header table.") + + shdr_table_input = process_section_header(input_elf) + shdr_table_output = process_section_header(input_elf) + + last_phdr = phdr_table[num_phdrs - 1] + if last_phdr.p_memsz == SWIV_SEG_SIZE_MIN and verify_swiv_seg(elf_in_fp, last_phdr.p_offset): + raise RuntimeError("A SWIV segment is detected in the input ELF.") + + last_shdr = shdr_table_input[elf_header.e_shnum - 1] + if last_shdr.sh_size == SWIV_SEG_SIZE_MIN and verify_swiv_seg(elf_in_fp, last_shdr.sh_offset): + raise RuntimeError("A SWIV section is detected in the input ELF.") + + if is_elf64(elf_header): + swiv_shdr = Elf64_Shdr(b'\0' * Elf64_Shdr.s.size) + else: + swiv_shdr = Elf32_Shdr(b'\0' * Elf32_Shdr.s.size) + + swiv_shdr.sh_name = 0 + swiv_shdr.sh_type = SWIV_SECTION_TYPE + swiv_shdr.sh_flags = 0 + swiv_shdr.sh_addr = 0 + swiv_shdr.sh_size = SWIV_SEG_SIZE_MIN + swiv_shdr.sh_offset = elf_header.e_shoff + (elf_header.e_shnum + 1) * elf_header.e_shentsize + shdr_table_output.append(swiv_shdr) + elf_header.e_shnum += 1 + + if elf_header.e_phoff != 0: + for phdr in phdr_table: + file_copy_offset(elf_in_fp, phdr.p_offset, elf_out_fp, phdr.p_offset, phdr.p_filesz) + + elf_out_fp.seek(0) + elf_out_fp.write(elf_header.get_packed_data()) + + phdr_start = elf_header.e_phoff + for phdr in phdr_table: + elf_out_fp.seek(phdr_start) + elf_out_fp.write(phdr.get_packed_data()) + phdr_start += phdr_size + + for shdr in shdr_table_input[:num_shdrs]: + if shdr.sh_type == SHT_NOBITS or shdr.sh_size == 0: + continue + file_copy_offset(elf_in_fp, shdr.sh_offset, elf_out_fp, shdr.sh_offset, shdr.sh_size) + + shdr_start = elf_header.e_shoff + for shdr in shdr_table_output: + elf_out_fp.seek(shdr_start) + elf_out_fp.write(shdr.get_packed_data()) + shdr_start += elf_header.e_shentsize + + elf_out_fp.flush() + + swiv_seg = SWIV_Seg() + swiv_seg.crc = crc32_calculate(elf_out_fp, phdr_table) + elf_out_fp.seek(swiv_shdr.sh_offset) + elf_out_fp.write(swiv_seg.get_packed_data()) + + if temp_output is not None: + os.replace(temp_output, output_elf) + temp_output = None + + return 0 + finally: + if temp_output is not None and os.path.exists(temp_output): + os.unlink(temp_output) + + +def main(argv): + parser = argparse.ArgumentParser(description="Add a Software Image Validation (SWIV) section to an HTP ELF file.") + parser.add_argument("-i", "--input", dest="input_file", metavar="INPUT", required=True) + parser.add_argument("-o", "--output", dest="output_file", metavar="OUTPUT", required=True) + parser.add_argument("-v", "--verbose", action="store_true", help="show calculated CRC") + args = parser.parse_args(argv) + + logging.basicConfig( + level=logging.INFO if args.verbose else logging.WARNING, + format="%(message)s", + ) + + try: + return swiv_generate(args.input_file, args.output_file) + except RuntimeError as exc: + logger.error("ERROR: %s", exc) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h index f7a966836..beacc7e4d 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.h +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -8,27 +8,42 @@ #include "hex-common.h" #include "hex-fastdiv.h" -enum { - HTP_TENSOR_MDEV_LINE_SIZE = 128, -}; +#define HTP_TENSOR_MDEV_LINE_SIZE 128 + +// Currently HMX requires 32x32 tile formats (do not change) +#define HTP_TENSOR_TILE_N_COLS 32 +#define HTP_TENSOR_TILE_N_ROWS 32 + +#define HTP_TENSOR_TILE_SIZE_F16 2048 +#define HTP_TENSOR_TILE_SIZE_Q4_0 576 +#define HTP_TENSOR_TILE_SIZE_Q4_1 640 +#define HTP_TENSOR_TILE_SIZE_Q5_0 704 +#define HTP_TENSOR_TILE_SIZE_Q5_1 768 +#define HTP_TENSOR_TILE_SIZE_Q8_0 1088 +#define HTP_TENSOR_TILE_SIZE_IQ4_NL 576 +#define HTP_TENSOR_TILE_SIZE_MXFP4 544 +#define HTP_TENSOR_TILE_SIZE_Q4_K 640 +#define HTP_TENSOR_TILE_SIZE_Q5_K 768 +#define HTP_TENSOR_TILE_SIZE_Q6_K 896 +#define HTP_TENSOR_TILE_SIZE_Q3_K 512 +#define HTP_TENSOR_TILE_SIZE_Q2_K 336 +#define HTP_TENSOR_TILE_K_Q1_0 128 +#define HTP_TENSOR_TILE_SIZE_Q1_0 576 +#define HTP_TENSOR_TILE_K_Q2_0 64 +#define HTP_TENSOR_TILE_SIZE_Q2_0 576 + +// Tiled weight tensor layouts (Q4_0, Q5_0, Q5_1, Q8_0, IQ4_NL, MXFP4, Q1_0, Q2_0, K-quants): +// See docs/backend/snapdragon/developer.md ("Tiled Weight Formats"). struct htp_tensor_mdev_range { uint32_t start; uint32_t count; }; -static inline void * htp_tensor_data(const struct htp_tensor * t) { - return (void *) (uintptr_t) t->data; -} - static inline bool htp_tensor_is_extended(const struct htp_tensor * t) { return t && (t->data >> 32) != 0; } -static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) { - return (uint32_t *) &t->flags; -} - static inline bool htp_tensor_is_contiguous(const struct htp_tensor * t, uint32_t type_size) { uint32_t next_nb = type_size; if (t->ne[0] != 1 && t->nb[0] != next_nb) { @@ -125,19 +140,66 @@ static inline struct htp_tensor_mdev_range htp_tensor_mdev_partition(uint32_t to return range; } -static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { +static inline uint32_t htp_tensor_get_tile_size(int type) { + switch (type) { + case HTP_TYPE_Q4_0: return HTP_TENSOR_TILE_SIZE_Q4_0; + case HTP_TYPE_IQ4_NL: return HTP_TENSOR_TILE_SIZE_IQ4_NL; + case HTP_TYPE_Q4_1: return HTP_TENSOR_TILE_SIZE_Q4_1; + case HTP_TYPE_Q5_0: return HTP_TENSOR_TILE_SIZE_Q5_0; + case HTP_TYPE_Q5_1: return HTP_TENSOR_TILE_SIZE_Q5_1; + case HTP_TYPE_Q4_K: return HTP_TENSOR_TILE_SIZE_Q4_K; + case HTP_TYPE_Q8_0: return HTP_TENSOR_TILE_SIZE_Q8_0; + case HTP_TYPE_Q5_K: return HTP_TENSOR_TILE_SIZE_Q5_K; + case HTP_TYPE_Q6_K: return HTP_TENSOR_TILE_SIZE_Q6_K; + case HTP_TYPE_Q3_K: return HTP_TENSOR_TILE_SIZE_Q3_K; + case HTP_TYPE_Q2_K: return HTP_TENSOR_TILE_SIZE_Q2_K; + case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_SIZE_Q1_0; + case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_SIZE_Q2_0; + case HTP_TYPE_MXFP4: return HTP_TENSOR_TILE_SIZE_MXFP4; + case HTP_TYPE_F16: return HTP_TENSOR_TILE_SIZE_F16; + default: return 0; + } +} + +static inline uint32_t htp_tensor_get_tile_k(int type) { switch (type) { - case HTP_TYPE_F32: return ne00 * 4; - case HTP_TYPE_F16: return ne00 * 2; - case HTP_TYPE_Q8_0: return (ne00 / 32) * 34; - case HTP_TYPE_I32: return ne00 * 4; - default: return 0; + case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_K_Q1_0; + case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_K_Q2_0; + default: return 32; } } -struct htp_context; -void htp_flush_dirty_ranges(struct htp_context * ctx); -void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); -void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); +static inline uint32_t htp_tensor_get_tile_padded_size(int type) { + return (htp_tensor_get_tile_size(type) + 127u) & ~127u; +} + +static inline bool htp_tensor_is_tiled(const struct htp_tensor * t) { + return t && (t->flags & HTP_TENSOR_TILED) != 0; +} + +static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { + switch (type) { + case HTP_TYPE_F32: return ne00 * 4; + case HTP_TYPE_F16: + case HTP_TYPE_BF16: return ne00 * 2; + case HTP_TYPE_Q8_0: return (ne00 / 32) * 34; + case HTP_TYPE_Q4_0: + case HTP_TYPE_IQ4_NL: return (ne00 / 32) * 18; + case HTP_TYPE_Q4_1: return (ne00 / 32) * 20; + case HTP_TYPE_Q5_0: return (ne00 / 32) * 22; + case HTP_TYPE_Q5_1: return (ne00 / 32) * 24; + case HTP_TYPE_MXFP4: return (ne00 / 32) * 17; + case HTP_TYPE_Q2_K: return (ne00 / 256) * 84; + case HTP_TYPE_Q2_0: return (ne00 / 64) * 18; + case HTP_TYPE_Q3_K: return (ne00 / 256) * 110; + case HTP_TYPE_Q4_K: return (ne00 / 256) * 144; + case HTP_TYPE_Q5_K: return (ne00 / 256) * 176; + case HTP_TYPE_Q6_K: return (ne00 / 256) * 210; + case HTP_TYPE_Q1_0: return (ne00 / HTP_TENSOR_TILE_K_Q1_0) * 18; + case HTP_TYPE_I32: return ne00 * 4; + case HTP_TYPE_I64: return ne00 * 8; + default: return 0; + } +} #endif // HTP_TENSOR_H diff --git a/ggml/src/ggml-hexagon/htp/htp-thread.h b/ggml/src/ggml-hexagon/htp/htp-thread.h new file mode 100644 index 000000000..e4ca874de --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-thread.h @@ -0,0 +1,33 @@ +#ifndef HTP_THREAD_H +#define HTP_THREAD_H + +#include <stdatomic.h> +#include <stdint.h> + +#include "dma-queue.h" +#include "htp-trace.h" + +typedef struct { + unsigned int tid; + dma_queue * dma_q; + struct htp_thread_trace * trace; +} htp_thread_t; + +extern atomic_uint htp_thread_count; + +static inline void htp_thread_reset(void) { + atomic_store_explicit(&htp_thread_count, 0, memory_order_relaxed); +} + +static inline unsigned int htp_thread_alloc_id(void) { + return atomic_fetch_add_explicit(&htp_thread_count, 1, memory_order_relaxed); +} + +static inline void htp_thread_init(htp_thread_t * th, dma_queue_t * dma, struct htp_thread_trace * trace) { + unsigned int tid = htp_thread_alloc_id(); + th->tid = tid; + th->dma_q = dma ? dma[tid] : NULL; + th->trace = trace ? &trace[tid] : NULL; +} + +#endif /* HTP_THREAD_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-trace.h b/ggml/src/ggml-hexagon/htp/htp-trace.h new file mode 100644 index 000000000..66bd9110d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-trace.h @@ -0,0 +1,38 @@ +#ifndef HTP_TRACE_H +#define HTP_TRACE_H + +#include <stdint.h> + +#include "hex-utils.h" +#include "htp-ops.h" + +#define HTP_TRACE_EVT_START 0 +#define HTP_TRACE_EVT_STOP 1 + +struct htp_thread_trace { + uint32_t count; + uint32_t max_events; + struct htp_trace_desc * events; +}; + +static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { + if (!tr->max_events) { + return; + } + uint32_t i = tr->count++; + if (i < tr->max_events) { + tr->events[i].id = id; + tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); + tr->events[i].cycles = (uint32_t) hex_get_cycles(); + } +} + +static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { + htp_trace_event(tr, id, info, HTP_TRACE_EVT_START); +} + +static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { + htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP); +} + +#endif /* HTP_TRACE_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp_iface.idl b/ggml/src/ggml-hexagon/htp/htp_iface.idl index b46e25296..6e3390ba1 100644 --- a/ggml/src/ggml-hexagon/htp/htp_iface.idl +++ b/ggml/src/ggml-hexagon/htp/htp_iface.idl @@ -11,7 +11,7 @@ struct htp_iface_pmu_conf { }; interface htp_iface : remote_handle64 { - AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem); + AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 vtcm_size, in uint64 max_vmem, in uint32 power_mode); AEEResult stop(); AEEResult mmap(in uint32 fd, in uint64 size); AEEResult munmap(in uint32 fd); diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index ab2064af3..ec0de8e19 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -7,6 +7,7 @@ #include <math.h> #include "hvx-base.h" +#include "hvx-floor.h" #include "hex-utils.h" // @@ -676,6 +677,56 @@ static inline void hvx_step_f32_aa(uint8_t * restrict dst, const uint8_t * restr } } +static inline void hvx_sgn_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_sgn_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_sgn_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +static inline void hvx_sgn_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(_Float16); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + vdst[i] = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p))); + } + if (nloe) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p))); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + static inline void hvx_step_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src % 128 == 0); @@ -700,6 +751,60 @@ static inline void hvx_step_f16_aa(uint8_t * restrict dst, const uint8_t * restr } } +// +// Ceil +// + +static inline void hvx_ceil_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_ceil_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_ceil_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +static inline void hvx_ceil_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(_Float16); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + vdst[i] = hvx_vec_f32_to_f16(hvx_vec_ceil_f32(Q6_V_lo_W(p)), hvx_vec_ceil_f32(Q6_V_hi_W(p))); + } + if (nloe) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_ceil_f32(Q6_V_lo_W(p)), hvx_vec_ceil_f32(Q6_V_hi_W(p))); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + // // Square // diff --git a/ggml/src/ggml-hexagon/htp/hvx-base.h b/ggml/src/ggml-hexagon/htp/hvx-base.h index 20e0d6e66..e87a30d14 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-base.h +++ b/ggml/src/ggml-hexagon/htp/hvx-base.h @@ -118,6 +118,17 @@ static inline HVX_Vector hvx_vec_step_f32(HVX_Vector v) { return Q6_V_vmux_QVV(q, one, zero); } +static inline HVX_Vector hvx_vec_sgn_f32(HVX_Vector v) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + const HVX_Vector sign_mask = Q6_V_vsplat_R(0x80000000); + const HVX_Vector mag = hvx_vec_abs_f32(v); + const HVX_Vector signed_one = Q6_V_vor_VV(one, Q6_V_vand_VV(v, sign_mask)); + // The ordered magnitude comparison maps NaN and both signed zeros to zero. + const HVX_VectorPred nonzero = Q6_Q_vcmp_gt_VsfVsf(mag, zero); + return Q6_V_vmux_QVV(nonzero, signed_one, zero); +} + static inline HVX_Vector hvx_vec_step_f16(HVX_Vector v) { const HVX_Vector zero = Q6_V_vzero(); const HVX_Vector one = hvx_vec_splat_f16((_Float16) 1.0f); diff --git a/ggml/src/ggml-hexagon/htp/hvx-copy.h b/ggml/src/ggml-hexagon/htp/hvx-copy.h index a3e33c3b3..08eb7cd61 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-copy.h +++ b/ggml/src/ggml-hexagon/htp/hvx-copy.h @@ -44,7 +44,7 @@ static inline void hvx_splat_f32_u(void * restrict dst, float v, uint32_t n) { } static inline void hvx_splat_f16_a(void * restrict dst, _Float16 v, uint32_t n) { - hvx_splat_u(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16)); + hvx_splat_a(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16)); } static inline void hvx_splat_f16_u(void * restrict dst, _Float16 v, uint32_t n) { @@ -106,42 +106,42 @@ static inline void hvx_copy_uu(uint8_t * restrict dst, const uint8_t * restrict hvx_copy_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); } -// copy n fp16 elements : source and destination are aligned to HVX Vector (128) +// copy n fp16 elements : destination and source are aligned to HVX Vector (128) static inline void hvx_copy_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_aa(dst, src, n, sizeof(__fp16)); } -// copy n fp16 elements : source is aligned, destination is potentially unaligned +// copy n fp16 elements : destination is aligned, source is unaligned static inline void hvx_copy_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_au(dst, src, n, sizeof(__fp16)); } -// copy n fp16 elements : source is aligned, destination is potentially unaligned +// copy n fp16 elements : destination is unaligned, source is aligned static inline void hvx_copy_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_ua(dst, src, n, sizeof(__fp16)); } -// copy n fp16 elements : source is aligned, destination is potentially unaligned +// copy n fp16 elements : destination and source are unaligned static inline void hvx_copy_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_uu(dst, src, n, sizeof(__fp16)); } -// copy n fp32 elements : source and destination are aligned to HVX Vector (128) +// copy n fp32 elements : destination and source are aligned to HVX Vector (128) static inline void hvx_copy_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_aa(dst, src, n, sizeof(float)); } -// copy n fp32 elements : source is aligned, destination is unaligned -static inline void hvx_copy_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { - hvx_copy_ua(dst, src, n, sizeof(float)); -} - -// copy n fp32 elements : source is unaligned, destination is aligned +// copy n fp32 elements : destination is aligned, source is unaligned static inline void hvx_copy_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_au(dst, src, n, sizeof(float)); } -// copy n fp32 elements : source is unaligned, destination unaligned +// copy n fp32 elements : destination is unaligned, source is aligned +static inline void hvx_copy_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_copy_ua(dst, src, n, sizeof(float)); +} + +// copy n fp32 elements : destination and source are unaligned static inline void hvx_copy_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_uu(dst, src, n, sizeof(float)); } @@ -170,26 +170,26 @@ static inline void hvx_copy_f32_uu(uint8_t * restrict dst, const uint8_t * restr } \ } while(0) -// copy/convert n fp32 elements into n fp16 elements : source is aligned, destination is aligned +// copy/convert n fp32 elements into n fp16 elements : destination and source are aligned static inline void hvx_copy_f16_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src % 128 == 0); hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); } -// copy/convert n fp32 elements into n fp16 elements : source is unaligned, destination is aligned +// copy/convert n fp32 elements into n fp16 elements : destination is aligned, source is unaligned static inline void hvx_copy_f16_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); } -// copy/convert n fp32 elements into n fp16 elements : source is aligned, destination is unaligned +// copy/convert n fp32 elements into n fp16 elements : destination is unaligned, source is aligned static inline void hvx_copy_f16_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) src % 128 == 0); hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); } -// copy/convert n fp32 elements into n fp16 elements : source is unaligned, destination is unaligned +// copy/convert n fp32 elements into n fp16 elements : destination and source are unaligned static inline void hvx_copy_f16_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); } @@ -235,28 +235,98 @@ static inline void hvx_copy_f16_f32_uu(uint8_t * restrict dst, const uint8_t * r } \ } while(0) -// copy/convert n fp16 elements into n fp32 elements : source is aligned, destination is aligned +// copy/convert n fp16 elements into n fp32 elements : destination and source are aligned static inline void hvx_copy_f32_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src % 128 == 0); hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); } -// copy/convert n fp16 elements into n fp32 elements : source is unaligned, destination is aligned +// copy/convert n fp16 elements into n fp32 elements : destination is aligned, source is unaligned static inline void hvx_copy_f32_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); } -// copy/convert n fp16 elements into n fp32 elements : source is aligned, destination is unaligned +// copy/convert n fp16 elements into n fp32 elements : destination is unaligned, source is aligned static inline void hvx_copy_f32_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) src % 128 == 0); hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); } -// copy/convert n fp16 elements into n fp32 elements : source is unaligned, destination is unaligned +// copy/convert n fp16 elements into n fp32 elements : destination and source are unaligned static inline void hvx_copy_f32_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); } +//// fp32 -> int32 + +#define hvx_copy_i32_f32_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(int32_t); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = Q6_Vw_equals_Vsf(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = Q6_Vw_equals_Vsf(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +// copy/convert n fp32 elements into n int32 elements : destination and source are aligned +static inline void hvx_copy_i32_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_copy_i32_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +// copy/convert n fp32 elements into n int32 elements : destination and source are unaligned +static inline void hvx_copy_i32_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_copy_i32_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +//// int32 -> fp32 + +#define hvx_copy_f32_i32_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(float); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = Q6_Vsf_equals_Vw(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = Q6_Vsf_equals_Vw(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +// copy/convert n int32 elements into n fp32 elements : destination and source are aligned +static inline void hvx_copy_f32_i32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_copy_f32_i32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +// copy/convert n int32 elements into n fp32 elements : destination and source are unaligned +static inline void hvx_copy_f32_i32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_copy_f32_i32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + #endif // HVX_COPY_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-erf.h b/ggml/src/ggml-hexagon/htp/hvx-erf.h index 6eefc300f..b17c657a8 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-erf.h +++ b/ggml/src/ggml-hexagon/htp/hvx-erf.h @@ -12,7 +12,7 @@ static __attribute__((noinline)) HVX_Vector hvx_vec_erf_f32(HVX_Vector x) { HVX_Vector t = hvx_vec_inverse_f32(hvx_vec_add_f32_f32( hvx_vec_splat_f32(1.0f), hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), ax))); - HVX_Vector poly = hvx_vec_mul_f32_f32(hvx_vec_splat_f32(1.061405429f), t); + HVX_Vector poly = hvx_vec_splat_f32(1.061405429f); poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(-1.453152027f), hvx_vec_mul_f32_f32(poly, t)); poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.421413741f), hvx_vec_mul_f32_f32(poly, t)); poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(-0.284496736f), hvx_vec_mul_f32_f32(poly, t)); @@ -27,6 +27,7 @@ static __attribute__((noinline)) HVX_Vector hvx_vec_erf_f32(HVX_Vector x) { return result; } +// GELU_ERF uses the erf definition. static inline HVX_Vector hvx_vec_gelu_erf_f32(HVX_Vector x) { const HVX_Vector scale = hvx_vec_splat_f32(0.7071067811865475f); const HVX_Vector half = hvx_vec_splat_f32(0.5f); diff --git a/ggml/src/ggml-hexagon/htp/hvx-floor.h b/ggml/src/ggml-hexagon/htp/hvx-floor.h index 6a1bfde56..6757d6184 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-floor.h +++ b/ggml/src/ggml-hexagon/htp/hvx-floor.h @@ -97,4 +97,12 @@ static inline HVX_Vector hvx_vec_floor_f32(HVX_Vector in_vec) { return vout; } +// ceil(x) = -floor(-x); keep +-0 as is since floor(-0) returns -1 +static inline HVX_Vector hvx_vec_ceil_f32(HVX_Vector in_vec) { + HVX_Vector vout = hvx_vec_neg_f32(hvx_vec_floor_f32(hvx_vec_neg_f32(in_vec))); + HVX_Vector vabs = Q6_V_vand_VV(in_vec, Q6_V_vsplat_R(0x7fffffff)); + HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(vabs, Q6_V_vzero()); + return Q6_V_vmux_QVV(q_zero, in_vec, vout); +} + #endif /* HVX_FLOOR_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h index 605892aa7..d7488976b 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h @@ -76,7 +76,7 @@ static inline void quantize_f16_f16_kernel( #define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) #endif -static inline void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { +static inline void flat_vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { const HVX_Vector * restrict x = (const HVX_Vector *) vx; const HVX_Vector * restrict y = (const HVX_Vector *) vy; @@ -104,7 +104,7 @@ static inline void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); } -static inline void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, +static inline void flat_vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, const void * restrict vx0, const void * restrict vx1, const void * restrict vy0) { const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; @@ -143,7 +143,7 @@ static inline void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, hvx_vec_store_u(s0, 8, rsum); } -static inline void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, +static inline void flat_vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx0, const void * restrict vx1, const void * restrict vy0, const void * restrict vy1) { const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; @@ -196,10 +196,203 @@ static inline void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, hvx_vec_store_u(s1, 8, r0_r1_c1_sum); } +static inline void flat_vec_dot_bf16_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / 64; + uint32_t rem = n % 64; + + HVX_Vector rsum0 = Q6_V_vzero(); + HVX_Vector rsum1 = Q6_V_vzero(); + + uint32_t i = 0; + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector v_bf16 = x[i]; + HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16); + HVX_Vector w0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16); + HVX_Vector w1_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p), 16); + + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y[2 * i + 0])); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w1_f32, y[2 * i + 1])); + } + + if (rem) { + HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2); + HVX_Vector v_bf16 = Q6_V_vand_QV(bmask, x[i]); + HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16); + HVX_Vector w0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16); + HVX_Vector w1_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p), 16); + + if (rem <= 32) { + HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4); + HVX_Vector y0 = Q6_V_vand_QV(a_mask, y[2 * i + 0]); + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y0)); + } else { + HVX_Vector y0 = y[2 * i + 0]; + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y0)); + + HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4); + HVX_Vector y1 = Q6_V_vand_QV(a_mask, y[2 * i + 1]); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w1_f32, y1)); + } + } + + HVX_Vector rsum = HVX_OP_ADD_F32(rsum0, rsum1); + *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void flat_vec_dot_bf16_f32_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / 64; + uint32_t rem = n % 64; + + HVX_Vector rsum0 = Q6_V_vzero(); + HVX_Vector rsum1 = Q6_V_vzero(); + + uint32_t i = 0; + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(x0[i]); + HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(x1[i]); + HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16); + HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16); + HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16); + HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16); + + HVX_Vector y0 = y[2 * i + 0]; + HVX_Vector y1 = y[2 * i + 1]; + + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0)); + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w01_f32, y1)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w11_f32, y1)); + } + + if (rem) { + HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2); + HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x0[i])); + HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x1[i])); + HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16); + HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16); + HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16); + HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16); + + if (rem <= 32) { + HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4); + HVX_Vector y0 = Q6_V_vand_QV(a_mask, y[2 * i + 0]); + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0)); + } else { + HVX_Vector y0 = y[2 * i + 0]; + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0)); + + HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4); + HVX_Vector y1 = Q6_V_vand_QV(a_mask, y[2 * i + 1]); + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w01_f32, y1)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w11_f32, y1)); + } + } + + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void flat_vec_dot_bf16_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / 64; + uint32_t rem = n % 64; + + HVX_Vector r0_c0_sum = Q6_V_vzero(); + HVX_Vector r0_c1_sum = Q6_V_vzero(); + HVX_Vector r1_c0_sum = Q6_V_vzero(); + HVX_Vector r1_c1_sum = Q6_V_vzero(); + + uint32_t i = 0; + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(x0[i]); + HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(x1[i]); + HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16); + HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16); + HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16); + HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16); + + HVX_Vector c0_0 = y0[2 * i + 0]; + HVX_Vector c0_1 = y0[2 * i + 1]; + HVX_Vector c1_0 = y1[2 * i + 0]; + HVX_Vector c1_1 = y1[2 * i + 1]; + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0)); + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w01_f32, c0_1)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w01_f32, c1_1)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w11_f32, c0_1)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w11_f32, c1_1)); + } + + if (rem) { + HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2); + HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x0[i])); + HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x1[i])); + HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16); + HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16); + HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16); + HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16); + + if (rem <= 32) { + HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4); + HVX_Vector c0_0 = Q6_V_vand_QV(a_mask, y0[2 * i + 0]); + HVX_Vector c1_0 = Q6_V_vand_QV(a_mask, y1[2 * i + 0]); + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0)); + } else { + HVX_Vector c0_0 = y0[2 * i + 0]; + HVX_Vector c1_0 = y1[2 * i + 0]; + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0)); + + HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4); + HVX_Vector c0_1 = Q6_V_vand_QV(a_mask, y0[2 * i + 1]); + HVX_Vector c1_1 = Q6_V_vand_QV(a_mask, y1[2 * i + 1]); + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w01_f32, c0_1)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w01_f32, c1_1)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w11_f32, c0_1)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w11_f32, c1_1)); + } + } + + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(s0, 8, r0_r1_c0_sum); + hvx_vec_store_u(s1, 8, r0_r1_c1_sum); +} + #undef HVX_OP_ADD_F32 #undef HVX_OP_MUL_F32 -static inline void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { +static inline void flat_vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { const HVX_Vector * restrict x = (const HVX_Vector *) vx; const HVX_Vector * restrict y = (const HVX_Vector *) vy; @@ -226,7 +419,7 @@ static inline void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum)); } -static inline void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, +static inline void flat_vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, const void * restrict vx0, const void * restrict vx1, const void * restrict vy0) { const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; @@ -263,7 +456,7 @@ static inline void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, hvx_vec_store_u(s0, 8, rsum); } -static inline void vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, +static inline void flat_vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx0, const void * restrict vx1, const void * restrict vy0, const void * restrict vy1) { const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h index 580bae4ac..4a6b2b5ea 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -1,45 +1,55 @@ // Dynamic quantizers that produce tiled activations -// vector 9: d * sum(q) of the 32 k, replicated (q8_1), or per 16 k for Q2_K (q8_1_s16): k 0..15 in lanes 0..31, -// k 16..31 in lanes 32..63, with d the fp16 scale of vector 8 +// q8_1: vector 9 holds d * sum(q) of the 32 k, replicated. +// q8_1_s16 (Q2_K, sums16): one fp16 d (vector 8) for the 256 k super-block starting at x_sb, and vector 9 holds the +// integer sums of k 0..15 (low halfword) and k 16..31 (high halfword) in every word __attribute__((always_inline)) -static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16) { +static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16, + const float * restrict x_sb) { assert((unsigned long) x % 128 == 0); assert((unsigned long) y_block % 128 == 0); HVX_Vector * vx = (HVX_Vector *) x; HVX_Vector zero = Q6_V_vzero(); - HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); - HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); - HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); - HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); + HVX_Vector vmax0_sf, vmax1_sf, vmax2_sf, vmax3_sf; + if (sums16) { + const HVX_Vector * vsb = (const HVX_Vector *) x_sb; + HVX_Vector vm = hvx_vec_abs_f32(vsb[0]); + for (int i = 1; i < 8; i++) { + vm = Q6_Vsf_vmax_VsfVsf(vm, hvx_vec_abs_f32(vsb[i])); + } + vmax0_sf = vmax1_sf = vmax2_sf = vmax3_sf = hvx_vec_reduce_max_f32(vm); + } else { + vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); + vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); + vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); + vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); + } HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); - HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); - HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); - HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); - HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); - HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); - - HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); - HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); - HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); - HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); + HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); + HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); + HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); + vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); - HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); - HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); + HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); + HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); + HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); + HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); - HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); + HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); + HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); - vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); - vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); + vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); @@ -61,15 +71,6 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ HVX_Vector vd3_sf = hvx_vec_mul_f32_f32(vmax3_sf, v_inv127); HVX_Vector v_sums_sf = Q6_Vsf_equals_Vw(v_sums); - if (sums16) { - // the fp16 d of vector 8 - HVX_VectorPair vd01_sf = hvx_vec_f16_to_f32(vd01_hf); - HVX_VectorPair vd23_sf = hvx_vec_f16_to_f32(vd23_hf); - vd0_sf = Q6_V_lo_W(vd01_sf); - vd1_sf = Q6_V_hi_W(vd01_sf); - vd2_sf = Q6_V_lo_W(vd23_sf); - vd3_sf = Q6_V_hi_W(vd23_sf); - } HVX_Vector voff0_sf = hvx_vec_mul_f32_f32(vd0_sf, v_sums_sf); HVX_Vector voff1_sf = hvx_vec_mul_f32_f32(vd1_sf, Q6_V_vror_VR(v_sums_sf, 32)); HVX_Vector voff2_sf = hvx_vec_mul_f32_f32(vd2_sf, Q6_V_vror_VR(v_sums_sf, 64)); @@ -78,25 +79,10 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ HVX_Vector voff01_hf = hvx_vec_f32_to_f16(voff0_sf, voff1_sf); HVX_Vector voff23_hf = hvx_vec_f32_to_f16(voff2_sf, voff3_sf); - HVX_Vector r_scale[4] = { - hvx_vec_repl_f16(vd01_hf), - hvx_vec_repl_f16(Q6_V_vror_VR(vd01_hf, 64)), - hvx_vec_repl_f16(vd23_hf), - hvx_vec_repl_f16(Q6_V_vror_VR(vd23_hf, 64)), - }; - HVX_Vector r_offset[4] = { - hvx_vec_repl_f16(voff01_hf), - hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 64)), - hvx_vec_repl_f16(voff23_hf), - hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)), - }; + int32_t __attribute__((aligned(128))) ws[32]; if (sums16) { - // the k 16..31 sums sit 4 halfwords after the k 0..15 sums - const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64); - r_offset[0] = Q6_V_vmux_QVV(q_lo, r_offset[0], hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 8))); - r_offset[1] = Q6_V_vmux_QVV(q_lo, r_offset[1], hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 72))); - r_offset[2] = Q6_V_vmux_QVV(q_lo, r_offset[2], hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 8))); - r_offset[3] = Q6_V_vmux_QVV(q_lo, r_offset[3], hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 72))); + // |sum of 16 int8| <= 2032 fits a halfword + *(HVX_Vector *) ws = v_sums; } static const uint8_t __attribute__((aligned(128))) repl[128] = { @@ -111,8 +97,15 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ }; HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; + #pragma unroll for (int b = 0; b < 4; b++) { HVX_Vector v_act = Q6_V_vror_VR(vx_i8, b * 32); + HVX_Vector vd_b = (b < 2) ? vd01_hf : vd23_hf; + HVX_Vector voff_b = (b < 2) ? voff01_hf : voff23_hf; + HVX_Vector r_scale = hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(vd_b, 64) : vd_b); + HVX_Vector r_offset = sums16 + ? Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF)) + : hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(voff_b, 64) : voff_b); HVX_Vector r0 = Q6_V_vdelta_VV(v_act, v_repl_ctrl); HVX_Vector r1 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 4), v_repl_ctrl); @@ -132,17 +125,17 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ dst[5] = r5; dst[6] = r6; dst[7] = r7; - dst[8] = r_scale[b]; - dst[9] = r_offset[b]; + dst[8] = r_scale; + dst[9] = r_offset; } } static inline void quantize_block_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y_block) { - quantize_block_f32_q8_1_tiled_impl(x, y_block, false); + quantize_block_f32_q8_1_tiled_impl(x, y_block, false, NULL); } -static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block) { - quantize_block_f32_q8_1_tiled_impl(x, y_block, true); +static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block, const float * restrict x_sb) { + quantize_block_f32_q8_1_tiled_impl(x, y_block, true, x_sb); } static inline void quantize_block_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y_block) { @@ -240,7 +233,7 @@ static inline void quantize_block_f32_q8_0_tiled(float * restrict x, uint8_t * r static void quantize_row_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) { assert(k % 32 == 0); - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (k + qk - 1) / qk; for (uint32_t i = 0; i < nb; i++) { @@ -251,7 +244,7 @@ static void quantize_row_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y static void quantize_row_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) { assert(k % 32 == 0); - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (k + qk - 1) / qk; for (uint32_t i = 0; i < nb; i++) { @@ -262,12 +255,78 @@ static void quantize_row_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y static void quantize_row_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) { assert(k % 32 == 0); - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (k + qk - 1) / qk; for (uint32_t i = 0; i < nb; i++) { uint8_t * restrict y_block = y + i * 4 * 1280; - quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block); + quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block, x + (i & ~1u) * qk); // k % 256 == 0 for Q2_K + } +} + +static inline void quantize_half_f16_tiled_splat(HVX_Vector v_src, HVX_Vector * restrict dst) { + static const uint8_t __attribute__((aligned(128))) repl[128] = { + 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + }; + HVX_Vector v_repl_ctrl = *(const HVX_Vector *) repl; + + #pragma unroll(4) + for (int cp = 0; cp < 32; cp++) { + dst[cp] = Q6_V_vdelta_VV(cp == 0 ? v_src : Q6_V_vror_VR(v_src, cp * 4), v_repl_ctrl); + } +} + +static inline void quantize_block_f32_f16_tiled(const float * restrict x, uint8_t * restrict y_block) { + assert((unsigned long) x % 128 == 0); + assert((unsigned long) y_block % 128 == 0); + + const HVX_Vector * vx = (const HVX_Vector *) x; + HVX_Vector * restrict dst = (HVX_Vector *) y_block; + + HVX_Vector vx01_hf = hvx_vec_f32_to_f16(vx[0], vx[1]); + quantize_half_f16_tiled_splat(vx01_hf, dst); + + HVX_Vector vx23_hf = hvx_vec_f32_to_f16(vx[2], vx[3]); + quantize_half_f16_tiled_splat(vx23_hf, dst + 32); +} + +static inline void quantize_block_f16_f16_tiled(const __fp16 * restrict x, uint8_t * restrict y_block) { + assert((unsigned long) x % 128 == 0); + assert((unsigned long) y_block % 128 == 0); + + const HVX_Vector * vx = (const HVX_Vector *) x; + HVX_Vector * restrict dst = (HVX_Vector *) y_block; + + quantize_half_f16_tiled_splat(vx[0], dst); + quantize_half_f16_tiled_splat(vx[1], dst + 32); +} + +static void quantize_row_f32_f16_tiled(const float * restrict x, uint8_t * restrict y, uint32_t k) { + assert(k % 32 == 0); + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (k + qk - 1) / qk; + + for (uint32_t i = 0; i < nb; i++) { + uint8_t * restrict y_block = y + i * 4 * 2048; + quantize_block_f32_f16_tiled(x + i * qk, y_block); + } +} + +static void quantize_row_f16_f16_tiled(const __fp16 * restrict x, uint8_t * restrict y, uint32_t k) { + assert(k % 32 == 0); + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (k + qk - 1) / qk; + + for (uint32_t i = 0; i < nb; i++) { + uint8_t * restrict y_block = y + i * 4 * 2048; + quantize_block_f16_f16_tiled(x + i * qk, y_block); } } @@ -326,6 +385,36 @@ static inline HVX_VectorPair unpack_and_interleave_4bit_x2(HVX_Vector v_src, HVX return Q6_W_vcombine_VV(v_W1, v_W0); } +static inline HVX_Vector unpack_q1_0_group( + HVX_Vector v_plane, + int g, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << g); + return Q6_V_vmux_QVV(q, v_pos_one, v_neg_one); +} + +static inline HVX_Vector accum_q1_0_32x1( + HVX_Vector v_plane, + const HVX_Vector * restrict v_act, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one); + v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi); +} + static inline HVX_Vector accum_4bit_32x1( const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act, @@ -368,6 +457,33 @@ static inline HVX_Vector accum_4bit_32x1_lut( return Q6_Vw_vadd_VwVw(v_sum0, v_sum1); } +static inline HVX_VectorPair accum_q1_0_32x2( + HVX_Vector v_plane, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + return Q6_W_vcombine_VV( + Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi), + Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi)); +} + static inline HVX_VectorPair accum_4bit_32x2( const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act0, @@ -451,7 +567,7 @@ static inline HVX_VectorPair accum_q8_0_32x2( return Q6_W_vcombine_VV(v_sum1, v_sum0); } -// Q5_K: OR 0x10 into every lane of v whose flag j is set in the plane (see HTP_MM_WEIGHT_TILE_SIZE_Q5_K) +// Q5_K: OR 0x10 into every lane of v whose flag j is set in the plane (see HTP_TENSOR_TILE_SIZE_Q5_K) static inline HVX_Vector hvx_q5k_or_hibit(HVX_Vector v, HVX_Vector v_plane, int j) { HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << j); return Q6_V_vandor_VQR(v, q, 0x10101010); @@ -523,7 +639,75 @@ static inline HVX_VectorPair accum_5bit_32x2( return Q6_W_vcombine_VV(v_sum1, v_sum0); } -// Q6_K weights are stored unsigned (0..63), see HTP_MM_WEIGHT_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32) +// OR 0xF0 into every lane of v whose flag j is set in the plane, i.e. 4-bit value | 0xF0 = q - 16 where the 5th bit is clear (Q5_0) +static inline HVX_Vector hvx_q5_0_or_neg(HVX_Vector v, HVX_Vector v_plane, int j) { + HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << j); + return Q6_V_vandor_VQR(v, q, 0xF0F0F0F0); +} + +// Q5_0: signed bytes (q - 16) straight from the nibbles and the plane +static inline HVX_VectorPair unpack_and_interleave_q5_0_x2(HVX_Vector v_src, HVX_Vector v_plane, int i, HVX_Vector mask_h4) { + HVX_Vector v_lo = hvx_q5_0_or_neg(Q6_V_vand_VV(v_src, mask_h4), v_plane, 2 * i); + HVX_Vector v_hi = hvx_q5_0_or_neg(Q6_Vub_vlsr_VubR(v_src, 4), v_plane, 2 * i + 1); + HVX_VectorPair v01_pair = Q6_W_vshuff_VVR(v_hi, v_lo, -1); + HVX_Vector v01_lo = Q6_V_lo_W(v01_pair); + HVX_Vector v01_hi = Q6_V_hi_W(v01_pair); + + HVX_Vector v23_lo = Q6_V_valign_VVR(v01_hi, v01_lo, 64); + HVX_Vector v_W0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v23_lo, v01_lo, -2)); + + HVX_Vector v67_lo = Q6_V_valign_VVR(v01_lo, v01_hi, 64); + HVX_Vector v_W1 = Q6_V_lo_W(Q6_W_vshuff_VVR(v67_lo, v01_hi, -2)); + + return Q6_W_vcombine_VV(v_W1, v_W0); +} + +static inline HVX_Vector accum_q5_0_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector v_plane = vptr[4]; + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_q5_0_x2(vptr[i], v_plane, i, mask_h4); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, Q6_V_lo_W(v_W_pair), v_act[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, Q6_V_hi_W(v_W_pair), v_act[i * 2 + 1]); + } + + return Q6_Vw_vadd_VwVw(v_sum0, v_sum1); +} + +static inline HVX_VectorPair accum_q5_0_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1 +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector v_plane = vptr[4]; + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_q5_0_x2(vptr[i], v_plane, i, mask_h4); + HVX_Vector v_W0 = Q6_V_lo_W(v_W_pair); + HVX_Vector v_W1 = Q6_V_hi_W(v_W_pair); + + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W0, v_act0[i * 2 + 0]); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W1, v_act0[i * 2 + 1]); + + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W0, v_act1[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W1, v_act1[i * 2 + 1]); + } + + return Q6_W_vcombine_VV(v_sum1, v_sum0); +} + +// Q6_K weights are stored unsigned (0..63), see HTP_TENSOR_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32) static inline HVX_Vector unpack_q6_k_group(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_0f, HVX_Vector mask_03, HVX_Vector i32) { HVX_Vector v_lo = (g & 1) ? Q6_Vub_vlsr_VubR(vptr[g >> 1], 4) : Q6_V_vand_VV(vptr[g >> 1], mask_0f); HVX_Vector v_hi = (g & 3) ? Q6_Vub_vlsr_VubR(vptr[4 + (g >> 2)], 2 * (g & 3)) : vptr[4 + (g >> 2)]; @@ -590,7 +774,7 @@ static inline HVX_Vector scale_q6_k_32x1(HVX_VectorPair v_sums, HVX_Vector v_sca return hvx_vec_add_f32_f32(v_lo, v_hi); } -// Q3_K / Q2_K: low 2 bits of k-group g, see HTP_MM_WEIGHT_TILE_SIZE_Q3_K +// Q3_K / Q2_K: low 2 bits of k-group g, see HTP_TENSOR_TILE_SIZE_Q3_K static inline HVX_Vector unpack_q3_k_low2(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_03) { HVX_Vector v = vptr[g >> 2]; if ((g & 3) == 3) { @@ -655,7 +839,7 @@ static inline void accum_q3_k_32x2( *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); } -// Q2_K (x = D * q + M): the Q3_K dot products without the -4 flags, M uses the q8_1_s16 sums +// Q2_K: the Q3_K dot products without the -4 flags, scales and mins are applied in q2_k_int_accum static inline HVX_VectorPair accum_q2_k_32x1( const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act @@ -702,11 +886,187 @@ static inline void accum_q2_k_32x2( *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); } -// D as the Q6_K scales, plus M (vector 3) times the per-16 activation sums in v_act[9] -static inline HVX_Vector scale_q2_k_32x1(HVX_VectorPair v_sums, const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act) { - HVX_VectorPair v_m = hvx_vec_mul_f16_f16_to_f32_pair(vptr[3], v_act[9]); - HVX_Vector v_d = scale_q6_k_32x1(v_sums, vptr[2], v_act[8]); - return hvx_vec_add_f32_f32(v_d, hvx_vec_add_f32_f32(Q6_V_lo_W(v_m), Q6_V_hi_W(v_m))); +// Q2_0: unsigned codes 0..3; the -1 offset is applied via the Q8_1 block sum (v_act[9]) +static inline HVX_Vector accum_q2_0_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03); + v_sum_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi); +} + +static inline void accum_q2_0_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector * v_sum0, + HVX_Vector * v_sum1 +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + *v_sum0 = Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi); + *v_sum1 = Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi); +} + +// Q2_K: fp16 d (*v_d) and -dmin (*v_m) of the 32 rows of a super-block (8 tiles, VTCM stride 384), each in both vector halves +static inline void q2_k_load_dmin(const uint8_t * restrict sb, HVX_Vector * restrict v_d, HVX_Vector * restrict v_m) { + HVX_Vector v_dd = Q6_V_vzero(); // lanes 0..31: d, lanes 32..63: -dmin + + #pragma unroll + for (int j = 0; j < 8; j++) { + HVX_Vector v = hvx_vmem(sb + j * 384 + 256); // the slice is at bytes 64..79 + v_dd = Q6_V_vmux_QVV(Q6_Q_vsetq_R(16 * j), v_dd, Q6_V_vror_VR(v, (64 - 16 * j) & 127)); + } + + HVX_VectorPred q_lo = Q6_Q_vsetq_R(64); + HVX_Vector v_r = Q6_V_vror_VR(v_dd, 64); + *v_d = Q6_V_vmux_QVV(q_lo, v_dd, v_r); + *v_m = Q6_V_vmux_QVV(q_lo, v_r, v_dd); +} + +// Q2_K: fp16 D = d * scale and M = -dmin * min of a tile from its scale bytes, in the lanes of the Q6_K scales +static inline void q2_k_unpack_scales(const uint8_t * restrict tile, HVX_Vector v_d, HVX_Vector v_m, + HVX_Vector * restrict v_D, HVX_Vector * restrict v_M) { + HVX_Vector v_sc = hvx_vmem(tile + 256); + HVX_Vector v_l = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_V_vand_VV(v_sc, Q6_Vb_vsplat_R(0x0F)))); + HVX_Vector v_mn = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_Vub_vlsr_VubR(v_sc, 4))); + // qf32 product: rounds like GGML_FP32_TO_FP16(d * scale) (qf16 is off by 1 ulp for many values on v73) + *v_D = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_l), v_d)); + *v_M = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_mn), v_m)); +} + +// Q2_K scale and min nibbles of a tile as halfword pairs: word r = sub-block 1 << 16 | sub-block 0 of row r +static inline void q2_k_scale_pairs(const uint8_t * restrict tile, HVX_Vector * restrict v_sc2, HVX_Vector * restrict v_mn2) { + HVX_Vector v_h = Q6_Vh_vshuff_Vh(Q6_V_lo_W(Q6_Wuh_vunpack_Vub(hvx_vmem(tile + 256)))); + *v_sc2 = Q6_V_vand_VV(v_h, Q6_Vh_vsplat_R(0x000F)); + *v_mn2 = Q6_Vuh_vlsr_VuhR(v_h, 4); +} + +// Q2_K with q8_1_s16 activations, one tile: isum += scale * (q . a), msum += min * sum(a), per row, in int32. +// |sum of 16 q * a| <= 3 * 128 * 16 fits a halfword, so both sub-blocks go through one vdmpy (no saturation) +static inline void q2_k_int_accum(HVX_VectorPair v_sums, HVX_Vector v_sc2, HVX_Vector v_mn2, HVX_Vector v_bsum, + HVX_Vector * restrict v_isum, HVX_Vector * restrict v_msum) { + HVX_Vector v_s2 = Q6_Vh_vshuffe_VhVh(Q6_V_hi_W(v_sums), Q6_V_lo_W(v_sums)); + *v_isum = Q6_Vw_vadd_VwVw(*v_isum, Q6_Vw_vdmpy_VhVh_sat(v_s2, v_sc2)); + *v_msum = Q6_Vw_vadd_VwVw(*v_msum, Q6_Vw_vdmpy_VhVh_sat(v_mn2, v_bsum)); +} + +// Q2_K super-block result: d_a * (d * isum - dmin * msum), with v_d / v_m the fp16 d / -dmin of q2_k_load_dmin +static inline HVX_Vector q2_k_int_scale(HVX_Vector v_isum, HVX_Vector v_msum, HVX_Vector v_d, HVX_Vector v_m, HVX_Vector v_act8) { + HVX_Vector v_d32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_d)); + HVX_Vector v_m32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_m)); + HVX_Vector v_da32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_act8)); + HVX_Vector v_t = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(v_d32, Q6_Vsf_equals_Vw(v_isum)), + hvx_vec_mul_f32_f32(v_m32, Q6_Vsf_equals_Vw(v_msum))); + return hvx_vec_mul_f32_f32(v_t, v_da32); +} + +static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_qf = Q6_V_vzero(); + HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0)); + HVX_Vector v_tile_qf = Q6_V_vzero(); + + #pragma unroll + for (int quarter = 0; quarter < 4; quarter++) { + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (4 * kt + quarter) * HTP_MM_ACT_TILE_SIZE_Q8_0); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q1_0_32x1(vptr[quarter], v_act, v_neg_one, v_pos_one)); + HVX_Vector v_da_sf = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8])); + HVX_Vector v_q_qf = Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf); + v_tile_qf = (quarter == 0) ? v_q_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_q_qf); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_qf = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf)); + } + + HVX_Vector v_sum_float = Q6_Vsf_equals_Vqf32(v_sum_qf); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_qf_c0 = Q6_V_vzero(); + HVX_Vector v_sum_qf_c1 = Q6_V_vzero(); + HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0)); + HVX_Vector v_tile_qf_c0 = Q6_V_vzero(); + HVX_Vector v_tile_qf_c1 = Q6_V_vzero(); + + for (int quarter = 0; quarter < 4; quarter++) { + const uint32_t act_tile = 4 * kt + quarter; + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0); + HVX_VectorPair v_sums = accum_q1_0_32x2(vptr[quarter], v_act0, v_act1, v_neg_one, v_pos_one); + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums)); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums)); + HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8])); + HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8])); + + v_tile_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c0, v_da_sf_c0)); + v_tile_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c1, v_da_sf_c1)); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c0, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf)); + v_sum_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c1, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf)); + } + + HVX_Vector v_sum_float_c0 = Q6_Vsf_equals_Vqf32(v_sum_qf_c0); + HVX_Vector v_sum_float_c1 = Q6_Vsf_equals_Vqf32(v_sum_qf_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { @@ -959,6 +1319,82 @@ static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float } } +static void tiled_vec_dot_q5_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 768); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_Vector v_sum = accum_q5_0_32x1(vptr, v_act); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); + + HVX_Vector v_scale_w = vptr[5]; + HVX_Vector v_scale_a = v_act[8]; + HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); + HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); + + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q5_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 768); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums = accum_q5_0_32x2(vptr, v_act0, v_act1); + HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); + HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); + + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); + + HVX_Vector v_scale_w = vptr[5]; + HVX_Vector v_scale_a_c0 = v_act0[8]; + HVX_Vector v_scale_a_c1 = v_act1[8]; + + HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c0); + HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c1); + + HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); + HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + static void tiled_vec_dot_q5_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -1174,12 +1610,22 @@ static void tiled_vec_dot_q2_k_32x1(const uint32_t n, float * restrict s, const HVX_Vector v_sum_float = Q6_V_vzero(); uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512); - const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280); - - HVX_VectorPair v_sums = accum_q2_k_32x1(vptr, v_act); - v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q2_k_32x1(v_sums, vptr, v_act)); + for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) { + HVX_Vector v_d, v_m; + q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m); + + HVX_Vector v_isum = Q6_V_vzero(); + HVX_Vector v_msum = Q6_V_vzero(); + for (uint32_t kt = kb; kt < kb + 8; kt++) { + const uint8_t * restrict tile = tile_ptr + kt * 384; + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280); + + HVX_Vector v_sc2, v_mn2; + q2_k_scale_pairs(tile, &v_sc2, &v_mn2); + q2_k_int_accum(accum_q2_k_32x1((const HVX_Vector *) tile, v_act), v_sc2, v_mn2, v_act[9], &v_isum, &v_msum); + } + const HVX_Vector * restrict v_act_sb = (const HVX_Vector *) (y_q + kb * 1280); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, q2_k_int_scale(v_isum, v_msum, v_d, v_m, v_act_sb[8])); } if (sz) { @@ -1198,16 +1644,118 @@ static void tiled_vec_dot_q2_k_32x2(const uint32_t n, float * restrict s0, float HVX_Vector v_sum_float_c1 = Q6_V_vzero(); uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512); - const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280); - const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280); + for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) { + HVX_Vector v_d, v_m; + q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m); + + HVX_Vector v_isum0 = Q6_V_vzero(); + HVX_Vector v_msum0 = Q6_V_vzero(); + HVX_Vector v_isum1 = Q6_V_vzero(); + HVX_Vector v_msum1 = Q6_V_vzero(); + for (uint32_t kt = kb; kt < kb + 8; kt++) { + const uint8_t * restrict tile = tile_ptr + kt * 384; + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280); + + HVX_Vector v_sc2, v_mn2; + q2_k_scale_pairs(tile, &v_sc2, &v_mn2); + + HVX_VectorPair v_sums0, v_sums1; + accum_q2_k_32x2((const HVX_Vector *) tile, v_act0, v_act1, &v_sums0, &v_sums1); + q2_k_int_accum(v_sums0, v_sc2, v_mn2, v_act0[9], &v_isum0, &v_msum0); + q2_k_int_accum(v_sums1, v_sc2, v_mn2, v_act1[9], &v_isum1, &v_msum1); + } + const HVX_Vector * restrict v_act0_sb = (const HVX_Vector *) (y0_q + kb * 1280); + const HVX_Vector * restrict v_act1_sb = (const HVX_Vector *) (y1_q + kb * 1280); + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, q2_k_int_scale(v_isum0, v_msum0, v_d, v_m, v_act0_sb[8])); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, q2_k_int_scale(v_isum1, v_msum1, v_d, v_m, v_act1_sb[8])); + } - HVX_VectorPair v_sums0, v_sums1; - accum_q2_k_32x2(vptr, v_act0, v_act1, &v_sums0, &v_sums1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +// One Q2_0 weight tile covers 64 k: act tiles 2 * kt and 2 * kt + 1, each 32 k wide (Q8_1) +static void tiled_vec_dot_q2_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0); + + uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0; + for (uint32_t kt = 0; kt < n_weight_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0); + HVX_Vector v_tile_qf = Q6_V_vzero(); + + #pragma unroll + for (int half = 0; half < 2; half++) { + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); + + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q2_0_32x1(vptr + 2 * half, v_act)); + HVX_Vector v_da_sf = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8])); + HVX_Vector v_sa_qf = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act[9], v_one_f16)); + HVX_Vector v_h_qf = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf), v_sa_qf); + v_tile_qf = (half == 0) ? v_h_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_h_qf); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf)); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q2_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0); + + uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0; + for (uint32_t kt = 0; kt < n_weight_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0); + HVX_Vector v_tile_qf_c0 = Q6_V_vzero(); + HVX_Vector v_tile_qf_c1 = Q6_V_vzero(); + + #pragma unroll + for (int half = 0; half < 2; half++) { + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q2_k_32x1(v_sums0, vptr, v_act0)); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q2_k_32x1(v_sums1, vptr, v_act1)); + HVX_Vector v_sum_c0, v_sum_c1; + accum_q2_0_32x2(vptr + 2 * half, v_act0, v_act1, &v_sum_c0, &v_sum_c1); + + HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8])); + HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8])); + HVX_Vector v_sa_qf_c0 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act0[9], v_one_f16)); + HVX_Vector v_sa_qf_c1 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act1[9], v_one_f16)); + + HVX_Vector v_h_qf_c0 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c0), v_da_sf_c0), v_sa_qf_c0); + HVX_Vector v_h_qf_c1 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c1), v_da_sf_c1), v_sa_qf_c1); + + v_tile_qf_c0 = (half == 0) ? v_h_qf_c0 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, v_h_qf_c0); + v_tile_qf_c1 = (half == 0) ? v_h_qf_c1 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, v_h_qf_c1); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf)); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf)); } if (sz0) { @@ -1408,6 +1956,59 @@ static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, floa } } +static void tiled_vec_dot_f16_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) vx; + const HVX_Vector * restrict v_act = (const HVX_Vector *) vy; + const uint32_t n_vecs = ((n + 31) / 32) * 16; + + HVX_VectorPair acc = Q6_W_vzero(); + + #pragma unroll(4) + for (uint32_t i = 0; i < n_vecs; i++) { + acc = hvx_vec_mpyacc_f32_f16(acc, vptr[i], v_act[i]); + } + + HVX_Vector v_sum = hvx_vec_add_f32_f32(Q6_V_lo_W(acc), Q6_V_hi_W(acc)); + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum); + } +} + +static void tiled_vec_dot_f16_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) vx; + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) vy1; + const uint32_t n_vecs = ((n + 31) / 32) * 16; + + HVX_VectorPair acc0 = Q6_W_vzero(); + HVX_VectorPair acc1 = Q6_W_vzero(); + + #pragma unroll(4) + for (uint32_t i = 0; i < n_vecs; i++) { + HVX_Vector v_w = vptr[i]; + acc0 = hvx_vec_mpyacc_f32_f16(acc0, v_w, v_act0[i]); + acc1 = hvx_vec_mpyacc_f32_f16(acc1, v_w, v_act1[i]); + } + + HVX_Vector v_sum0 = hvx_vec_add_f32_f32(Q6_V_lo_W(acc0), Q6_V_hi_W(acc0)); + HVX_Vector v_sum1 = hvx_vec_add_f32_f32(Q6_V_lo_W(acc1), Q6_V_hi_W(acc1)); + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum1); + } +} + + static inline void quantize_f32_q8_0_tiled_kernel( const uint8_t * restrict src_data, uint8_t * restrict dst_data, @@ -1459,6 +2060,40 @@ static inline void quantize_f32_q8_1_s16_tiled_kernel( } } +static inline void quantize_f32_f16_tiled_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_row_size, + size_t dst_row_size +) { + (void) tmp_data; + for (uint32_t i = 0; i < nrows; ++i) { + quantize_row_f32_f16_tiled((const float *) src_data, dst_data, ne0); + dst_data += dst_row_size; + src_data += src_row_size; + } +} + +static inline void quantize_f16_f16_tiled_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_row_size, + size_t dst_row_size +) { + (void) tmp_data; + for (uint32_t i = 0; i < nrows; ++i) { + quantize_row_f16_f16_tiled((const __fp16 *) src_data, dst_data, ne0); + dst_data += dst_row_size; + src_data += src_row_size; + } +} + static inline void quantize_f32_q8_0_tiled_block_kernel( const float * restrict src, uint8_t * restrict dst, @@ -1472,7 +2107,7 @@ static inline void quantize_f32_q8_0_tiled_block_kernel( uint32_t c ) { (void) tmp_data; - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (ne0 + qk - 1) / qk; for (uint32_t ib = ib_first; ib < ib_last; ++ib) { @@ -1502,7 +2137,7 @@ static inline void quantize_f32_q8_1_tiled_block_kernel( uint32_t c ) { (void) tmp_data; - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (ne0 + qk - 1) / qk; for (uint32_t ib = ib_first; ib < ib_last; ++ib) { @@ -1532,14 +2167,74 @@ static inline void quantize_f32_q8_1_s16_tiled_block_kernel( uint32_t c ) { (void) tmp_data; - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (ne0 + qk - 1) / qk; for (uint32_t ib = ib_first; ib < ib_last; ++ib) { const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float)); uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 1280; - quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr); + quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr, src_ptr - (c & 1) * qk); // super-block of 256 k + + c++; + if (c == nb) { + c = 0; + r++; + } + } +} + +static inline void quantize_f32_f16_tiled_block_kernel( + const float * restrict src, + uint8_t * restrict dst, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t ib_first, + uint32_t ib_last, + size_t src_row_size, + size_t dst_row_size, + uint32_t r, + uint32_t c +) { + (void) tmp_data; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne0 + qk - 1) / qk; + + for (uint32_t ib = ib_first; ib < ib_last; ++ib) { + const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float)); + uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 2048; + + quantize_block_f32_f16_tiled(src_ptr, dst_ptr); + + c++; + if (c == nb) { + c = 0; + r++; + } + } +} + +static inline void quantize_f16_f16_tiled_block_kernel( + const __fp16 * restrict src, + uint8_t * restrict dst, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t ib_first, + uint32_t ib_last, + size_t src_row_size, + size_t dst_row_size, + uint32_t r, + uint32_t c +) { + (void) tmp_data; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne0 + qk - 1) / qk; + + for (uint32_t ib = ib_first; ib < ib_last; ++ib) { + const __fp16 * restrict src_ptr = (const __fp16 *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(__fp16)); + uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 2048; + + quantize_block_f16_f16_tiled(src_ptr, dst_ptr); c++; if (c == nb) { diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.h index 7ea945a33..5d13acc87 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-norm.h +++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h @@ -11,7 +11,8 @@ static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src, uint8_t * restrict dst, const int num_elems, - float epsilon) { + float epsilon, + float scale) { const HVX_Vector * restrict v_src = (HVX_Vector *) src; HVX_Vector * restrict v_dst = (HVX_Vector *) dst; @@ -48,6 +49,9 @@ static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src, // Scale full vectors HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + if (scale != 0.0f) { + scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale))); + } #pragma unroll(4) for (int i = 0; i < nvec; i++) { @@ -72,7 +76,8 @@ static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src, const uint8_t * restrict weight, uint8_t * restrict dst, const int num_elems, - float epsilon) { + float epsilon, + float scale) { const HVX_Vector * restrict v_src = (const HVX_Vector *) src; const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight; HVX_Vector * restrict v_dst = (HVX_Vector *) dst; @@ -109,6 +114,9 @@ static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src, // Scale and multiply HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + if (scale != 0.0f) { + scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale))); + } #pragma unroll(4) for (int i = 0; i < nvec; i++) { @@ -133,6 +141,71 @@ static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src, } } +static inline void hvx_fast_rms_norm_mul_add_f32(const uint8_t * restrict src, + const uint8_t * restrict weight, + const uint8_t * restrict add, + uint8_t * restrict dst, + const int num_elems, + float epsilon, + float scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight; + const HVX_Vector * restrict v_add = (const HVX_Vector *) add; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP32; + const int nloe = num_elems % VLEN_FP32; + + HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); + } + + sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v); + HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v); + + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + if (scale != 0.0f) { + scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale))); + } + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2); + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[i])); + v_dst[i] = hvx_vec_add_f32_f32(v4, v_add[i]); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2); + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[nvec])); + HVX_Vector res_v = hvx_vec_add_f32_f32(v4, v_add[nvec]); + + hvx_vec_store_a(&v_dst[nvec], nloe * 4, res_v); + } +} + static inline void hvx_fast_norm_f32(const uint8_t * restrict src, uint8_t * restrict dst, const int num_elems, diff --git a/ggml/src/ggml-hexagon/htp/hvx-quant.h b/ggml/src/ggml-hexagon/htp/hvx-quant.h index 6b172cd63..b0ca6ecf6 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-quant.h +++ b/ggml/src/ggml-hexagon/htp/hvx-quant.h @@ -5,6 +5,7 @@ #include <stdint.h> #include <string.h> +#include "htp-tensor.h" #include "hvx-arith.h" #include "hvx-base.h" #include "hvx-reduce.h" @@ -96,7 +97,7 @@ static inline void hvx_quantize_row_q8_0_f32(void * restrict dst_ptr, const floa } } -static inline void hvx_dequantize_row_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) { +static inline void hvx_dequantize_row_flat_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { const int nb = n / QK8_0; const block_q8_0 * src = (const block_q8_0 *) src_ptr; @@ -120,7 +121,7 @@ static inline void hvx_dequantize_row_q8_0_f32(float * restrict dst_ptr, const v } } -static inline void hvx_dequantize_row_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, int n) { +static inline void hvx_dequantize_row_flat_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { const int nb = n / QK8_0; const block_q8_0 * src = (const block_q8_0 *) src_ptr; @@ -146,20 +147,580 @@ static inline void hvx_dequantize_row_q8_0_f16(__fp16 * restrict dst_ptr, const } } -static inline void hvx_dequantize_row_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) { - const int nb = n / 32; +static inline void hvx_dequantize_row_flat_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; const _Float16 * src = (const _Float16 *) src_ptr; - for (int i = 0; i < nb; i++) { - HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * 32); + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * VLEN_FP32); HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16); HVX_Vector res = Q6_V_lo_W(vp_f32); - float * block_dst = dst_ptr + i * 32; - hvx_vmem(block_dst) = res; + hvx_vmem(dst_ptr + i * VLEN_FP32) = res; + } + + if (nloe) { + HVX_Vector v_f16 = *(const HVX_UVector *) (src + nvec * VLEN_FP32); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16); + HVX_Vector res = Q6_V_lo_W(vp_f32); + + hvx_vec_store_a(dst_ptr + nvec * VLEN_FP32, nloe * sizeof(float), res); } } +static inline void hvx_dequantize_row_flat_bf16_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + const uint16_t * src = (const uint16_t *) src_ptr; + + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector v_bf16 = *(const HVX_UVector *) (src + i * VLEN_FP32); + HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16); + HVX_Vector res = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16); + + hvx_vmem(dst_ptr + i * VLEN_FP32) = res; + } + + if (nloe) { + HVX_Vector v_bf16 = *(const HVX_UVector *) (src + nvec * VLEN_FP32); + HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16); + HVX_Vector res = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16); + + hvx_vec_store_a(dst_ptr + nvec * VLEN_FP32, nloe * sizeof(float), res); + } +} + +static inline void hvx_dequantize_row_flat_q1_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + static const uint32_t __attribute__((aligned(128))) bit_mask[32] = { + 1u << 0, 1u << 1, 1u << 2, 1u << 3, 1u << 4, 1u << 5, 1u << 6, 1u << 7, + 1u << 8, 1u << 9, 1u << 10, 1u << 11, 1u << 12, 1u << 13, 1u << 14, 1u << 15, + 1u << 16, 1u << 17, 1u << 18, 1u << 19, 1u << 20, 1u << 21, 1u << 22, 1u << 23, + 1u << 24, 1u << 25, 1u << 26, 1u << 27, 1u << 28, 1u << 29, 1u << 30, 1u << 31, + }; + const int nb = n / QK1_0; + const block_q1_0 * src = (const block_q1_0 *) src_ptr; + const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask; + const HVX_Vector v_zero = Q6_V_vzero(); + const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000); + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16)); + HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit); + HVX_Vector v_qs = *(const HVX_UVector *) src[i].qs; + + float * block_dst = dst_ptr + i * QK1_0; + #pragma unroll + for (int q = 0; q < 4; q++) { + HVX_Vector v_bits = hvx_vec_repl_u32(q == 0 ? v_qs : Q6_V_vror_VR(v_qs, 4 * q)); + HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero); + hvx_vmem(block_dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd); + } + } +} + +static inline void hvx_dequantize_row_flat_q2_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK2_0; + const block_q2_0 * src = (const block_q2_0 *) src_ptr; + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector v_one = Q6_Vb_vsplat_R(1); + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16)); + + HVX_Vector vq = *(const HVX_UVector *) src[i].qs; + HVX_Vector v0 = Q6_V_vand_VV(vq, mask_03); + HVX_Vector v1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 2), mask_03); + HVX_Vector v2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 4), mask_03); + HVX_Vector v3 = Q6_Vub_vlsr_VubR(vq, 6); + + HVX_Vector v01 = Q6_V_lo_W(Q6_W_vshuff_VVR(v1, v0, -1)); + HVX_Vector v23 = Q6_V_lo_W(Q6_W_vshuff_VVR(v3, v2, -1)); + HVX_Vector v64 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(v23, v01, -2)), v_one); + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v64); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + + float * block_dst = dst_ptr + i * QK2_0; + hvx_vmem(block_dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), vd); + hvx_vmem(block_dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), vd); + } +} + +static inline void hvx_dequantize_row_flat_q4_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK4_0; + const block_q4_0 * src = (const block_q4_0 *) src_ptr; + + const HVX_VectorPred q16 = Q6_Q_vsetq_R(16); + const HVX_Vector v_eight = Q6_Vb_vsplat_R(8); + const HVX_Vector v_zero = Q6_V_vzero(); + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16); + HVX_Vector vd = Q6_V_lo_W(vp_f32); + + HVX_Vector vq = *(const HVX_UVector *) src[i].qs; + HVX_Vector v_lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + HVX_Vector v_hi = Q6_Vub_vlsr_VubR(vq, 4); + + HVX_Vector v_lo_s8 = Q6_Vb_vsub_VbVb(v_lo, v_eight); + HVX_Vector v_hi_s8 = Q6_Vb_vsub_VbVb(v_hi, v_eight); + + HVX_Vector v32 = Q6_V_vmux_QVV(q16, v_lo_s8, Q6_V_vlalign_VVR(v_hi_s8, v_zero, 16)); + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v32); + HVX_Vector v_i16 = Q6_V_lo_W(p16); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16); + HVX_Vector v_i32 = Q6_V_lo_W(p32); + + HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32); + HVX_Vector res = hvx_vec_mul_f32_f32(v_f32, vd); + + hvx_vmem(dst_ptr + i * QK4_0) = res; + } +} + +static inline void hvx_dequantize_row_flat_q4_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK_K; + const block_q4_K * src = (const block_q4_K *) src_ptr; + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(src[i].d); + const float min = GGML_FP16_TO_FP32(src[i].dmin); + const uint8_t * sc = src[i].scales; + + float * y = dst_ptr + i * QK_K; + + for (int step = 0; step < 4; ++step) { + uint8_t sc1, m1_u, sc2, m2_u; + if (step < 2) { + sc1 = sc[2 * step + 0] & 63; + m1_u = sc[2 * step + 4] & 63; + sc2 = sc[2 * step + 1] & 63; + m2_u = sc[2 * step + 5] & 63; + } else { + const int s = step - 2; + sc1 = (sc[8 + 2 * s + 0] & 0xF) | ((sc[2 * s + 0] >> 6) << 4); + m1_u = (sc[8 + 2 * s + 0] >> 4) | ((sc[4 + 2 * s + 0] >> 6) << 4); + sc2 = (sc[8 + 2 * s + 1] & 0xF) | ((sc[2 * s + 1] >> 6) << 4); + m2_u = (sc[8 + 2 * s + 1] >> 4) | ((sc[4 + 2 * s + 1] >> 6) << 4); + } + + HVX_Vector vq = *(const HVX_UVector *) (src[i].qs + step * 32); + HVX_Vector v_lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + HVX_Vector v_hi = Q6_Vub_vlsr_VubR(vq, 4); + + HVX_VectorPair p16_lo = Q6_Wuh_vunpack_Vub(v_lo); + HVX_VectorPair p32_lo = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_lo)); + HVX_Vector v_i32_lo = Q6_V_lo_W(p32_lo); + HVX_Vector v_f32_lo = Q6_Vsf_equals_Vw(v_i32_lo); + + HVX_Vector vd1 = hvx_vec_splat_f32(d * sc1); + HVX_Vector vm1 = hvx_vec_splat_f32(min * m1_u); + HVX_Vector v_out1 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_lo, vd1), vm1); + hvx_vmem(y + 0) = v_out1; + + HVX_VectorPair p16_hi = Q6_Wuh_vunpack_Vub(v_hi); + HVX_VectorPair p32_hi = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_hi)); + HVX_Vector v_i32_hi = Q6_V_lo_W(p32_hi); + HVX_Vector v_f32_hi = Q6_Vsf_equals_Vw(v_i32_hi); + + HVX_Vector vd2 = hvx_vec_splat_f32(d * sc2); + HVX_Vector vm2 = hvx_vec_splat_f32(min * m2_u); + HVX_Vector v_out2 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_hi, vd2), vm2); + hvx_vmem(y + 32) = v_out2; + + y += 64; + } + } +} +static inline void hvx_dequantize_row_flat_q5_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK_K; + const block_q5_K * src = (const block_q5_K *) src_ptr; + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(src[i].d); + const float min = GGML_FP16_TO_FP32(src[i].dmin); + const uint8_t * sc = src[i].scales; + + HVX_Vector v_qh = *(const HVX_UVector *) src[i].qh; + float * y = dst_ptr + i * QK_K; + + for (int step = 0; step < 4; ++step) { + uint8_t sc1, m1_u, sc2, m2_u; + if (step < 2) { + sc1 = sc[2 * step + 0] & 63; + m1_u = sc[2 * step + 4] & 63; + sc2 = sc[2 * step + 1] & 63; + m2_u = sc[2 * step + 5] & 63; + } else { + const int s = step - 2; + sc1 = (sc[8 + 2 * s + 0] & 0xF) | ((sc[2 * s + 0] >> 6) << 4); + m1_u = (sc[8 + 2 * s + 0] >> 4) | ((sc[4 + 2 * s + 0] >> 6) << 4); + sc2 = (sc[8 + 2 * s + 1] & 0xF) | ((sc[2 * s + 1] >> 6) << 4); + m2_u = (sc[8 + 2 * s + 1] >> 4) | ((sc[4 + 2 * s + 1] >> 6) << 4); + } + + HVX_Vector v_ql = *(const HVX_UVector *) (src[i].qs + step * 32); + HVX_Vector v_lo = Q6_V_vand_VV(v_ql, Q6_Vb_vsplat_R(0x0F)); + HVX_Vector v_hi = Q6_Vub_vlsr_VubR(v_ql, 4); + + HVX_Vector v_h1 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2 * step), Q6_Vb_vsplat_R(1)), 4); + HVX_Vector v_val1 = Q6_V_vor_VV(v_lo, v_h1); + + HVX_VectorPair p16_1 = Q6_Wuh_vunpack_Vub(v_val1); + HVX_VectorPair p32_1 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_1)); + HVX_Vector v_i32_1 = Q6_V_lo_W(p32_1); + HVX_Vector v_f32_1 = Q6_Vsf_equals_Vw(v_i32_1); + + HVX_Vector vd1 = hvx_vec_splat_f32(d * sc1); + HVX_Vector vm1 = hvx_vec_splat_f32(min * m1_u); + HVX_Vector v_out1 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_1, vd1), vm1); + hvx_vmem(y + 0) = v_out1; + + HVX_Vector v_h2 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2 * step + 1), Q6_Vb_vsplat_R(1)), 4); + HVX_Vector v_val2 = Q6_V_vor_VV(v_hi, v_h2); + + HVX_VectorPair p16_2 = Q6_Wuh_vunpack_Vub(v_val2); + HVX_VectorPair p32_2 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_2)); + HVX_Vector v_i32_2 = Q6_V_lo_W(p32_2); + HVX_Vector v_f32_2 = Q6_Vsf_equals_Vw(v_i32_2); + + HVX_Vector vd2 = hvx_vec_splat_f32(d * sc2); + HVX_Vector vm2 = hvx_vec_splat_f32(min * m2_u); + HVX_Vector v_out2 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_2, vd2), vm2); + hvx_vmem(y + 32) = v_out2; + + y += 64; + } + } +} + +static inline void hvx_dequantize_row_flat_q6_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK_K; + const block_q6_K * src = (const block_q6_K *) src_ptr; + + const HVX_VectorPred q64 = Q6_Q_vsetq_R(64); + const HVX_Vector v32_b = Q6_Vb_vsplat_R(32); + const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(src[i].d); + + const uint8_t * restrict ql = src[i].ql; + const uint8_t * restrict qh = src[i].qh; + const int8_t * restrict sc = src[i].scales; + + float * y = dst_ptr + i * QK_K; + + for (int half = 0; half < 2; ++half) { + float s[8]; + for (int j = 0; j < 8; ++j) { + s[j] = d * sc[j]; + } + + HVX_Vector vscale1 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[0]), hvx_vec_splat_f32(s[1])); + HVX_Vector vscale2 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[2]), hvx_vec_splat_f32(s[3])); + HVX_Vector vscale3 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[4]), hvx_vec_splat_f32(s[5])); + HVX_Vector vscale4 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[6]), hvx_vec_splat_f32(s[7])); + + HVX_Vector v_ql0 = *(const HVX_UVector *) (ql); + HVX_Vector v_ql1 = *(const HVX_UVector *) (ql + 32); + HVX_Vector v_qh = *(const HVX_UVector *) (qh); + + HVX_Vector ql0_lo = Q6_V_vand_VV(v_ql0, mask_0f); + HVX_Vector qh_0 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(v_qh, mask_03), 4); + HVX_Vector v_q1 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql0_lo, qh_0), v32_b); + + HVX_Vector ql1_lo = Q6_V_vand_VV(v_ql1, mask_0f); + HVX_Vector qh_1 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2), mask_03), 4); + HVX_Vector v_q2 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql1_lo, qh_1), v32_b); + + HVX_Vector ql0_hi = Q6_Vub_vlsr_VubR(v_ql0, 4); + HVX_Vector qh_2 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 4), mask_03), 4); + HVX_Vector v_q3 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql0_hi, qh_2), v32_b); + + HVX_Vector ql1_hi = Q6_Vub_vlsr_VubR(v_ql1, 4); + HVX_Vector qh_3 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 6), mask_03), 4); + HVX_Vector v_q4 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql1_hi, qh_3), v32_b); + + HVX_VectorPair p16_1 = Q6_Wh_vunpack_Vb(v_q1); + HVX_VectorPair p32_1 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_1)); + HVX_Vector v_f32_1 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_1)); + hvx_vmem(y + 0) = hvx_vec_mul_f32_f32(v_f32_1, vscale1); + + HVX_VectorPair p16_2 = Q6_Wh_vunpack_Vb(v_q2); + HVX_VectorPair p32_2 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_2)); + HVX_Vector v_f32_2 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_2)); + hvx_vmem(y + 32) = hvx_vec_mul_f32_f32(v_f32_2, vscale2); + + HVX_VectorPair p16_3 = Q6_Wh_vunpack_Vb(v_q3); + HVX_VectorPair p32_3 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_3)); + HVX_Vector v_f32_3 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_3)); + hvx_vmem(y + 64) = hvx_vec_mul_f32_f32(v_f32_3, vscale3); + + HVX_VectorPair p16_4 = Q6_Wh_vunpack_Vb(v_q4); + HVX_VectorPair p32_4 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_4)); + HVX_Vector v_f32_4 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_4)); + hvx_vmem(y + 96) = hvx_vec_mul_f32_f32(v_f32_4, vscale4); + + y += 128; + ql += 64; + qh += 32; + sc += 8; + } + } +} + +// bytes row, row + 32, row + 64 and row + 96 of v, to bytes 0..3 +static inline HVX_Vector hvx_dequantize_gather4(HVX_Vector v, uint32_t row) { + v = Q6_V_vror_VR(v, row); + HVX_Vector packed = Q6_V_vmux_QVV(Q6_Q_vsetq_R(1), v, Q6_V_vror_VR(v, 31)); + packed = Q6_V_vmux_QVV(Q6_Q_vsetq_R(2), packed, Q6_V_vror_VR(v, 62)); + return Q6_V_vmux_QVV(Q6_Q_vsetq_R(3), packed, Q6_V_vror_VR(v, 93)); +} + +static inline void hvx_dequantize_row_tiled_q8_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first2 = Q6_Q_vsetq_R(2); + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector vq = Q6_V_vzero(); + for (int group = 7; group >= 0; --group) { + const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row); + const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62)); + vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector scales = hvx_vmem(tile + 1024); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = values; + } +} + +static inline void hvx_dequantize_row_tiled_q4_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector vq = Q6_V_vzero(); + for (int group = 3; group >= 0; --group) { + vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4); + vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); + vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8)); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = values; + } +} + +static inline void hvx_dequantize_row_tiled_q4_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector vq = Q6_V_vzero(); + for (int group = 3; group >= 0; --group) { + vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4); + vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2)); + const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf)); + + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = hvx_vec_add_f32_f32(values, offset); + } +} + +static inline void hvx_dequantize_row_tiled_q5_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + const HVX_Vector one = Q6_Vb_vsplat_R(1); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + const HVX_Vector p5 = hvx_dequantize_gather4(hvx_vmem(tile + 640), row); + HVX_Vector vq = Q6_V_vzero(); + HVX_Vector vh = Q6_V_vzero(); + for (int group = 3; group >= 0; --group) { + vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4)); + vh = Q6_V_vmux_QVV(first4, Q6_Vub_vlsr_VubR(p5, 2 * group), Q6_V_vror_VR(vh, VLEN - 4)); + } + + const HVX_Vector lo = Q6_V_vor_VV(Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)), Q6_Vw_vasl_VwR(Q6_V_vand_VV(vh, one), 4)); + const HVX_Vector hi = Q6_V_vor_VV(Q6_Vub_vlsr_VubR(vq, 4), Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vh, 1), one), 4)); + vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2)); + const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf)); + + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = hvx_vec_add_f32_f32(values, offset); + } +} + +static inline void hvx_dequantize_row_tiled_q6_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + const HVX_VectorPred first16 = Q6_Q_vsetq_R(16 * sizeof(float)); + const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector vq = Q6_V_vzero(); + for (int group = 7; group >= 0; --group) { + const HVX_Vector lo_plane = Q6_V_vror_VR(hvx_vmem(tile + (group >> 1) * VLEN), 4 * row); + const HVX_Vector hi_plane = Q6_V_vror_VR(hvx_vmem(tile + 512 + (group >> 2) * VLEN), 4 * row); + const HVX_Vector lo = (group & 1) ? Q6_Vub_vlsr_VubR(lo_plane, 4) : Q6_V_vand_VV(lo_plane, mask_0f); + const HVX_Vector hi = Q6_Vub_vlsr_VubR(hi_plane, 2 * (group & 3)); + const HVX_Vector packed = Q6_V_vor_VV(lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(hi, mask_03), 4)); + vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector scales = hvx_vmem(tile + 768); + const HVX_Vector scale_lo_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale_hi_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 64 + 2 * row)); + const HVX_Vector scale_lo = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_lo_hf)); + const HVX_Vector scale_hi = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hi_hf)); + const HVX_Vector scale = Q6_V_vmux_QVV(first16, scale_lo, scale_hi); + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(32))); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + *(HVX_Vector *) dst = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + } +} + +static inline HVX_Vector hvx_dequantize_tiled_f16_gather4(const uint8_t * p, uint32_t r_byte) { + HVX_Vector w0 = Q6_V_vror_VR(hvx_vmem(p + 0 * VLEN), r_byte); + HVX_Vector w1 = Q6_V_vror_VR(hvx_vmem(p + 1 * VLEN), r_byte); + HVX_Vector w01 = Q6_V_lo_W(Q6_W_vshuff_VVR(w1, w0, 4)); + HVX_Vector w2 = Q6_V_vror_VR(hvx_vmem(p + 2 * VLEN), r_byte); + HVX_Vector w3 = Q6_V_vror_VR(hvx_vmem(p + 3 * VLEN), r_byte); + HVX_Vector w23 = Q6_V_lo_W(Q6_W_vshuff_VVR(w3, w2, 4)); + return Q6_V_lo_W(Q6_W_vshuff_VVR(w23, w01, 8)); +} + +static inline void hvx_dequantize_row_tiled_f16_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const uint32_t r_byte = 4 * row; + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector g0 = hvx_dequantize_tiled_f16_gather4(tile + 0 * VLEN, r_byte); + HVX_Vector g1 = hvx_dequantize_tiled_f16_gather4(tile + 4 * VLEN, r_byte); + HVX_Vector g01 = Q6_V_lo_W(Q6_W_vshuff_VVR(g1, g0, 16)); + + HVX_Vector g2 = hvx_dequantize_tiled_f16_gather4(tile + 8 * VLEN, r_byte); + HVX_Vector g3 = hvx_dequantize_tiled_f16_gather4(tile + 12 * VLEN, r_byte); + HVX_Vector g23 = Q6_V_lo_W(Q6_W_vshuff_VVR(g3, g2, 16)); + + HVX_Vector g = Q6_V_lo_W(Q6_W_vshuff_VVR(g23, g01, 32)); + *(HVX_Vector *) dst = Q6_V_lo_W(hvx_vec_f16_to_f32(g)); + } +} + +static inline void hvx_dequantize_row_tiled_q1_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + static const uint32_t __attribute__((aligned(128))) bit_mask[32] = { + 1u << 0, 1u << 8, 1u << 16, 1u << 24, + 1u << 1, 1u << 9, 1u << 17, 1u << 25, + 1u << 2, 1u << 10, 1u << 18, 1u << 26, + 1u << 3, 1u << 11, 1u << 19, 1u << 27, + 1u << 4, 1u << 12, 1u << 20, 1u << 28, + 1u << 5, 1u << 13, 1u << 21, 1u << 29, + 1u << 6, 1u << 14, 1u << 22, 1u << 30, + 1u << 7, 1u << 15, 1u << 23, 1u << 31, + }; + const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask; + const HVX_Vector v_zero = Q6_V_vzero(); + const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q1_0; + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit); + + #pragma unroll + for (int q = 0; q < 4; ++q) { + const HVX_Vector v_bits = hvx_vec_repl_u32(Q6_V_vror_VR(hvx_vmem(tile + q * VLEN), 4 * row)); + const HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero); + hvx_vmem(dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd); + } + } +} + +static inline void hvx_dequantize_row_tiled_q2_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector v_one = Q6_Vb_vsplat_R(1); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q2_0; + + const HVX_Vector v_packed = hvx_dequantize_tiled_f16_gather4(tile, 4 * row); + const HVX_Vector p0 = Q6_V_vand_VV(v_packed, mask_03); + const HVX_Vector p1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 2), mask_03); + const HVX_Vector p2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 4), mask_03); + const HVX_Vector p3 = Q6_Vub_vlsr_VubR(v_packed, 6); + + const HVX_Vector p01 = Q6_V_lo_W(Q6_W_vshuff_VVR(p1, p0, -4)); + const HVX_Vector p23 = Q6_V_lo_W(Q6_W_vshuff_VVR(p3, p2, -4)); + const HVX_Vector vq = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(p23, p01, -8)), v_one); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + hvx_vmem(dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + hvx_vmem(dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), scale); + } +} #endif // HVX_QUANT_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index b7c2d7409..3f253a0b0 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -52,6 +52,53 @@ static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard(HVX_Vector v, return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); } +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) { + v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3)); + + HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Newton-Raphson with 2 iterations + HVX_Vector two_sf = hvx_vec_splat_f32(2.0f); + HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); + HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( + i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); + r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( + r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); + HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf); + + res = Q6_Vqf32_vmpy_VsfVsf(v3, res); + + return Q6_Vsf_equals_Vqf32(res); +} + +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v, + HVX_Vector one, + HVX_Vector max_exp, + HVX_Vector min_exp) { + const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v); + const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp); + + HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v); + out = Q6_V_vmux_QVV(pred_max, out, one); + return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); +} + static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) { // tanh(x) = 2 * sigmoid(2x) - 1 HVX_Vector two = hvx_vec_splat_f32(2.0f); @@ -68,6 +115,14 @@ static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) { return Q6_Vsf_equals_Vqf32(res); } +static inline HVX_Vector hvx_vec_silu_f32(HVX_Vector x) { + HVX_Vector one = hvx_vec_splat_f32(1.0f); + HVX_Vector max_exp = hvx_vec_splat_f32(87.f); + HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); + HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(x, one, max_exp, min_exp); + return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, sig)); +} + #define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ do { \ dst_type * restrict vdst = (dst_type *) dst; \ diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c index 2e05cf3e1..602ee1598 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.c +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c @@ -15,572 +15,1099 @@ #include "htp-ops.h" #include "hvx-utils.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-vtcm.h" #include "htp-tensor.h" +#include "im2col-ops.h" struct htp_im2col_context { struct htp_ops_context * octx; - uint32_t patch_base; // first patch index assigned to this dev - uint32_t npatches; // number of patches assigned to this dev - uint32_t npatches_per_thread; // patches = N*OH*OW (pure-DDR kernel) - - uint32_t pe_row_base; // first N*OH row index assigned to this dev (DMA path) - uint32_t pe_nrows; // number of N*OH rows assigned to this dev (DMA path) - uint32_t pe_rows_per_thread; // N*OH rows per worker - uint32_t pe_src_row_bytes; // one output row's source: IC*KH*IW*4, rounded 256 + + uint32_t pe_row_base; // first output row index assigned to this dev (DMA path) + uint32_t pe_nrows; // number of output rows assigned to this dev (DMA path) + uint32_t pe_rows_per_hvx; // output rows per worker + uint32_t pe_blocks_per_hvx; // output-col blocks per worker (blocked DMA path) + uint32_t pe_src_row_bytes; // one output row's source, rounded 256 uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 // Patch-embed DMA path VTCM ping-pong. uint8_t * pe_vtcm_src; // base of the 2x src buffers region uint8_t * pe_vtcm_dst; // base of the 2x dst buffers region - uint32_t pe_src_size_per_thread; // 2 * pe_src_row_bytes - uint32_t pe_dst_size_per_thread; // 2 * pe_dst_row_bytes + uint32_t pe_src_size_per_hvx; // 2 * pe_src_row_bytes + uint32_t pe_dst_size_per_hvx; // 2 * pe_dst_row_bytes uint32_t pe_owb; // output-col block size uint32_t pe_wb; // staged source window width + uint32_t pe_nbpr; // blocks per row + uint32_t pe_gather_nvec; // vectors per gathered patch (KH == 1 blocked path) + + // Precomputed dimensions for worker threads + uint32_t N; + uint32_t IC; + uint32_t ID, IH, IW; + uint32_t KD, KH, KW; + uint32_t OD, OH, OW; + uint32_t patch_stride; + + // Per-thread initial coordinates (removes division from thread prologue) + uint32_t thread_in[HTP_MAX_NHVX]; + uint32_t thread_iod[HTP_MAX_NHVX]; + uint32_t thread_ioh[HTP_MAX_NHVX]; + uint32_t thread_c0[HTP_MAX_NHVX]; + + void * compute; + uint32_t dst_elem; }; -// Per-op VTCM layout for the patch-embed DMA path -struct htp_im2col_vtcm_layout { - size_t off_src; - size_t off_dst; - size_t src_bytes_per_thread; - size_t dst_bytes_per_thread; - size_t total_bytes; +typedef void (*im2col_blocked_compute_fn_t)( + const struct htp_im2col_context * ictx, + void * restrict dstb, const float * restrict srcb, + float * restrict scratch, + uint32_t nb, uint32_t c0, uint32_t ioh); + +typedef void (*im2col_patchembed_compute_fn_t)( + const struct htp_im2col_context * ictx, + void * restrict dstb, const float * restrict srcb, uint32_t ioh); + +typedef void (*im2col_3d_blocked_compute_fn_t)( + const struct htp_im2col_context * ictx, + void * restrict dstb, const float * restrict srcb, + float * restrict scratch, + uint32_t nb, uint32_t c0, uint32_t iod, uint32_t ioh); + +typedef void (*im2col_3d_patchembed_compute_fn_t)( + const struct htp_im2col_context * ictx, + void * restrict dstb, const float * restrict srcb); + +struct im2col_cursor { + uint32_t r; + uint32_t in; + uint32_t iod; + uint32_t ioh; + uint32_t c0; }; -static inline void htp_im2col_vtcm_layout_build(struct htp_im2col_vtcm_layout * L, - size_t src_row_bytes, - size_t dst_row_bytes, - uint32_t n_threads) { - L->src_bytes_per_thread = 2 * src_row_bytes; - L->dst_bytes_per_thread = 2 * dst_row_bytes; +static inline void im2col_step_row(struct im2col_cursor * c, const struct htp_im2col_context * ictx) { + c->r++; + c->ioh++; + if (c->ioh >= ictx->OH) { + c->ioh = 0; + c->in++; + } +} - L->off_src = 0; - L->off_dst = L->off_src + L->src_bytes_per_thread * n_threads; - L->total_bytes = L->off_dst + L->dst_bytes_per_thread * n_threads; +static inline void im2col_step_block(struct im2col_cursor * c, const struct htp_im2col_context * ictx) { + c->c0 += ictx->pe_owb; + if (c->c0 >= ictx->OW) { + c->c0 = 0; + c->r++; + c->ioh++; + if (c->ioh >= ictx->OH) { + c->ioh = 0; + c->in++; + } + } } -#define IM2COL_PATCHEMBED_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN, DST_ELEM, TAG) \ - static void FNAME(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ - struct htp_ops_context * octx = ictx->octx; \ - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ - const struct htp_tensor * restrict src0 = octx->src[0]; \ - const struct htp_tensor * restrict src1 = octx->src[1]; \ - const struct htp_tensor * restrict dst = octx->dst; \ - const int32_t s0 = octx->op_params[0]; \ - const int32_t s1 = octx->op_params[1]; \ - const int32_t p0 = octx->op_params[2]; \ - const int32_t p1 = octx->op_params[3]; \ - const int32_t d0 = octx->op_params[4]; \ - const int32_t d1 = octx->op_params[5]; \ - const int32_t is_2D = octx->op_params[6] == 1; \ - const uint32_t N = is_2D ? src1->ne[3] : src1->ne[2]; \ - const uint32_t IC = is_2D ? src1->ne[2] : src1->ne[1]; \ - const uint32_t IH = is_2D ? src1->ne[1] : 1; \ - const uint32_t IW = src1->ne[0]; \ - const uint32_t KH = is_2D ? src0->ne[1] : 1; \ - const uint32_t KW = src0->ne[0]; \ - const uint32_t OH = is_2D ? dst->ne[2] : 1; \ - const uint32_t OW = dst->ne[1]; \ - const uint32_t patch_stride = IC * KH * KW; \ - const float * restrict src_data = (const float *) (uintptr_t) src1->data; \ - DST_CTYPE * restrict dst_data = (DST_CTYPE *) (uintptr_t) dst->data; \ - const uint32_t patch_end = ictx->patch_base + ictx->npatches; \ - const uint32_t patch_start = ictx->patch_base + ictx->npatches_per_thread * ith; \ - const uint32_t patch_stop = MIN(patch_start + ictx->npatches_per_thread, patch_end); \ - if (patch_start >= patch_stop) { \ - return; \ - } \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \ - for (uint32_t p = patch_start; p < patch_stop; p++) { \ - const uint32_t iow = p % OW; \ - const uint32_t ioh = (p / OW) % OH; \ - const uint32_t in = p / (OW * OH); \ - DST_CTYPE * restrict dst_patch = dst_data + (uint64_t) p * patch_stride; \ - for (uint32_t iic = 0; iic < IC; iic++) { \ - const float * restrict src_plane = src_data + ((uint64_t) in * IC + iic) * IH * IW; \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ - DST_CTYPE * restrict out_run = dst_patch + iic * (KH * KW) + ikh * KW; \ - if (iih < 0 || iih >= (int32_t) IH) { \ - SPLAT_FN(out_run, 0.0f, KW); \ - continue; \ - } \ - const int32_t iiw0 = (int32_t) iow * s0 - p0; \ - const float * restrict src_run = src_plane + (uint64_t) iih * IW + iiw0; \ - if (d0 == 1) { \ - /* contiguous source run: [lo,hi) is in-bounds, tails are zero pad */ \ - const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ - int32_t hi = (int32_t) IW - iiw0; \ - if (hi > (int32_t) KW) { \ - hi = (int32_t) KW; \ - } \ - if (hi <= lo) { \ - SPLAT_FN(out_run, 0.0f, KW); \ - } else { \ - if (lo > 0) { \ - SPLAT_FN(out_run, 0.0f, (uint32_t) lo); \ - } \ - COPY_FN((uint8_t *) (out_run + lo), (const uint8_t *) (src_run + lo), \ - (uint32_t) (hi - lo)); \ - if (hi < (int32_t) KW) { \ - SPLAT_FN(out_run + hi, 0.0f, (KW - (uint32_t) hi)); \ - } \ - } \ - continue; \ - } \ - for (uint32_t ikw = 0; ikw < KW; ikw++) { \ - const int32_t iiw = (int32_t) iow * s0 + (int32_t) ikw * d0 - p0; \ - out_run[ikw] = (iiw < 0 || iiw >= (int32_t) IW) ? \ - (DST_CTYPE) 0.0f : \ - (DST_CTYPE) src_plane[(uint64_t) iih * IW + iiw]; \ - } \ - } \ - } \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \ - } - -IM2COL_PATCHEMBED_BODY(im2col_patchembed_thread, __fp16, hvx_copy_f16_f32_uu, hvx_splat_f16_u, sizeof(__fp16), "f32-f16") -IM2COL_PATCHEMBED_BODY(im2col_patchembed_f32_thread, float, hvx_copy_f32_uu, hvx_splat_f32_u, sizeof(float), "f32-f32") - -// Software-pipelined 2-deep: while HVX computes block bi from buffer slot -// (bi&1), the DMA engine stages block bi+1 into the other slot concurrently. -// A single dma_queue_flush per iteration (after issuing the next stage-in and -// this block's store-out) waits for both - safe because the ring is strict -// FIFO and each buffer slot is only reused after its prior consumer (compute -// or store-out) already finished in program order. -#define IM2COL_BLOCKED_DMA_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN, DST_ELEM, TAG) \ - static void FNAME(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ - struct htp_ops_context * octx = ictx->octx; \ - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ - const struct htp_tensor * restrict src1 = octx->src[1]; \ - const struct htp_tensor * restrict dst = octx->dst; \ - const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; \ - const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; \ - const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; \ - const int32_t is_2D = octx->op_params[6] == 1; \ - const uint32_t N = is_2D ? src1->ne[3] : src1->ne[2]; \ - const uint32_t IC = is_2D ? src1->ne[2] : src1->ne[1]; \ - const uint32_t IH = is_2D ? src1->ne[1] : 1; \ - const uint32_t IW = src1->ne[0]; \ - const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; \ - const uint32_t KW = octx->src[0]->ne[0]; \ - const uint32_t OH = is_2D ? dst->ne[2] : 1; \ - const uint32_t OW = dst->ne[1]; \ - const uint32_t owb = ictx->pe_owb, Wb = ictx->pe_wb; \ - const uint32_t patch_stride = IC * KH * KW; \ - const dma_addr_t src_data = src1->data; \ - const dma_addr_t dst_data = dst->data; \ - dma_queue * dmaq = octx->ctx->dma[ith]; \ - uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; \ - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; \ - float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; \ - DST_CTYPE * dstb2[2] = { (DST_CTYPE *) dstb_base, (DST_CTYPE *) (dstb_base + ictx->pe_dst_row_bytes) }; \ - const uint32_t nrows = N * OH; \ - const uint32_t per_thread = ictx->pe_rows_per_thread; \ - const uint32_t row_start = per_thread * ith; \ - const uint32_t row_end = MIN(row_start + per_thread, nrows); \ - if (row_start >= row_end) \ - return; \ - const uint32_t nbpr = (OW + owb - 1) / owb; \ - const uint32_t nrows_local = row_end - row_start; \ - const uint32_t total_blocks = nrows_local * nbpr; \ - for (uint32_t bi = 0; bi < total_blocks; bi++) { \ - const uint32_t buf = bi & 1u; \ - float * srcb = srcb2[buf]; \ - DST_CTYPE * dstb = dstb2[buf]; \ - const uint32_t r = row_start + bi / nbpr; \ - const uint32_t in = r / OH; \ - const uint32_t ioh = r % OH; \ - const uint32_t c0 = (bi % nbpr) * owb; \ - const uint32_t nb = MIN(owb, OW - c0); \ - const int32_t win0 = (int32_t) c0 * s0 - p0; \ - if (bi == 0) { \ - /* prologue: stage block 0 and wait - nothing to overlap with yet */ \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ - if (iih < 0 || iih >= (int32_t) IH) \ - continue; \ - const int32_t lo = win0 < 0 ? -win0 : 0; \ - int32_t hi = (int32_t) IW - win0; \ - if (hi > (int32_t) Wb) \ - hi = (int32_t) Wb; \ - if (hi <= lo) \ - continue; \ - const uint32_t cpw = (uint32_t) (hi - lo); \ - float * vdst = srcb + (size_t) ikh * Wb + lo; \ - const dma_addr_t vsrc = src_data + (size_t) (((in * IC) * IH + iih) * IW + (win0 + lo)) * sizeof(float); \ - while (!dma_queue_push(dmaq, dma_make_data(vdst, vsrc), \ - (size_t) KH * Wb * sizeof(float), (size_t) IH * IW * sizeof(float), \ - cpw * sizeof(float), IC)) { \ - dma_queue_pop(dmaq); \ - } \ - } \ - dma_queue_flush(dmaq); \ - } \ - if (bi + 1 < total_blocks) { \ - /* prefetch: stage block bi+1 into the other slot; overlaps with this block's compute below */ \ - const uint32_t nbuf = 1u - buf; \ - float * nsrcb = srcb2[nbuf]; \ - const uint32_t nr = row_start + (bi + 1) / nbpr; \ - const uint32_t nin = nr / OH; \ - const uint32_t nioh = nr % OH; \ - const uint32_t nc0 = ((bi + 1) % nbpr) * owb; \ - const int32_t nwin0 = (int32_t) nc0 * s0 - p0; \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - const int32_t iih = (int32_t) nioh * s1 + (int32_t) ikh * d1 - p1; \ - if (iih < 0 || iih >= (int32_t) IH) \ - continue; \ - const int32_t lo = nwin0 < 0 ? -nwin0 : 0; \ - int32_t hi = (int32_t) IW - nwin0; \ - if (hi > (int32_t) Wb) \ - hi = (int32_t) Wb; \ - if (hi <= lo) \ - continue; \ - const uint32_t cpw = (uint32_t) (hi - lo); \ - float * vdst = nsrcb + (size_t) ikh * Wb + lo; \ - const dma_addr_t vsrc = src_data + (size_t) (((nin * IC) * IH + iih) * IW + (nwin0 + lo)) * sizeof(float); \ - while (!dma_queue_push(dmaq, dma_make_data(vdst, vsrc), \ - (size_t) KH * Wb * sizeof(float), (size_t) IH * IW * sizeof(float), \ - cpw * sizeof(float), IC)) { \ - dma_queue_pop(dmaq); \ - } \ - } \ - } \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); \ - for (uint32_t j = 0; j < nb; j++) { \ - const uint32_t iow = c0 + j; \ - DST_CTYPE * dst_patch = dstb + (uint64_t) j * patch_stride; \ - const int32_t iiw0 = (int32_t) iow * s0 - p0; \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ - const int okh = (iih >= 0 && iih < (int32_t) IH); \ - for (uint32_t iic = 0; iic < IC; iic++) { \ - DST_CTYPE * out_run = dst_patch + iic * (KH * KW) + ikh * KW; \ - if (!okh) { \ - SPLAT_FN(out_run, 0.0f, KW); \ - continue; \ - } \ - const float * vrow = srcb + ((uint64_t) (iic * KH + ikh)) * Wb; /* col win0 at idx 0*/ \ - if (d0 == 1) { \ - /* contiguous run within the staged window: [lo,hi) in-bounds, tails zero pad */ \ - const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ - int32_t hi = (int32_t) IW - iiw0; \ - if (hi > (int32_t) KW) { \ - hi = (int32_t) KW; \ - } \ - if (hi <= lo) { \ - SPLAT_FN(out_run, 0.0f, KW); \ - } else { \ - if (lo > 0) { \ - SPLAT_FN(out_run, 0.0f, (uint32_t) lo); \ - } \ - COPY_FN((uint8_t *) (out_run + lo), (const uint8_t *) (vrow + (iiw0 + lo - win0)), \ - (uint32_t) (hi - lo)); \ - if (hi < (int32_t) KW) { \ - SPLAT_FN(out_run + hi, 0.0f, (KW - (uint32_t) hi)); \ - } \ - } \ - continue; \ - } \ - for (uint32_t ikw = 0; ikw < KW; ikw++) { \ - const int32_t iiw = iiw0 + (int32_t) ikw * d0; \ - out_run[ikw] = \ - (iiw < 0 || iiw >= (int32_t) IW) ? (DST_CTYPE) 0.0f : (DST_CTYPE) vrow[iiw - win0]; \ - } \ - } \ - } \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); \ - const dma_addr_t ddr = dst_data + (size_t) ((in * OH + ioh) * OW + c0) * patch_stride * (DST_ELEM); \ - dma_queue_push(dmaq, dma_make_data(ddr, dstb), \ - nb * patch_stride * (DST_ELEM), nb * patch_stride * (DST_ELEM), \ - nb * patch_stride * (DST_ELEM), 1); \ - dma_queue_flush(dmaq); \ - } \ - } -IM2COL_BLOCKED_DMA_BODY(im2col_blocked_dma_thread, __fp16, hvx_copy_f16_f32_uu, hvx_splat_f16_u, sizeof(__fp16), "blk-dma-f16") -IM2COL_BLOCKED_DMA_BODY(im2col_blocked_dma_f32_thread, float, hvx_copy_f32_uu, hvx_splat_f32_u, sizeof(float), "blk-dma-f32") - -// Exact-tiling patch-embed DMA fast path (s0==KW, p0=0, d0=1; and 2D s1==KH, -// p1=0, d1=1). Intentionally reads no stride/pad/dilation params so the inner -// copy stays tight and fully hoisted - do NOT graft the general gather in here. -#define IM2COL_PATCHEMBED_DMA_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN, DST_ELEM, TAG) \ - static void FNAME(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ - struct htp_ops_context * octx = ictx->octx; \ - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ - const struct htp_tensor * restrict src1 = octx->src[1]; \ - const struct htp_tensor * restrict dst = octx->dst; \ - const int32_t is_2D = octx->op_params[6] == 1; \ - const uint32_t N = is_2D ? src1->ne[3] : src1->ne[2]; \ - const uint32_t IC = is_2D ? src1->ne[2] : src1->ne[1]; \ - const uint32_t IH = is_2D ? src1->ne[1] : 1; \ - const uint32_t IW = src1->ne[0]; \ - const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; \ - const uint32_t KW = octx->src[0]->ne[0]; \ - const uint32_t OH = is_2D ? dst->ne[2] : 1; \ - const uint32_t OW = dst->ne[1]; \ - const uint32_t patch_stride = IC * KH * KW; \ - const dma_addr_t src_data = src1->data; \ - const dma_addr_t dst_data = dst->data; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; \ - uint8_t * dst_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; \ - float * srcb = (float *) src_base; \ - DST_CTYPE * dstb = (DST_CTYPE *) dst_base; \ - const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; \ - const uint32_t per_thread = ictx->pe_rows_per_thread; \ - const uint32_t row_start = ictx->pe_row_base + per_thread * ith; \ - const uint32_t row_end = MIN(row_start + per_thread, row_end_max); \ - if (row_start >= row_end) \ - return; \ - for (uint32_t r = row_start; r < row_end; r++) { \ - const uint32_t in = r / OH; \ - const uint32_t ioh = r % OH; \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; \ - int ok = (iih >= 0 && iih < (int32_t) IH); \ - for (uint32_t iic = 0; iic < IC; iic++) { \ - float * vdst = srcb + (size_t) (iic * KH + ikh) * IW; \ - const dma_addr_t vsrc = ok \ - ? (src_data + (size_t) ((in * IC + iic) * IH + iih) * IW * sizeof(float)) \ - : src_data; \ - dma_queue_push(dma_q, dma_make_data(vdst, vsrc), \ - IW * sizeof(float), IW * sizeof(float), IW * sizeof(float), ok ? 1 : 0); \ - } \ - } \ - for (uint32_t i = 0; i < IC * KH; i++) \ - dma_queue_pop(dma_q); \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); \ - for (uint32_t iow = 0; iow < OW; iow++) { \ - DST_CTYPE * dst_patch = dstb + (uint64_t) iow * patch_stride; \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; \ - for (uint32_t iic = 0; iic < IC; iic++) { \ - DST_CTYPE * out_run = dst_patch + iic * (KH * KW) + ikh * KW; \ - if (iih < 0 || iih >= (int32_t) IH) { \ - SPLAT_FN(out_run, 0.0f, KW); \ - continue; \ - } \ - const float * src_run = srcb + ((uint64_t) (iic * KH + ikh)) * IW + (uint64_t) iow * KW; \ - COPY_FN((uint8_t *) out_run, (const uint8_t *) src_run, KW); \ - } \ - } \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); \ - const dma_addr_t ddr_row = dst_data + (size_t) (in * OH + ioh) * OW * patch_stride * (DST_ELEM); \ - dma_queue_push(dma_q, dma_make_data(ddr_row, dstb), \ - OW * patch_stride * (DST_ELEM), OW * patch_stride * (DST_ELEM), \ - OW * patch_stride * (DST_ELEM), 1); \ - dma_queue_flush(dma_q); \ - } \ - } - -IM2COL_PATCHEMBED_DMA_BODY(im2col_patchembed_dma_thread, __fp16, hvx_copy_f16_f32_uu, hvx_splat_f16_u, sizeof(__fp16), "pe-dma-f16") -IM2COL_PATCHEMBED_DMA_BODY(im2col_patchembed_dma_f32_thread, float, hvx_copy_f32_uu, hvx_splat_f32_u, sizeof(float), "pe-dma-f32") - -static bool im2col_use_patchembed_dma(const struct htp_ops_context * octx) { +static void im2col_blocked_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q, + float * srcb, const struct im2col_cursor * c) { + const uint32_t in = c->in, ioh = c->ioh, c0 = c->c0; + const struct htp_ops_context * octx = ictx->octx; + const struct htp_tensor * src1 = octx->src[1]; const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; - const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; - const int is_2D = octx->op_params[6] == 1; - if (octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_F32) { - return false; + const int32_t d1 = octx->op_params[5]; + const bool is_2D = (octx->op_params[6] == 1); + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; + const uint32_t KH = ictx->KH; + const uint32_t Wb = ictx->pe_wb; + const dma_addr_t src_data = src1->data; + const int32_t win0 = (int32_t) c0 * s0 - p0; + const size_t src_channel_stride = is_2D ? src1->nb[2] : src1->nb[1]; + const size_t batch_offset = is_2D ? (size_t) in * src1->nb[3] : (size_t) in * src1->nb[2]; + + for (uint32_t ikh = 0; ikh < KH; ikh++) { + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; + if (iih < 0 || iih >= (int32_t) IH) { + dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0); + continue; + } + const int32_t lo = win0 < 0 ? -win0 : 0; + int32_t hi = (int32_t) IW - win0; + if (hi > (int32_t) Wb) { + hi = (int32_t) Wb; + } + if (hi <= lo) { + dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0); + continue; + } + const uint32_t cpw = (uint32_t) (hi - lo); + float * vdst = srcb + (size_t) ikh * Wb + lo; + const dma_addr_t vsrc = src_data + batch_offset + (size_t) iih * src1->nb[1] + (size_t) (win0 + lo) * src1->nb[0]; + dma_queue_push(dma_q, dma_make_data(vdst, vsrc), + (size_t) KH * Wb * sizeof(float), src_channel_stride, + cpw * sizeof(float), IC); + } +} + +#define IM2COL_BLOCKED_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN) \ +static void FNAME( \ + const struct htp_im2col_context * ictx, \ + void * restrict dstb_raw, const float * restrict srcb, \ + float * restrict scratch, \ + uint32_t nb, uint32_t c0, uint32_t ioh) { \ + DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \ + const struct htp_ops_context * octx = ictx->octx; \ + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; \ + const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; \ + const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; \ + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; \ + const uint32_t KH = ictx->KH, KW = ictx->KW; \ + const uint32_t Wb = ictx->pe_wb; \ + const uint32_t patch_stride = ictx->patch_stride; \ + const int32_t win0 = (int32_t) c0 * s0 - p0; \ + \ + for (uint32_t j = 0; j < nb; j++) { \ + const uint32_t iow = c0 + j; \ + DST_CTYPE * dst_patch = dstb + (size_t) j * patch_stride; \ + const int32_t iiw0 = (int32_t) iow * s0 - p0; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ + const int okh = (iih >= 0 && iih < (int32_t) IH); \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + DST_CTYPE * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW; \ + if (!okh) { \ + SPLAT_FN(out_run, 0, KW); \ + continue; \ + } \ + const float * vrow = srcb + (size_t) (iic * KH + ikh) * Wb; \ + if (d0 == 1) { \ + const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ + int32_t hi = (int32_t) IW - iiw0; \ + if (hi > (int32_t) KW) { \ + hi = (int32_t) KW; \ + } \ + if (hi <= lo) { \ + SPLAT_FN(out_run, 0, KW); \ + } else { \ + if (lo > 0) { \ + SPLAT_FN(out_run, 0, (uint32_t) lo); \ + } \ + COPY_FN((uint8_t *) (out_run + lo), \ + (const uint8_t *) (vrow + (iiw0 + lo - win0)), \ + (uint32_t) (hi - lo)); \ + if (hi < (int32_t) KW) { \ + SPLAT_FN(out_run + hi, 0, (uint32_t) (KW - hi)); \ + } \ + } \ + continue; \ + } \ + for (uint32_t ikw = 0; ikw < KW; ikw += 64) { \ + const uint32_t chunk = MIN(KW - ikw, 64); \ + for (uint32_t c = 0; c < chunk; c++) { \ + const int32_t iiw = iiw0 + (int32_t) (ikw + c) * d0; \ + scratch[c] = (iiw < 0 || iiw >= (int32_t) IW) ? 0.0f : vrow[iiw - win0]; \ + } \ + COPY_FN((uint8_t *) (out_run + ikw), (const uint8_t *) scratch, chunk); \ + } \ + } \ + } \ + } \ +} + +IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u) +IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u) + +// Offsets of one output patch in the staged window, for KH == 1: lane k reads channel k / KW at tap k % KW. +// Lanes past the patch read the first element and are not stored. +// The first vector is built lane by lane; each next one is 32 lanes further: the tap advances by 32 % KW +// and the channel by 32 / KW, plus one more when the tap wraps past KW. +static __attribute__((noinline)) void im2col_gather_table_build(const struct htp_im2col_context * ictx, + HVX_Vector * restrict table) { + const int32_t d0 = ictx->octx->op_params[4]; + const uint32_t KW = ictx->KW, Wb = ictx->pe_wb; + + int32_t tap0[32] __attribute__((aligned(128))); + int32_t off0[32] __attribute__((aligned(128))); + uint32_t iic = 0, ikw = 0; + for (uint32_t k = 0; k < 32; k++) { + tap0[k] = (int32_t) ikw; + off0[k] = (int32_t) ((iic * Wb + ikw * d0) * sizeof(float)); + if (++ikw == KW) { + ikw = 0; + iic++; + } } - const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; - const uint32_t KW = octx->src[0]->ne[0]; - if (s0 != (int32_t) KW) { - return false; // non-overlapping (width) + // (iic, ikw) is now (32 / KW, 32 % KW) + const HVX_Vector v_kw = Q6_V_vsplat_R(KW); + const HVX_Vector v_kw_m1 = Q6_V_vsplat_R(KW - 1); + const HVX_Vector v_tap_step = Q6_V_vsplat_R(ikw); + const HVX_Vector v_off_step = Q6_V_vsplat_R((iic * Wb + ikw * d0) * sizeof(float)); + const HVX_Vector v_off_wrap = Q6_V_vsplat_R((Wb - KW * d0) * sizeof(float)); + const HVX_Vector v_32 = Q6_V_vsplat_R(32); + const HVX_Vector v_n = Q6_V_vsplat_R(ictx->patch_stride); + + HVX_Vector lane = hvx_vec_gather_offsets_w(1); + HVX_Vector tap = *(HVX_Vector *) tap0; + HVX_Vector off = *(HVX_Vector *) off0; + for (uint32_t g = 0; g < ictx->pe_gather_nvec; g++) { + table[g] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(v_n, lane), off, Q6_V_vzero()); + + lane = Q6_Vw_vadd_VwVw(lane, v_32); + tap = Q6_Vw_vadd_VwVw(tap, v_tap_step); + off = Q6_Vw_vadd_VwVw(off, v_off_step); + + const HVX_VectorPred wrap = Q6_Q_vcmp_gt_VwVw(tap, v_kw_m1); + tap = Q6_V_vmux_QVV(wrap, Q6_Vw_vsub_VwVw(tap, v_kw), tap); + off = Q6_V_vmux_QVV(wrap, Q6_Vw_vadd_VwVw(off, v_off_wrap), off); + } +} + +// KH == 1: each output row is one patch, gathered from the staged [IC][Wb] window with a fixed offset +// table shifted by s0 per row, then converted with a single copy. +// scratch holds the offset table followed by the gathered row (both VTCM). +#define IM2COL_GATHER_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN) \ +static void FNAME( \ + const struct htp_im2col_context * ictx, \ + void * restrict dstb_raw, const float * restrict srcb_c, \ + float * restrict scratch, \ + uint32_t nb, uint32_t c0, uint32_t ioh) { \ + DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \ + float * restrict srcb = (float *) srcb_c; \ + const int32_t s0 = ictx->octx->op_params[0], p0 = ictx->octx->op_params[2]; \ + const int32_t s1 = ictx->octx->op_params[1], p1 = ictx->octx->op_params[3]; \ + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW, Wb = ictx->pe_wb; \ + const uint32_t patch_stride = ictx->patch_stride; \ + const uint32_t ng = ictx->pe_gather_nvec; \ + const int32_t win0 = (int32_t) c0 * s0 - p0; \ + const int32_t iih = (int32_t) ioh * s1 - p1; \ + \ + /* the DMA stage fills only the columns inside [0, IW): zero the rest of the window */ \ + const int32_t lo = win0 < 0 ? -win0 : 0; \ + const int32_t hi = MIN((int32_t) Wb, (int32_t) IW - win0); \ + if (iih < 0 || iih >= (int32_t) IH || hi <= lo) { \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + hvx_splat_f32_u(srcb + (size_t) iic * Wb, 0, Wb); \ + } \ + } else if (lo > 0 || hi < (int32_t) Wb) { \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + float * row = srcb + (size_t) iic * Wb; \ + if (lo > 0) { \ + hvx_splat_f32_u(row, 0, (uint32_t) lo); \ + } \ + if (hi < (int32_t) Wb) { \ + hvx_splat_f32_u(row + hi, 0, Wb - (uint32_t) hi); \ + } \ + } \ + } \ + \ + const HVX_Vector * table = (const HVX_Vector *) scratch; \ + float * row = scratch + (size_t) ng * 32; \ + const uint32_t region = IC * Wb * sizeof(float); \ + \ + for (uint32_t j = 0; j < nb; j++) { \ + const uint32_t shift = j * (uint32_t) s0 * sizeof(float); \ + const size_t base = (size_t) ((const uint8_t *) srcb + shift); \ + const uint32_t mu = region - shift - 1; \ + for (uint32_t g = 0; g < ng; g++) { \ + Q6_vgather_ARMVw((HVX_Vector *) (row + g * 32), base, mu, table[g]); \ + } \ + hvx_gather_sync(row + (ng - 1) * 32); \ + DST_CTYPE * out = dstb + (size_t) j * patch_stride; \ + COPY_FN((uint8_t *) out, (const uint8_t *) row, patch_stride); \ + } \ +} + +IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f16, uint16_t, hvx_copy_f16_f32_uu) +IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f32, float, hvx_copy_f32_uu) + +static void im2col_blocked_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; + struct htp_ops_context * octx = ictx->octx; + struct htp_thread_trace * restrict tr = th->trace; + const uint32_t OW = ictx->OW; + const uint32_t KH = ictx->KH; + const uint32_t owb = ictx->pe_owb; + const uint32_t nbpr = ictx->pe_nbpr; + const uint32_t patch_stride = ictx->patch_stride; + const uint32_t dst_elem = ictx->dst_elem; + im2col_blocked_compute_fn_t compute = (im2col_blocked_compute_fn_t) ictx->compute; + const dma_addr_t dst_data = octx->dst->data; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; + float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; + void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; + float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes); + // work is split over output-col blocks, so a single output row (1D) still uses every thread + const uint32_t blk_end_max = ictx->pe_nrows * nbpr; + const uint32_t blk_start = ictx->pe_blocks_per_hvx * ith; + const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_hvx, blk_end_max); + if (blk_start >= blk_end) { + return; } - if (p0 != 0) { - return false; // no padding (width) + const uint32_t total_blocks = blk_end - blk_start; + struct im2col_cursor cur = { + .r = ictx->thread_in[ith] * ictx->OH + ictx->thread_ioh[ith], + .in = ictx->thread_in[ith], + .iod = 0, + .ioh = ictx->thread_ioh[ith], + .c0 = ictx->thread_c0[ith], + }; + if (KH == 1) { + im2col_gather_table_build(ictx, (HVX_Vector *) scratch); } - if (d0 != 1) { - return false; // no dilation (width) + struct im2col_cursor stage = cur; + for (uint32_t i = 0; i < total_blocks && i < 2; i++) { + dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0); + im2col_blocked_dma_stage(ictx, dma_q, srcb2[i], &stage); + im2col_step_block(&stage, ictx); } - if (is_2D) { - if (s1 != (int32_t) KH) { - return false; // non-overlapping (height) + for (uint32_t bi = 0; bi < total_blocks; bi++) { + const uint32_t buf = bi & 1u; + float * srcb = srcb2[buf]; + void * dstb = dstb2[buf]; + const uint32_t nb = MIN(owb, OW - cur.c0); + dma_queue_pop(dma_q); + for (uint32_t k = 0; k < KH; k++) { + dma_queue_pop(dma_q); } - if (p1 != 0) { - return false; // no padding (height) + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + compute(ictx, dstb, srcb, scratch, nb, cur.c0, cur.ioh); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + const dma_addr_t ddr = dst_data + (size_t) (cur.r * OW + cur.c0) * patch_stride * dst_elem; + dma_queue_push(dma_q, dma_make_data(ddr, dstb), + nb * patch_stride * dst_elem, nb * patch_stride * dst_elem, + nb * patch_stride * dst_elem, 1); + if (bi + 2 < total_blocks) { + im2col_blocked_dma_stage(ictx, dma_q, srcb, &stage); + im2col_step_block(&stage, ictx); } - if (d1 != 1) { - return false; // no dilation (height) + im2col_step_block(&cur, ictx); + } + dma_queue_flush(dma_q); +} + +static void im2col_patchembed_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q, + float * srcb, const struct im2col_cursor * c) { + const struct htp_ops_context * octx = ictx->octx; + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; + const uint32_t KH = ictx->KH; + const uint32_t in = c->in, ioh = c->ioh; + const size_t src_channel_stride = (size_t) IH * IW * sizeof(float); + const size_t vtcm_channel_stride = (size_t) KH * IW * sizeof(float); + const dma_addr_t src_data = octx->src[1]->data; + const dma_addr_t vsrc = src_data + (size_t) (((in * IC) * IH + ioh * KH) * IW) * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(srcb, vsrc), + vtcm_channel_stride, src_channel_stride, + vtcm_channel_stride, IC); +} + +static void im2col_patchembed_compute_f16( + const struct htp_im2col_context * ictx, + void * restrict dstb_raw, const float * restrict srcb, uint32_t ioh) { + uint16_t * restrict dstb = (uint16_t *) dstb_raw; + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; + const uint32_t KH = ictx->KH, KW = ictx->KW; + const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride; + + if (KW == 16 && (KH % 4 == 0)) { + const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64); + for (uint32_t iow = 0; iow < OW; iow++) { + uint16_t * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t iic = 0; iic < IC; iic++) { + for (uint32_t ikh = 0; ikh < KH; ikh += 4) { + const int32_t iih0 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 0); + const int32_t iih1 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 1); + const int32_t iih2 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 2); + const int32_t iih3 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 3); + + const float * s0 = srcb + (size_t) (iic * KH + ikh + 0) * IW + (size_t) iow * 16; + const float * s1 = srcb + (size_t) (iic * KH + ikh + 1) * IW + (size_t) iow * 16; + const float * s2 = srcb + (size_t) (iic * KH + ikh + 2) * IW + (size_t) iow * 16; + const float * s3 = srcb + (size_t) (iic * KH + ikh + 3) * IW + (size_t) iow * 16; + + const HVX_Vector v0 = (iih0 >= 0 && iih0 < (int32_t) IH) ? *((const HVX_UVector *) s0) : Q6_V_vzero(); + const HVX_Vector v1 = (iih1 >= 0 && iih1 < (int32_t) IH) ? *((const HVX_UVector *) s1) : Q6_V_vzero(); + const HVX_Vector v2 = (iih2 >= 0 && iih2 < (int32_t) IH) ? *((const HVX_UVector *) s2) : Q6_V_vzero(); + const HVX_Vector v3 = (iih3 >= 0 && iih3 < (int32_t) IH) ? *((const HVX_UVector *) s3) : Q6_V_vzero(); + + const HVX_Vector w01 = Q6_V_vmux_QVV(q_lo, v0, Q6_V_vror_VR(v1, 64)); + const HVX_Vector w23 = Q6_V_vmux_QVV(q_lo, v2, Q6_V_vror_VR(v3, 64)); + + *((HVX_Vector *) (dst_patch + (size_t) (iic * KH + ikh) * 16)) = hvx_vec_f32_to_f16(w01, w23); + } + } + } + return; + } + + for (uint32_t iow = 0; iow < OW; iow++) { + uint16_t * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t ikh = 0; ikh < KH; ikh++) { + const int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; + for (uint32_t iic = 0; iic < IC; iic++) { + uint16_t * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW; + if (iih < 0 || iih >= (int32_t) IH) { + hvx_splat_u16_u(out_run, 0, KW); + continue; + } + const float * src_run = srcb + (size_t) (iic * KH + ikh) * IW + (size_t) iow * KW; + hvx_copy_f16_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW); + } } } - return true; } -// Sizes the per-thread 2x(src,dst) VTCM ping-pong for the patch-embed DMA path. -// Returns false if it doesn't fit the VTCM budget (caller falls back). -static bool im2col_patchembed_dma_fits(struct htp_ops_context * octx, - struct htp_im2col_context * ictx, - uint32_t n_threads) { - const int32_t is_2D = octx->op_params[6] == 1; - const uint32_t IC = is_2D ? octx->src[1]->ne[2] : octx->src[1]->ne[1]; - const uint32_t IW = octx->src[1]->ne[0]; - const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; - const uint32_t KW = octx->src[0]->ne[0]; - const uint32_t OW = octx->dst->ne[1]; - const uint32_t patch_stride = IC * KH * KW; - - ictx->pe_src_row_bytes = hex_round_up(IC * KH * IW * sizeof(float), 256); - const uint32_t dst_elem = (octx->dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); - ictx->pe_dst_row_bytes = hex_round_up(OW * patch_stride * dst_elem, 256); - - // 2 src + 2 dst buffers per thread (ping-pong), src region first then dst. - struct htp_im2col_vtcm_layout L; - htp_im2col_vtcm_layout_build(&L, ictx->pe_src_row_bytes, ictx->pe_dst_row_bytes, n_threads); - if (L.total_bytes > octx->ctx->vtcm_size) { - return false; - } - - uint8_t * const base = octx->ctx->vtcm_base; - ictx->pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src); - ictx->pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - ictx->pe_src_size_per_thread = (uint32_t) L.src_bytes_per_thread; - ictx->pe_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; - return true; +static void im2col_patchembed_compute_f32( + const struct htp_im2col_context * ictx, + void * restrict dstb_raw, const float * restrict srcb, uint32_t ioh) { + float * restrict dstb = (float *) dstb_raw; + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; + const uint32_t KH = ictx->KH, KW = ictx->KW; + const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride; + + if (KW == 16 && (KH % 2 == 0)) { + const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64); + for (uint32_t iow = 0; iow < OW; iow++) { + float * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t iic = 0; iic < IC; iic++) { + for (uint32_t ikh = 0; ikh < KH; ikh += 2) { + const int32_t iih0 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 0); + const int32_t iih1 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 1); + + const float * s0 = srcb + (size_t) (iic * KH + ikh + 0) * IW + (size_t) iow * 16; + const float * s1 = srcb + (size_t) (iic * KH + ikh + 1) * IW + (size_t) iow * 16; + + const HVX_Vector v0 = (iih0 >= 0 && iih0 < (int32_t) IH) ? *((const HVX_UVector *) s0) : Q6_V_vzero(); + const HVX_Vector v1 = (iih1 >= 0 && iih1 < (int32_t) IH) ? *((const HVX_UVector *) s1) : Q6_V_vzero(); + + const HVX_Vector w01 = Q6_V_vmux_QVV(q_lo, v0, Q6_V_vror_VR(v1, 64)); + + *((HVX_Vector *) (dst_patch + (size_t) (iic * KH + ikh) * 16)) = w01; + } + } + } + return; + } + + for (uint32_t iow = 0; iow < OW; iow++) { + float * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t ikh = 0; ikh < KH; ikh++) { + const int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; + for (uint32_t iic = 0; iic < IC; iic++) { + float * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW; + if (iih < 0 || iih >= (int32_t) IH) { + hvx_splat_f32_u(out_run, 0, KW); + continue; + } + const float * src_run = srcb + (size_t) (iic * KH + ikh) * IW + (size_t) iow * KW; + hvx_copy_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW); + } + } + } +} + +static void im2col_patchembed_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; + struct htp_ops_context * octx = ictx->octx; + struct htp_thread_trace * restrict tr = th->trace; + const uint32_t OW = ictx->OW; + const uint32_t patch_stride = ictx->patch_stride; + const uint32_t dst_elem = ictx->dst_elem; + im2col_patchembed_compute_fn_t compute = (im2col_patchembed_compute_fn_t) ictx->compute; + const dma_addr_t dst_data = octx->dst->data; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; + float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; + void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); + if (row_start >= row_end) { + return; + } + const uint32_t total_rows = row_end - row_start; + if (total_rows == 0) { + return; + } + struct im2col_cursor cur = { + .r = row_start, + .in = ictx->thread_in[ith], + .iod = 0, + .ioh = ictx->thread_ioh[ith], + .c0 = 0, + }; + struct im2col_cursor stage = cur; + for (uint32_t i = 0; i < total_rows && i < 2; i++) { + dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0); + im2col_patchembed_dma_stage(ictx, dma_q, srcb2[i], &stage); + im2col_step_row(&stage, ictx); + } + for (uint32_t ri = 0; ri < total_rows; ri++) { + const uint32_t buf = ri & 1u; + float * srcb = srcb2[buf]; + void * dstb = dstb2[buf]; + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + compute(ictx, dstb, srcb, cur.ioh); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + const dma_addr_t ddr_row = dst_data + (size_t) cur.r * OW * patch_stride * dst_elem; + dma_queue_push(dma_q, dma_make_data(ddr_row, dstb), + OW * patch_stride * dst_elem, OW * patch_stride * dst_elem, + OW * patch_stride * dst_elem, 1); + if (ri + 2 < total_rows) { + im2col_patchembed_dma_stage(ictx, dma_q, srcb, &stage); + im2col_step_row(&stage, ictx); + } + im2col_step_row(&cur, ictx); + } + dma_queue_flush(dma_q); } -// Sizes a per-thread 2x(src,dst) VTCM ping-pong for the blocked general kernel. -// Stages Wb=(owb-1)*s0+(KW-1)*d0+1 source cols per (iic,ikh) row and owb patches -// of dst. Picks the largest owb that fits; returns false if even owb=1 does not. -static bool im2col_blocked_dma_fits(struct htp_ops_context * octx, - struct htp_im2col_context * ictx, - uint32_t n_threads) { - const int32_t is_2D = octx->op_params[6] == 1; - const int32_t s0 = octx->op_params[0]; - const int32_t d0 = octx->op_params[4]; - const uint32_t IC = is_2D ? octx->src[1]->ne[2] : octx->src[1]->ne[1]; - const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; - const uint32_t KW = octx->src[0]->ne[0]; - const uint32_t OW = octx->dst->ne[1]; - const uint32_t patch_stride = IC * KH * KW; - const uint32_t dst_elem = (octx->dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); - - for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) { - const uint32_t Wb = (owb - 1) * (uint32_t) s0 + (KW - 1) * (uint32_t) d0 + 1; - const uint32_t src_row_bytes = hex_round_up(IC * KH * Wb * sizeof(float), 256); - const uint32_t dst_row_bytes = hex_round_up(owb * patch_stride * dst_elem, 256); - struct htp_im2col_vtcm_layout L; - htp_im2col_vtcm_layout_build(&L, src_row_bytes, dst_row_bytes, n_threads); - if (L.total_bytes <= octx->ctx->vtcm_size) { - uint8_t * const base = octx->ctx->vtcm_base; - ictx->pe_owb = owb; - ictx->pe_wb = Wb; - ictx->pe_src_row_bytes = src_row_bytes; - ictx->pe_dst_row_bytes = dst_row_bytes; - ictx->pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src); - ictx->pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - ictx->pe_src_size_per_thread = (uint32_t) L.src_bytes_per_thread; - ictx->pe_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; - return true; +static inline void im2col_3d_step_row(struct im2col_cursor * c, const struct htp_im2col_context * ictx) { + c->r++; + c->ioh++; + if (c->ioh >= ictx->OH) { + c->ioh = 0; + c->iod++; + if (c->iod >= ictx->OD) { + c->iod = 0; + c->in++; } - if (owb == 1) break; // avoid unsigned underflow } - return false; } -int op_im2col(struct htp_ops_context * octx) { +static inline void im2col_3d_step_block(struct im2col_cursor * c, const struct htp_im2col_context * ictx) { + c->c0 += ictx->pe_owb; + if (c->c0 >= ictx->OW) { + c->c0 = 0; + c->r++; + c->ioh++; + if (c->ioh >= ictx->OH) { + c->ioh = 0; + c->iod++; + if (c->iod >= ictx->OD) { + c->iod = 0; + c->in++; + } + } + } +} + +static void im2col_3d_blocked_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q, + float * srcb, const struct im2col_cursor * c) { + const uint32_t in = c->in, iod = c->iod, ioh = c->ioh, c0 = c->c0; + const struct htp_ops_context * octx = ictx->octx; const struct htp_tensor * src1 = octx->src[1]; - const struct htp_tensor * dst = octx->dst; + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1], s2 = octx->op_params[2]; + const int32_t p0 = octx->op_params[3], p1 = octx->op_params[4], p2 = octx->op_params[5]; + const int32_t d1 = octx->op_params[7], d2 = octx->op_params[8]; + const uint32_t IC = ictx->IC; + const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW; + const uint32_t KD = ictx->KD, KH = ictx->KH; + const uint32_t Wb = ictx->pe_wb; + const int32_t win0 = (int32_t) c0 * s0 - p0; + const dma_addr_t src_data = src1->data; + const size_t src_channel_stride = src1->nb[3]; + const size_t vtcm_channel_stride = (size_t) KD * KH * Wb * sizeof(float); + const size_t batch_offset = (size_t) (in * IC) * src1->nb[3]; - if (src1->type != HTP_TYPE_F32 || (dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_F32)) { - FARF(ERROR, "im2col: only (F32 image -> F16/F32 columns) supported"); - return HTP_STATUS_NO_SUPPORT; + for (uint32_t ikd = 0; ikd < KD; ikd++) { + const int32_t iid = (int32_t) iod * s2 + (int32_t) ikd * d2 - p2; + const bool valid_d = (iid >= 0 && iid < (int32_t) ID); + for (uint32_t ikh = 0; ikh < KH; ikh++) { + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; + if (!valid_d || iih < 0 || iih >= (int32_t) IH) { + dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0); + continue; + } + const int32_t lo = win0 < 0 ? -win0 : 0; + int32_t hi = (int32_t) IW - win0; + if (hi > (int32_t) Wb) { + hi = (int32_t) Wb; + } + if (hi <= lo) { + dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0); + continue; + } + float * vdst = srcb + (size_t) (ikd * KH + ikh) * Wb + lo; + const dma_addr_t vsrc = src_data + batch_offset + + (size_t) iid * src1->nb[2] + + (size_t) iih * src1->nb[1] + + (size_t) (win0 + lo) * src1->nb[0]; + dma_queue_push(dma_q, dma_make_data(vdst, vsrc), + vtcm_channel_stride, src_channel_stride, + (size_t) (hi - lo) * sizeof(float), IC); + } } +} - const int32_t is_2D = octx->op_params[6] == 1; - const uint32_t N = is_2D ? src1->ne[3] : src1->ne[2]; - const uint32_t OH = is_2D ? dst->ne[2] : 1; - const uint32_t OW = dst->ne[1]; - const uint32_t total_patches = N * OH * OW; - const uint32_t total_rows = N * OH; +#define IM2COL_3D_BLOCKED_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN) \ +static void FNAME(const struct htp_im2col_context * ictx, \ + void * restrict dstb_raw, const float * restrict srcb, \ + float * restrict scratch, \ + uint32_t nb, uint32_t c0, uint32_t iod, uint32_t ioh) { \ + DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \ + const struct htp_ops_context * octx = ictx->octx; \ + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1], s2 = octx->op_params[2]; \ + const int32_t p0 = octx->op_params[3], p1 = octx->op_params[4], p2 = octx->op_params[5]; \ + const int32_t d0 = octx->op_params[6], d1 = octx->op_params[7], d2 = octx->op_params[8]; \ + const uint32_t IC = ictx->IC; \ + const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW; \ + const uint32_t KD = ictx->KD, KH = ictx->KH, KW = ictx->KW; \ + const uint32_t patch_stride = ictx->patch_stride; \ + const uint32_t Wb = ictx->pe_wb; \ + const int32_t win0 = (int32_t) c0 * s0 - p0; \ + for (uint32_t j = 0; j < nb; j++) { \ + const uint32_t iow = c0 + j; \ + const int32_t iiw0 = (int32_t) iow * s0 - p0; \ + DST_CTYPE * dst_patch = dstb + (size_t) j * patch_stride; \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + for (uint32_t ikd = 0; ikd < KD; ikd++) { \ + const int32_t iid = (int32_t) iod * s2 + (int32_t) ikd * d2 - p2; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ + DST_CTYPE * out_run = dst_patch + \ + (size_t) (iic * KD * KH + ikd * KH + ikh) * KW; \ + if (iid < 0 || iid >= (int32_t) ID || iih < 0 || iih >= (int32_t) IH) { \ + SPLAT_FN(out_run, 0, KW); \ + continue; \ + } \ + const float * vrow = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * Wb; \ + if (d0 == 1) { \ + const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ + int32_t hi = (int32_t) IW - iiw0; \ + if (hi > (int32_t) KW) { \ + hi = (int32_t) KW; \ + } \ + if (hi <= lo) { \ + SPLAT_FN(out_run, 0, KW); \ + } else { \ + if (lo > 0) { \ + SPLAT_FN(out_run, 0, (uint32_t) lo); \ + } \ + COPY_FN((uint8_t *) (out_run + lo), \ + (const uint8_t *) (vrow + (iiw0 + lo - win0)), \ + (uint32_t) (hi - lo)); \ + if (hi < (int32_t) KW) { \ + SPLAT_FN(out_run + hi, 0, (uint32_t) (KW - hi)); \ + } \ + } \ + continue; \ + } \ + for (uint32_t ikw = 0; ikw < KW; ikw += 64) { \ + const uint32_t chunk = MIN(KW - ikw, 64); \ + for (uint32_t c = 0; c < chunk; c++) { \ + const int32_t iiw = iiw0 + (int32_t) (ikw + c) * d0; \ + scratch[c] = (iiw < 0 || iiw >= (int32_t) IW) ? 0.0f : vrow[iiw - win0]; \ + } \ + COPY_FN((uint8_t *) (out_run + ikw), (const uint8_t *) scratch, chunk); \ + } \ + } \ + } \ + } \ + } \ +} - uint32_t patch_base = 0; - uint32_t npatches = total_patches; - if (octx->ctx->mdev.count > 1) { - const uint32_t patch_size = dst->nb[1]; - const uint32_t patches_per_chunk = - (patch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(patch_size, HEX_L2_LINE_SIZE)) : 1; - const struct htp_tensor_mdev_range range = - htp_tensor_mdev_partition(total_patches, htp_tensor_mdev_data_aligned(dst) ? patches_per_chunk : 0, - octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); - patch_base = range.start; - npatches = range.count; +IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u) +IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u) + +static void im2col_3d_blocked_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; + struct htp_ops_context * octx = ictx->octx; + struct htp_thread_trace * restrict tr = th->trace; + const struct htp_tensor * restrict dst = octx->dst; + const uint32_t OW = ictx->OW; + const uint32_t KD = ictx->KD, KH = ictx->KH; + const uint32_t kd_kh = KD * KH; + const uint32_t patch_stride = ictx->patch_stride; + const uint32_t owb = ictx->pe_owb; + const uint32_t nbpr = ictx->pe_nbpr; + const uint32_t dst_elem = ictx->dst_elem; + const dma_addr_t dst_data = dst->data; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; + float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; + void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; + float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes); + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); + if (row_start >= row_end) { + return; + } + const uint32_t nrows_local = row_end - row_start; + const uint32_t total_blocks = nrows_local * nbpr; + if (total_blocks == 0) { + return; + } + struct im2col_cursor cur = { + .r = row_start, + .in = ictx->thread_in[ith], + .iod = ictx->thread_iod[ith], + .ioh = ictx->thread_ioh[ith], + .c0 = 0, + }; + struct im2col_cursor stage = cur; + for (uint32_t i = 0; i < total_blocks && i < 2; i++) { + dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0); + im2col_3d_blocked_dma_stage(ictx, dma_q, srcb2[i], &stage); + im2col_3d_step_block(&stage, ictx); + } + + im2col_3d_blocked_compute_fn_t compute = (im2col_3d_blocked_compute_fn_t) ictx->compute; + for (uint32_t bi = 0; bi < total_blocks; bi++) { + const uint32_t buf = bi & 1u; + float * srcb = srcb2[buf]; + void * dstb = dstb2[buf]; + const uint32_t nb = MIN(owb, OW - cur.c0); + dma_queue_pop(dma_q); + for (uint32_t p = 0; p < kd_kh; p++) { + dma_queue_pop(dma_q); + } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + compute(ictx, dstb, srcb, scratch, nb, cur.c0, cur.iod, cur.ioh); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + const dma_addr_t ddr = dst_data + (size_t) (cur.r * OW + cur.c0) * patch_stride * dst_elem; + dma_queue_push(dma_q, dma_make_data(ddr, dstb), + nb * patch_stride * dst_elem, nb * patch_stride * dst_elem, + nb * patch_stride * dst_elem, 1); + if (bi + 2 < total_blocks) { + im2col_3d_blocked_dma_stage(ictx, dma_q, srcb, &stage); + im2col_3d_step_block(&stage, ictx); + } + im2col_3d_step_block(&cur, ictx); + } + dma_queue_flush(dma_q); +} +static void im2col_3d_patchembed_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q, + float * srcb, const struct im2col_cursor * c) { + const struct htp_ops_context * octx = ictx->octx; + const uint32_t IC = ictx->IC; + const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW; + const uint32_t KD = ictx->KD, KH = ictx->KH; + const dma_addr_t src_data = octx->src[1]->data; + const uint32_t in = c->in, iod = c->iod, ioh = c->ioh; + const size_t src_channel_stride = (size_t) ID * IH * IW * sizeof(float); + const size_t vtcm_channel_stride = (size_t) KD * KH * IW * sizeof(float); + const size_t row_size = (size_t) KH * IW * sizeof(float); + + for (uint32_t ikd = 0; ikd < KD; ikd++) { + const uint32_t iid = iod * KD + ikd; + float * vdst = srcb + (size_t) ikd * KH * IW; + const dma_addr_t vsrc = src_data + (size_t) (((in * IC) * ID + iid) * IH + ioh * KH) * IW * sizeof(float); + dma_queue_push(dma_q, dma_make_data(vdst, vsrc), + vtcm_channel_stride, src_channel_stride, row_size, IC); + } +} +static void im2col_3d_patchembed_compute_f16( + const struct htp_im2col_context * ictx, + void * restrict dstb_raw, const float * restrict srcb) { + uint16_t * restrict dstb = (uint16_t *) dstb_raw; + const uint32_t OW = ictx->OW; + const uint32_t IC = ictx->IC; + const uint32_t KD = ictx->KD; + const uint32_t KH = ictx->KH; + const uint32_t KW = ictx->KW; + const uint32_t IW = ictx->IW; + const uint32_t patch_stride = ictx->patch_stride; + + for (uint32_t iow = 0; iow < OW; iow++) { + uint16_t * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t iic = 0; iic < IC; iic++) { + for (uint32_t ikd = 0; ikd < KD; ikd++) { + for (uint32_t ikh = 0; ikh < KH; ikh++) { + uint16_t * out_run = dst_patch + (size_t) (iic * KD * KH + ikd * KH + ikh) * KW; + const float * src_run = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * IW + (size_t) iow * KW; + hvx_copy_f16_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW); + } + } + } + } +} + +static void im2col_3d_patchembed_compute_f32( + const struct htp_im2col_context * ictx, + void * restrict dstb_raw, const float * restrict srcb) { + float * restrict dstb = (float *) dstb_raw; + const uint32_t OW = ictx->OW; + const uint32_t IC = ictx->IC; + const uint32_t KD = ictx->KD; + const uint32_t KH = ictx->KH; + const uint32_t KW = ictx->KW; + const uint32_t IW = ictx->IW; + const uint32_t patch_stride = ictx->patch_stride; + + for (uint32_t iow = 0; iow < OW; iow++) { + float * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t iic = 0; iic < IC; iic++) { + for (uint32_t ikd = 0; ikd < KD; ikd++) { + for (uint32_t ikh = 0; ikh < KH; ikh++) { + float * out_run = dst_patch + (size_t) (iic * KD * KH + ikd * KH + ikh) * KW; + const float * src_run = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * IW + (size_t) iow * KW; + hvx_copy_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW); + } + } + } + } +} + +static void im2col_3d_patchembed_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; + struct htp_ops_context * octx = ictx->octx; + struct htp_thread_trace * restrict tr = th->trace; + const struct htp_tensor * restrict dst = octx->dst; + const uint32_t KD = ictx->KD; + const uint32_t OW = ictx->OW; + const uint32_t patch_stride = ictx->patch_stride; + const uint32_t dst_elem = ictx->dst_elem; + const dma_addr_t dst_data = dst->data; + dma_queue * dma_q = th->dma_q; + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); + if (row_start >= row_end) { + return; + } + + const uint32_t nrows_local = row_end - row_start; + uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; + float * srcb2[2] = { (float *) src_base, (float *) (src_base + ictx->pe_src_row_bytes) }; + void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; + struct im2col_cursor cur = { + .r = row_start, + .in = ictx->thread_in[ith], + .iod = ictx->thread_iod[ith], + .ioh = ictx->thread_ioh[ith], + .c0 = 0, + }; + struct im2col_cursor stage = cur; + for (uint32_t i = 0; i < nrows_local && i < 2; i++) { + dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0); + im2col_3d_patchembed_dma_stage(ictx, dma_q, srcb2[i], &stage); + im2col_3d_step_row(&stage, ictx); + } + + im2col_3d_patchembed_compute_fn_t compute = (im2col_3d_patchembed_compute_fn_t) ictx->compute; + for (uint32_t idx = 0; idx < nrows_local; idx++) { + const uint32_t buf = idx & 1u; + float * srcb = srcb2[buf]; + void * dstb = dstb2[buf]; + dma_queue_pop(dma_q); + for (uint32_t p = 0; p < KD; p++) { + dma_queue_pop(dma_q); + } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + compute(ictx, dstb, srcb); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + const dma_addr_t ddr_row = dst_data + (size_t) cur.r * OW * patch_stride * dst_elem; + dma_queue_push(dma_q, dma_make_data(ddr_row, dstb), + OW * patch_stride * dst_elem, OW * patch_stride * dst_elem, + OW * patch_stride * dst_elem, 1); + if (idx + 2 < nrows_local) { + im2col_3d_patchembed_dma_stage(ictx, dma_q, srcb, &stage); + im2col_3d_step_row(&stage, ictx); + } + im2col_3d_step_row(&cur, ictx); + } + dma_queue_flush(dma_q); +} + +int op_im2col(struct htp_ops_context * octx) { + const struct htp_im2col_kernel_params * kparams = (const struct htp_im2col_kernel_params *) octx->kernel_params; + if (kparams->kernel_type == HTP_IM2COL_KERNEL_UNSUPPORTED) { + return HTP_STATUS_NO_SUPPORT; } + const struct htp_tensor * dst = octx->dst; + uint32_t row_base = 0; - uint32_t nrows = total_rows; + uint32_t nrows = kparams->total_rows; if (octx->ctx->mdev.count > 1) { - const uint32_t row_size = dst->nb[2]; - const uint32_t rows_per_chunk = - (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; const struct htp_tensor_mdev_range range = - htp_tensor_mdev_partition(total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, + htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); row_base = range.start; nrows = range.count; } - if (npatches == 0 && nrows == 0) { + if (nrows == 0) { return HTP_STATUS_OK; } - const uint32_t n_threads = MIN(octx->n_threads, MAX(npatches, 1)); + // the blocked path splits work over output-col blocks, so a single output row (1D) can use every thread + const bool blocked = kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA; + const uint32_t nwork = blocked ? nrows * kparams->nbpr : nrows; + if (!htp_ops_context_set_n_hvx(octx, MIN((uint32_t) kparams->n_hvx, nwork))) { + return HTP_STATUS_INVAL_PARAMS; + } + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_hvx, + htp_im2col_scratch_bytes(kparams->kernel_type, kparams->KH, kparams->patch_stride)); struct htp_im2col_context ictx = { 0 }; - ictx.octx = octx; - ictx.patch_base = patch_base; - ictx.npatches = npatches; - ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads; - - // Clean non-overlapping patch-embed -> DMA kernel (if it fits VTCM); - // everything else (padding/dilation/stride edges) -> blocked-staging DMA - // kernel; if neither fits VTCM -> pure-DDR kernel. - if (nrows > 0) { - const uint32_t pth = MIN(octx->n_threads, nrows); - if (pth > 0) { - ictx.pe_row_base = row_base; - ictx.pe_nrows = nrows; - const bool exact = im2col_use_patchembed_dma(octx); - if (exact && im2col_patchembed_dma_fits(octx, &ictx, pth)) { - ictx.pe_rows_per_thread = (nrows + pth - 1) / pth; - work_queue_run(octx->ctx->work_queue, - dst->type == HTP_TYPE_F16 ? im2col_patchembed_dma_thread - : im2col_patchembed_dma_f32_thread, &ictx, pth); - return HTP_STATUS_OK; - } - if (!exact && im2col_blocked_dma_fits(octx, &ictx, pth)) { - ictx.pe_rows_per_thread = (nrows + pth - 1) / pth; - work_queue_run(octx->ctx->work_queue, - dst->type == HTP_TYPE_F16 ? im2col_blocked_dma_thread - : im2col_blocked_dma_f32_thread, &ictx, pth); - return HTP_STATUS_OK; - } + uint8_t * const base = octx->ctx->vtcm_base; + + const uint32_t pth = octx->n_hvx; + ictx.octx = octx; + ictx.pe_row_base = row_base; + ictx.pe_nrows = nrows; + ictx.pe_rows_per_hvx = fastdiv(nrows + pth - 1, &octx->n_hvx_div); + ictx.pe_blocks_per_hvx = fastdiv(nwork + pth - 1, &octx->n_hvx_div); + ictx.pe_src_row_bytes = kparams->src_row_bytes; + ictx.pe_dst_row_bytes = kparams->dst_row_bytes; + ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src); + ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst); + ictx.pe_src_size_per_hvx = layout.src_bytes_per_hvx; + ictx.pe_dst_size_per_hvx = layout.dst_bytes_per_hvx; + ictx.pe_owb = kparams->owb; + ictx.pe_wb = kparams->wb; + ictx.pe_nbpr = kparams->nbpr; + ictx.N = kparams->N; + ictx.IC = kparams->IC; + ictx.ID = kparams->ID; + ictx.IH = kparams->IH; + ictx.IW = kparams->IW; + ictx.KD = kparams->KD; + ictx.KH = kparams->KH; + ictx.KW = kparams->KW; + ictx.OD = kparams->OD; + ictx.OH = kparams->OH; + ictx.OW = kparams->OW; + ictx.patch_stride = kparams->patch_stride; + ictx.pe_gather_nvec = hex_round_up(kparams->patch_stride, 32) / 32; + + // a thread starts at a row, or at an output-col block (possibly mid-row) on the blocked path + for (uint32_t ith = 0; ith < pth; ith++) { + uint32_t r = row_base + ictx.pe_rows_per_hvx * ith; + uint32_t c0 = 0; + if (blocked) { + const uint32_t blk = ictx.pe_blocks_per_hvx * ith; + const uint32_t q = fastdiv(blk, &kparams->div_nbpr); + r = row_base + q; + c0 = (blk - q * kparams->nbpr) * kparams->owb; } + const uint32_t in = fastdiv(r, &kparams->div_OH); + ictx.thread_ioh[ith] = r - in * kparams->OH; + ictx.thread_in[ith] = in; + ictx.thread_c0[ith] = c0; } - // Fall through to pure-DDR. - if (npatches == 0) { + + ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float); + + if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_patchembed_compute_f16 + : (void *) im2col_patchembed_compute_f32; + work_queue_run(octx->ctx->hvx_queue, im2col_patchembed_dma_thread, &ictx, pth); + return HTP_STATUS_OK; + } else if (blocked) { + if (kparams->KH == 1) { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_gather_compute_f16 + : (void *) im2col_gather_compute_f32; + } else { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16 + : (void *) im2col_blocked_compute_f32; + } + work_queue_run(octx->ctx->hvx_queue, im2col_blocked_dma_thread, &ictx, pth); return HTP_STATUS_OK; } - if (htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; +} + +int op_im2col_3d(struct htp_ops_context * octx) { + const struct htp_im2col_kernel_params * kparams = + (const struct htp_im2col_kernel_params *) octx->kernel_params; + if (kparams->kernel_type == HTP_IM2COL_KERNEL_UNSUPPORTED) { return HTP_STATUS_NO_SUPPORT; } - if (dst->type == HTP_TYPE_F16) { - work_queue_run(octx->ctx->work_queue, im2col_patchembed_thread, &ictx, n_threads); - } else { - work_queue_run(octx->ctx->work_queue, im2col_patchembed_f32_thread, &ictx, n_threads); + const struct htp_tensor * dst = octx->dst; + + uint32_t row_base = 0; + uint32_t nrows = kparams->total_rows; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = + htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_base = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; } - return HTP_STATUS_OK; + + if (!htp_ops_context_set_n_hvx(octx, MIN((uint32_t) kparams->n_hvx, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_hvx, 256); + + struct htp_im2col_context ictx = { 0 }; + uint8_t * const base = octx->ctx->vtcm_base; + + const uint32_t pth = octx->n_hvx; + ictx.octx = octx; + ictx.pe_row_base = row_base; + ictx.pe_nrows = nrows; + ictx.pe_rows_per_hvx = fastdiv(nrows + pth - 1, &octx->n_hvx_div); + ictx.pe_src_row_bytes = kparams->src_row_bytes; + ictx.pe_dst_row_bytes = kparams->dst_row_bytes; + ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src); + ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst); + ictx.pe_src_size_per_hvx = layout.src_bytes_per_hvx; + ictx.pe_dst_size_per_hvx = layout.dst_bytes_per_hvx; + ictx.pe_owb = kparams->owb; + ictx.pe_wb = kparams->wb; + ictx.pe_nbpr = kparams->nbpr; + ictx.N = kparams->N; + ictx.IC = kparams->IC; + ictx.ID = kparams->ID; + ictx.IH = kparams->IH; + ictx.IW = kparams->IW; + ictx.KD = kparams->KD; + ictx.KH = kparams->KH; + ictx.KW = kparams->KW; + ictx.OD = kparams->OD; + ictx.OH = kparams->OH; + ictx.OW = kparams->OW; + ictx.patch_stride = kparams->patch_stride; + + for (uint32_t ith = 0; ith < pth; ith++) { + const uint32_t rem = row_base + ictx.pe_rows_per_hvx * ith; + const uint32_t q_oh = fastdiv(rem, &kparams->div_OH); + ictx.thread_ioh[ith] = rem - q_oh * kparams->OH; + const uint32_t in = fastdiv(q_oh, &kparams->div_OD); + ictx.thread_iod[ith] = q_oh - in * kparams->OD; + ictx.thread_in[ith] = in; + } + + ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float); + + if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_patchembed_compute_f16 + : (void *) im2col_3d_patchembed_compute_f32; + work_queue_run(octx->ctx->hvx_queue, im2col_3d_patchembed_dma_thread, &ictx, pth); + return HTP_STATUS_OK; + } else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_blocked_compute_f16 + : (void *) im2col_3d_blocked_compute_f32; + work_queue_run(octx->ctx->hvx_queue, im2col_3d_blocked_dma_thread, &ictx, pth); + return HTP_STATUS_OK; + } + + return HTP_STATUS_NO_SUPPORT; } diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.h b/ggml/src/ggml-hexagon/htp/im2col-ops.h new file mode 100644 index 000000000..23497d01d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.h @@ -0,0 +1,81 @@ +#ifndef HTP_IM2COL_OPS_H +#define HTP_IM2COL_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include <stdint.h> + +enum htp_im2col_kernel_type { + HTP_IM2COL_KERNEL_UNSUPPORTED = 0, + HTP_IM2COL_KERNEL_PATCHEMBED_DMA = 1, + HTP_IM2COL_KERNEL_BLOCKED_DMA = 2, +}; + +struct htp_im2col_vtcm_layout { + size_t total_bytes; + size_t off_src; + size_t off_dst; + uint32_t src_bytes_per_hvx; + uint32_t dst_bytes_per_hvx; +}; + +// scratch_bytes: per-hvx VTCM after the two src buffers (at least 256) +static inline void htp_im2col_vtcm_layout_build( + struct htp_im2col_vtcm_layout * L, + uint32_t src_row_bytes, + uint32_t dst_row_bytes, + uint32_t n_hvx, + uint32_t scratch_bytes) { + + L->src_bytes_per_hvx = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256); + L->dst_bytes_per_hvx = 2 * dst_row_bytes; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_hvx * n_hvx); + L->total_bytes = off; +} + +// per-hvx scratch: the KH == 1 blocked path keeps its offset table and one gathered row there, +// both padded to whole vectors +static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t KH, uint32_t patch_stride) { + if (kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA && KH == 1) { + return 2 * hex_round_up(patch_stride, 32) * (uint32_t) sizeof(float); + } + return 256; +} + +struct htp_im2col_kernel_params { + uint8_t kernel_type; + uint8_t n_hvx; + + uint32_t vtcm_size; + uint32_t src_row_bytes; + uint32_t dst_row_bytes; + + uint32_t owb; + uint32_t wb; + uint32_t nbpr; + + uint32_t N; + uint32_t IC; + uint32_t ID, IH, IW; + uint32_t KD, KH, KW; + uint32_t OD, OH, OW; + uint32_t patch_stride; + uint32_t total_rows; + uint32_t rows_per_chunk; + + struct fastdiv_values div_OH; + struct fastdiv_values div_OD; + struct fastdiv_values div_nbpr; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_im2col_kernel_params) <= 128, "htp_im2col_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_im2col_kernel_params) <= 128, "htp_im2col_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_IM2COL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c new file mode 100644 index 000000000..5d60c506e --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c @@ -0,0 +1,1722 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include <HAP_farf.h> +#include <string.h> + +#include "dma-queue.h" +#include "htp-trace.h" +#include "hvx-utils.h" +#include "hvx-copy.h" +#include "htp-vtcm.h" +#include "work-queue.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-quant.h" +#include "hvx-fa-kernels.h" +#include "hmx-utils.h" +#include "hmx-fa-kernels.h" +#include "hmx-queue.h" + +#include "lightning-indexer-ops.h" + +struct htp_lid_context { + const struct htp_ops_context * octx; + const struct htp_lid_kernel_params * kp; + + uint32_t n_blocks; + uint32_t blocks_per_hvx; + + // current stream and token tile + uint32_t is; + uint32_t im3; + uint32_t t0; + uint32_t nt; + + struct htp_lid_vtcm_layout L; + + uint8_t * spad_q; + uint8_t * spad_w; + uint8_t * spad_qs; + uint8_t * spad_k; + uint8_t * spad_kt; + uint8_t * spad_m; + uint8_t * spad_o; +}; + +static inline void lid_prep_step_info(uint32_t k, uint32_t K, uint32_t t_first, uint32_t i0, uint32_t i1, uint32_t n_head, + uint32_t * out_t, uint32_t * out_ha, uint32_t * out_hb) { + const uint32_t t = t_first + k; + const uint32_t ha = (k == 0) ? (i0 - t * n_head) : 0; + const uint32_t hb = (k == K - 1) ? (i1 - t * n_head) : n_head; + *out_t = t; + *out_ha = ha; + *out_hb = hb; +} + +static inline void lid_prep_push_step(dma_queue * dma_q, const struct htp_lid_context * lctx, + const struct htp_tensor * q, const struct htp_tensor * w, + uint8_t * qs, uint8_t * ws, uint32_t qs_row_size, uint32_t w_row_size, + uint32_t D, uint32_t n_head, uint32_t t, uint32_t ha, uint32_t hb) { + const uint32_t it = lctx->t0 + t; + dma_queue_push(dma_q, dma_make_data(qs, q->data + it * q->nb[2] + lctx->is * q->nb[3] + ha * q->nb[1]), + qs_row_size, q->nb[1], D * sizeof(float), hb - ha); + dma_queue_push(dma_q, dma_make_data(ws, w->data + it * w->nb[1] + lctx->is * w->nb[3]), + w_row_size, w_row_size, n_head * sizeof(float), 1); +} + +// Splat every Q element (f16) and every head weight (f32) into a full vector. +// Work is split over (token, head) pairs. +static void lid_prep_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_context * lctx = (struct htp_lid_context *) data; + const struct htp_ops_context * octx = lctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * w = octx->src[2]; + + const struct htp_lid_vtcm_layout * L = &lctx->L; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + + // nth is min(n_hvx, n_items): the same chunk as a split over n_hvx + const uint32_t n_items = lctx->nt * n_head; + const uint32_t chunk = fastdiv(n_items + octx->n_hvx - 1, &octx->n_hvx_div); + const uint32_t i0 = ith * chunk; + const uint32_t i1 = MIN(i0 + chunk, n_items); + + if (i0 >= i1) { + return; + } + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + const uint32_t w_row_size = hex_round_up(n_head * sizeof(float), 128); + const size_t qs_size = n_head * L->size_qs_row; + const size_t slot_size = qs_size + w_row_size; + + uint8_t * qs[2]; + uint8_t * ws[2]; + qs[0] = lctx->spad_qs + ith * L->size_qs; + ws[0] = qs[0] + qs_size; + qs[1] = qs[0] + slot_size; + ws[1] = qs[1] + qs_size; + uint8_t * qh = qs[0] + 2 * slot_size; + + const uint32_t t_first = fastdiv(i0, &lctx->kp->div_n_head); + const uint32_t t_last = fastdiv(i1 - 1, &lctx->kp->div_n_head); + const uint32_t n_steps = t_last - t_first + 1; + + // Prologue: prime both slots + uint32_t t, ha, hb; + lid_prep_step_info(0, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[0], ws[0], L->size_qs_row, w_row_size, D, n_head, t, ha, hb); + + if (n_steps > 1) { + lid_prep_step_info(1, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[1], ws[1], L->size_qs_row, w_row_size, D, n_head, t, ha, hb); + } + + // Loop: pop(slot) -> compute(slot) -> push(slot, k + 2) + for (uint32_t k = 0; k < n_steps; ++k) { + const uint32_t slot = k & 1; + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + lid_prep_step_info(k, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + const uint32_t it = lctx->t0 + t; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) it); + + HVX_Vector * restrict q_sp = (HVX_Vector *) lctx->spad_q + (t * n_head + ha) * D; + for (uint32_t h = ha; h < hb; ++h) { + hvx_copy_f16_f32_aa(qh, qs[slot] + (h - ha) * L->size_qs_row, D); + + const HVX_Vector * restrict qv = (const HVX_Vector *) qh; + for (uint32_t d0 = 0; d0 < D; d0 += VLEN_FP16) { + const HVX_Vector v = qv[d0 / VLEN_FP16]; + const uint32_t nd = MIN(VLEN_FP16, D - d0); + for (uint32_t j = 0; j < nd; ++j) { + *q_sp++ = hvx_vec_repl_f16(Q6_V_vror_VR(v, j * SIZEOF_FP16)); + } + } + } + + const HVX_Vector * restrict wv = (const HVX_Vector *) ws[slot]; + HVX_Vector * restrict w_dst = (HVX_Vector *) (lctx->spad_w + t * n_head * VLEN); + for (uint32_t h = ha; h < hb; ++h) { + w_dst[h] = hvx_vec_repl4(Q6_V_vror_VR(wv[h / VLEN_FP32], (h % VLEN_FP32) * SIZEOF_FP32)); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) it); + + if (k + 2 < n_steps) { + uint32_t next_t, next_ha, next_hb; + lid_prep_step_info(k + 2, n_steps, t_first, i0, i1, n_head, &next_t, &next_ha, &next_hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[slot], ws[slot], L->size_qs_row, w_row_size, D, n_head, next_t, next_ha, next_hb); + } + } +} + +#if __HVX_ARCH__ < 79 +#define LID_MUL_F16(a, b) Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(a, b)) +#else +#define LID_MUL_F16(a, b) Q6_Vhf_vmpy_VhfVhf(a, b) +#endif + +// Q8_0 row to f16 in place, 4 blocks (128 values) per step. +// Go backwards: the f16 output of a group is wider than its input and covers the input of the next group. +static inline void lid_dequant_row_q8_0(uint8_t * restrict row, uint32_t D) { + const int32_t nb = D / QK8_0; + + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred q64 = Q6_Q_vsetq_R(64); + const HVX_VectorPred q96 = Q6_Q_vsetq_R(96); + + for (int32_t g = (nb - 1) / 4; g >= 0; --g) { + const uint8_t * src = row + g * 4 * sizeof(block_q8_0); + const HVX_Vector v0 = *(const HVX_UVector *) src; + const HVX_Vector v1 = *(const HVX_UVector *) (src + VLEN); + + HVX_Vector qs = Q6_V_vmux_QVV(q96, Q6_V_valign_VVR(v1, v0, 6), Q6_V_valign_VVR(v1, v0, 8)); + qs = Q6_V_vmux_QVV(q64, Q6_V_valign_VVR(v1, v0, 4), qs); + qs = Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(v1, v0, 2), qs); + + const HVX_Vector s01 = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(v0), hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 34))); + const HVX_Vector s23 = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 68)), hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 102))); + + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(qs); + + HVX_Vector * dst = (HVX_Vector *) (row + g * 2 * VLEN); + dst[0] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(qh)), s01)); + if (g * 4 + 2 < nb) { + dst[1] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(qh)), s23)); + } + } +} + +// Q4_0 blocks i, i + 1 (v loaded at block i) to signed quants: block i in bytes 0..31, block i + 1 in bytes 32..63. +// qs[j] holds value j (low nibble) and j + 16 (high nibble). lo16 selects bytes 0..15 and 32..47. +static inline HVX_Vector lid_q4_0_pair_quants(HVX_Vector v, HVX_VectorPred q32, HVX_VectorPred lo16) { + const HVX_Vector qs = Q6_V_vmux_QVV(q32, Q6_V_vror_VR(v, sizeof(ggml_half)), + Q6_V_vror_VR(v, VLEN - 32 + sizeof(block_q4_0) + sizeof(ggml_half))); + const HVX_Vector lo = Q6_Vb_vsub_VbVb(Q6_V_vand_VV(qs, Q6_Vb_vsplat_R(0x0F)), Q6_Vb_vsplat_R(8)); + const HVX_Vector hi = Q6_Vb_vsub_VbVb(Q6_Vub_vlsr_VubR(qs, 4), Q6_Vb_vsplat_R(8)); + return Q6_V_vmux_QVV(lo16, lo, Q6_V_vlalign_VVR(hi, Q6_V_vzero(), 16)); +} + +// Q4_0 row to f16 in place, 2 blocks (64 values) per step. Backwards, as for Q8_0. +static inline void lid_dequant_row_q4_0(uint8_t * restrict row, uint32_t D) { + const int32_t nb = D / QK4_0; + + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred q64 = Q6_Q_vsetq_R(64); + const HVX_VectorPred lo16 = Q6_Q_or_QQ(Q6_Q_vsetq_R(16), Q6_Q_and_QQn(Q6_Q_vsetq_R(48), q32)); + + for (int32_t p = (nb - 1) / 2; p >= 0; --p) { + const HVX_Vector v = *(const HVX_UVector *) (row + p * 2 * sizeof(block_q4_0)); + const HVX_Vector q = lid_q4_0_pair_quants(v, q32, lo16); + const HVX_Vector d1 = 2 * p + 1 < nb ? hvx_vec_repl_f16(Q6_V_vror_VR(v, sizeof(block_q4_0))) : Q6_V_vzero(); + const HVX_Vector d = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(v), d1); + + ((HVX_Vector *) row)[p] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_lo_W(Q6_Wh_vunpack_Vb(q))), d); + } +} + +// Quantized kv block to f16 rows in place. Not inlined: the dequant constants would stay live across the compute loop. +static __attribute__((noinline)) void lid_dequant_block(uint8_t * restrict k_buf, size_t row_stride, uint32_t nkv, uint32_t D, bool q4) { + if (q4) { + for (uint32_t r = 0; r < nkv; ++r) { + lid_dequant_row_q4_0(k_buf + r * row_stride, D); + } + } else { + for (uint32_t r = 0; r < nkv; ++r) { + lid_dequant_row_q8_0(k_buf + r * row_stride, D); + } + } +} + +static inline void lid_convert_row_f32_to_f16(uint8_t * row, uint32_t D) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector * src = (const HVX_Vector *) row; + HVX_Vector * dst = (HVX_Vector *) row; + + for (uint32_t d = 0; d < D; d += 64) { + dst[d / 64] = hvx_vec_f32_to_f16(src[d / 32], d + 32 < D ? src[d / 32 + 1] : zero); + } +} + +// Q8_0 gather transpose limits: tmp (two 64x64 scratch tiles) holds D / 2 pairs, gather offsets are 16 bit +static inline bool lid_q8_0_gather_ok(size_t row_size, uint32_t D) { + return D <= 4 * HTP_LID_KV_BLOCK && (HTP_LID_KV_BLOCK - 1) * row_size + (D / QK8_0) * sizeof(block_q8_0) <= 32767; +} + +// Transpose a raw Q8_0 kv block: one halfword gather reads q[d], q[d + 1] of all 64 rows, then split, widen to f16 and scale. +// Lanes of missing rows hold stale data; they are not stored. +static inline void lid_transpose_q8_0_gather(HVX_Vector * restrict kt, + HVX_Vector * restrict tmp, + HVX_Vector * restrict sc, + const uint8_t * restrict k_buf, + size_t row_size, + HVX_Vector offsets, + uint32_t D) { + const uint32_t mu = HTP_LID_KV_BLOCK * row_size; + for (uint32_t b = 0; b < D / QK8_0; ++b) { + Q6_vgather_ARMVh(sc + b, (size_t) (k_buf + b * sizeof(block_q8_0)), mu, offsets); + } + for (uint32_t d = 0; d < D; d += 2) { + const uint32_t off = (d / QK8_0) * sizeof(block_q8_0) + sizeof(ggml_half) + d % QK8_0; + Q6_vgather_ARMVh(tmp + d / 2, (size_t) (k_buf + off), mu, offsets); + } + hvx_gather_sync(sc); + hvx_gather_sync(tmp); + + for (uint32_t d = 0; d < D; d += 2) { + const HVX_Vector s = sc[d / QK8_0]; + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(tmp[d / 2]); + const HVX_VectorPair qd = Q6_W_vdeal_VVR(Q6_V_hi_W(qh), Q6_V_lo_W(qh), -2); + kt[d + 0] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(qd)), s)); + kt[d + 1] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(qd)), s)); + } +} + +static inline void lid_shuff_pass(HVX_Vector * restrict dst, const HVX_Vector * restrict src) { + for (uint32_t j = 0; j < 32; ++j) { + HVX_VectorPair p = Q6_W_vshuff_VVR(src[j + 32], src[j], -SIZEOF_FP16); + dst[2 * j + 0] = Q6_V_lo_W(p); + dst[2 * j + 1] = Q6_V_hi_W(p); + } +} + +// 64x64 f16 transpose: each shuffle pass rotates the 12-bit (row, lane) index by one bit, six passes swap row and lane +static inline void lid_transpose_64x64_f16(HVX_Vector * restrict dst, + const uint8_t * restrict src, + size_t src_stride, + uint32_t nkv, + HVX_Vector * restrict t0, + HVX_Vector * restrict t1) { + if (nkv >= HTP_LID_KV_BLOCK) { + for (uint32_t j = 0; j < 32; ++j) { + const HVX_Vector r0 = *(const HVX_Vector *) (src + j * src_stride); + const HVX_Vector r1 = *(const HVX_Vector *) (src + (j + 32) * src_stride); + HVX_VectorPair p = Q6_W_vshuff_VVR(r1, r0, -SIZEOF_FP16); + t0[2 * j + 0] = Q6_V_lo_W(p); + t0[2 * j + 1] = Q6_V_hi_W(p); + } + } else { + const HVX_Vector zero = Q6_V_vzero(); + for (uint32_t j = 0; j < 32; ++j) { + const HVX_Vector r0 = (j < nkv) ? *(const HVX_Vector *) (src + j * src_stride) : zero; + const HVX_Vector r1 = (j + 32 < nkv) ? *(const HVX_Vector *) (src + (j + 32) * src_stride) : zero; + HVX_VectorPair p = Q6_W_vshuff_VVR(r1, r0, -SIZEOF_FP16); + t0[2 * j + 0] = Q6_V_lo_W(p); + t0[2 * j + 1] = Q6_V_hi_W(p); + } + } + lid_shuff_pass(t1, t0); + lid_shuff_pass(t0, t1); + lid_shuff_pass(t1, t0); + lid_shuff_pass(t0, t1); + lid_shuff_pass(dst, t0); +} + +#if __HVX_ARCH__ < 79 +// qf32 accumulators, converted to sf once per head +static inline HVX_VectorPair lid_mpyacc(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y) { + HVX_VectorPair m = Q6_Wqf32_vmpy_VhfVhf(x, y); + return Q6_W_vcombine_VV(Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_hi_W(acc), Q6_V_hi_W(m)), + Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(acc), Q6_V_lo_W(m))); +} +static inline HVX_Vector lid_acc_to_sf(HVX_Vector v) { + return Q6_Vsf_equals_Vqf32(v); +} +#else +static inline HVX_VectorPair lid_mpyacc(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y) { + return Q6_Wsf_vmpyacc_WsfVhfVhf(acc, x, y); +} +static inline HVX_Vector lid_acc_to_sf(HVX_Vector v) { + return v; +} +#endif + +// out += relu(s) * w, for the even and odd kv lanes +static inline HVX_VectorPair lid_head_acc(HVX_VectorPair out, HVX_VectorPair s, HVX_Vector w) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector e = HVX_OP_ADD_F32(Q6_V_lo_W(out), HVX_OP_MUL_F32(Q6_Vsf_vmax_VsfVsf(lid_acc_to_sf(Q6_V_lo_W(s)), zero), w)); + HVX_Vector o = HVX_OP_ADD_F32(Q6_V_hi_W(out), HVX_OP_MUL_F32(Q6_Vsf_vmax_VsfVsf(lid_acc_to_sf(Q6_V_hi_W(s)), zero), w)); + return Q6_W_vcombine_VV(o, e); +} + +// Store one output row: (even, odd) lanes back to natural order, add the mask +static inline void lid_store_row(float * restrict o, HVX_VectorPair out, const __fp16 * restrict m) { + const HVX_Vector zero = Q6_V_vzero(); + + const HVX_VectorPair out_nat = Q6_W_vshuff_VVR(Q6_V_hi_W(out), Q6_V_lo_W(out), -SIZEOF_FP32); + + HVX_Vector acc0 = Q6_V_lo_W(out_nat); + HVX_Vector acc1 = Q6_V_hi_W(out_nat); + + // qf32 math does not keep inf: add a finite mask, then put -inf back + const HVX_Vector m_f16 = *(const HVX_Vector *) m; + const HVX_VectorPred m_inf = Q6_Q_vcmp_eq_VhVh(m_f16, Q6_Vh_vsplat_R(0xFC00)); + + const HVX_VectorPair m_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, zero, m_f16)); + const HVX_VectorPair f_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, Q6_Vh_vsplat_R(0x3C00), zero)); + + const HVX_Vector ninf = Q6_V_vsplat_R(0xFF800000); + + acc0 = HVX_OP_ADD_F32(acc0, Q6_V_lo_W(m_f32)); + acc1 = HVX_OP_ADD_F32(acc1, Q6_V_hi_W(m_f32)); + acc0 = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_lo_W(f_f32), zero), ninf, acc0); + acc1 = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_hi_W(f_f32), zero), ninf, acc1); + + HVX_Vector * restrict vo = (HVX_Vector *) o; + vo[0] = acc0; + vo[1] = acc1; +} + +// One output row (token) of a kv block: sum_h relu(q_h . k) * w_h + mask +// kt holds the K block transposed (one f16 vector of 64 kv per d), q_t holds the Q splats of the token. +static inline void lid_compute_row(float * restrict o, + const HVX_Vector * restrict q_t, + const HVX_Vector * restrict w_t, + const HVX_Vector * restrict kt, + const __fp16 * restrict m, + uint32_t D, + uint32_t n_head) { + // f16 x f16 products land in (even kv, odd kv) lane order + HVX_VectorPair out = Q6_W_vzero(); + + uint32_t h = 0; + for (; h + 4 <= n_head; h += 4) { + const HVX_Vector * restrict q0 = q_t + h * D; + const HVX_Vector * restrict q1 = q0 + D; + const HVX_Vector * restrict q2 = q1 + D; + const HVX_Vector * restrict q3 = q2 + D; + + HVX_VectorPair s0 = Q6_W_vzero(); + HVX_VectorPair s1 = Q6_W_vzero(); + HVX_VectorPair s2 = Q6_W_vzero(); + HVX_VectorPair s3 = Q6_W_vzero(); + + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector k = kt[d]; + s0 = lid_mpyacc(s0, k, q0[d]); + s1 = lid_mpyacc(s1, k, q1[d]); + s2 = lid_mpyacc(s2, k, q2[d]); + s3 = lid_mpyacc(s3, k, q3[d]); + } + + out = lid_head_acc(out, s0, w_t[h + 0]); + out = lid_head_acc(out, s1, w_t[h + 1]); + out = lid_head_acc(out, s2, w_t[h + 2]); + out = lid_head_acc(out, s3, w_t[h + 3]); + } + for (; h < n_head; ++h) { + const HVX_Vector * restrict q0 = q_t + h * D; + + HVX_VectorPair s0 = Q6_W_vzero(); + for (uint32_t d = 0; d < D; ++d) { + s0 = lid_mpyacc(s0, kt[d], q0[d]); + } + out = lid_head_acc(out, s0, w_t[h]); + } + + lid_store_row(o, out, m); +} + +// Same as lid_compute_row for two kv blocks at once: each Q splat load feeds both blocks +static inline void lid_compute_row2(float * restrict o0, + float * restrict o1, + const HVX_Vector * restrict q_t, + const HVX_Vector * restrict w_t, + const HVX_Vector * restrict kt0, + const HVX_Vector * restrict kt1, + const __fp16 * restrict m0, + const __fp16 * restrict m1, + uint32_t D, + uint32_t n_head) { + HVX_VectorPair out0 = Q6_W_vzero(); + HVX_VectorPair out1 = Q6_W_vzero(); + + uint32_t h = 0; + for (; h + 4 <= n_head; h += 4) { + const HVX_Vector * restrict q0 = q_t + h * D; + const HVX_Vector * restrict q1 = q0 + D; + const HVX_Vector * restrict q2 = q1 + D; + const HVX_Vector * restrict q3 = q2 + D; + + HVX_VectorPair a0 = Q6_W_vzero(); + HVX_VectorPair a1 = Q6_W_vzero(); + HVX_VectorPair a2 = Q6_W_vzero(); + HVX_VectorPair a3 = Q6_W_vzero(); + HVX_VectorPair b0 = Q6_W_vzero(); + HVX_VectorPair b1 = Q6_W_vzero(); + HVX_VectorPair b2 = Q6_W_vzero(); + HVX_VectorPair b3 = Q6_W_vzero(); + + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector k0 = kt0[d]; + const HVX_Vector k1 = kt1[d]; + const HVX_Vector v0 = q0[d]; + const HVX_Vector v1 = q1[d]; + const HVX_Vector v2 = q2[d]; + const HVX_Vector v3 = q3[d]; + a0 = lid_mpyacc(a0, k0, v0); + b0 = lid_mpyacc(b0, k1, v0); + a1 = lid_mpyacc(a1, k0, v1); + b1 = lid_mpyacc(b1, k1, v1); + a2 = lid_mpyacc(a2, k0, v2); + b2 = lid_mpyacc(b2, k1, v2); + a3 = lid_mpyacc(a3, k0, v3); + b3 = lid_mpyacc(b3, k1, v3); + } + + out0 = lid_head_acc(out0, a0, w_t[h + 0]); + out0 = lid_head_acc(out0, a1, w_t[h + 1]); + out0 = lid_head_acc(out0, a2, w_t[h + 2]); + out0 = lid_head_acc(out0, a3, w_t[h + 3]); + out1 = lid_head_acc(out1, b0, w_t[h + 0]); + out1 = lid_head_acc(out1, b1, w_t[h + 1]); + out1 = lid_head_acc(out1, b2, w_t[h + 2]); + out1 = lid_head_acc(out1, b3, w_t[h + 3]); + } + for (; h < n_head; ++h) { + const HVX_Vector * restrict q0 = q_t + h * D; + + HVX_VectorPair a0 = Q6_W_vzero(); + HVX_VectorPair b0 = Q6_W_vzero(); + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector v0 = q0[d]; + a0 = lid_mpyacc(a0, kt0[d], v0); + b0 = lid_mpyacc(b0, kt1[d], v0); + } + out0 = lid_head_acc(out0, a0, w_t[h]); + out1 = lid_head_acc(out1, b0, w_t[h]); + } + + lid_store_row(o0, out0, m0); + lid_store_row(o1, out1, m1); +} + +// Each thread owns a range of kv blocks and streams K / mask in, output out (HTP_LID_N_BUF buffers) +static void lid_compute_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_context * lctx = (struct htp_lid_context *) data; + const struct htp_ops_context * octx = lctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * m = octx->src[3]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t b0 = lctx->blocks_per_hvx * ith; + const uint32_t b1 = MIN(b0 + lctx->blocks_per_hvx, lctx->n_blocks); + + if (b0 >= b1) { + return; + } + + const struct htp_lid_vtcm_layout * L = &lctx->L; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_kv = k->ne[2]; + const uint32_t nt = lctx->nt; + + const bool is_f32 = (k->type == HTP_TYPE_F32); + const bool is_q8_0 = (k->type == HTP_TYPE_Q8_0); + const bool is_q4_0 = (k->type == HTP_TYPE_Q4_0); + const uint32_t k_row_size = lctx->kp->k_row_size; + + const uint32_t nbk2 = k->nb[2]; + const uint32_t nbm1 = m->nb[1]; + const uint32_t nb1 = dst->nb[1]; + + const dma_addr_t k_base = k->data + lctx->is * k->nb[3]; + const dma_addr_t m_base = m->data + lctx->t0 * nbm1 + lctx->im3 * m->nb[3]; + const dma_addr_t d_base = dst->data + lctx->t0 * nb1 + lctx->is * dst->nb[3]; + + uint8_t * spad_k = lctx->spad_k + ith * HTP_LID_N_BUF * L->size_k_buf; + uint8_t * spad_m = lctx->spad_m + ith * HTP_LID_N_BUF * L->size_m_buf; + uint8_t * spad_o = lctx->spad_o + ith * HTP_LID_N_BUF * L->size_o_buf; + + HVX_Vector * kt[2]; + kt[0] = (HVX_Vector *) (lctx->spad_kt + ith * L->size_kt); + kt[1] = kt[0] + hex_round_up(D, HTP_LID_KV_BLOCK); + HVX_Vector * kt_t0 = kt[1] + hex_round_up(D, HTP_LID_KV_BLOCK); + HVX_Vector * kt_t1 = kt_t0 + HTP_LID_KV_BLOCK; + + // Q8_0: transpose by gather from the raw rows (sc: one vector of block scales per 32 d) + const bool q8_g = is_q8_0 && lid_q8_0_gather_ok(L->size_k_raw_row, D); + HVX_Vector * sc = kt_t1 + HTP_LID_KV_BLOCK; + const HVX_Vector g_offs = hvx_vec_gather_offsets_h(L->size_k_raw_row); + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + for (uint32_t b = b0, i = 0; b < b1 && i < HTP_LID_N_BUF; ++b, ++i) { + const uint32_t kv0 = b * HTP_LID_KV_BLOCK; + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - kv0); + + dma_queue_push(dma_q, dma_make_data(d_base, spad_o + i * L->size_o_buf), + nb1, L->size_o_row, nkv * sizeof(float), 0); + dma_queue_push(dma_q, dma_make_data(spad_k + i * L->size_k_buf, k_base + kv0 * nbk2), + L->size_k_raw_row, nbk2, k_row_size, nkv); + dma_queue_push(dma_q, dma_make_data(spad_m + i * L->size_m_buf, m_base + kv0 * sizeof(__fp16)), + L->size_m_row, nbm1, nkv * sizeof(__fp16), nt); + } + + // Two kv blocks per step, the next two are in flight + for (uint32_t b = b0; b < b1; b += 2) { + const uint32_t nb = MIN(2, b1 - b); + + uint8_t * o_buf[2]; + uint8_t * k_buf[2]; + uint8_t * m_buf[2]; + + for (uint32_t j = 0; j < nb; ++j) { + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - (b + j) * HTP_LID_KV_BLOCK); + + o_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; + k_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; + m_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; + + if ((is_q8_0 && !q8_g) || is_q4_0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, (uint16_t) (b + j)); + lid_dequant_block(k_buf[j], L->size_k_raw_row, nkv, D, is_q4_0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, (uint16_t) (b + j)); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (b + j)); + if (is_f32) { + for (uint32_t r = 0; r < nkv; ++r) { + lid_convert_row_f32_to_f16(k_buf[j] + r * L->size_k_raw_row, D); + } + } + if (q8_g) { + lid_transpose_q8_0_gather(kt[j], kt_t0, sc, k_buf[j], L->size_k_raw_row, g_offs, D); + } else { + for (uint32_t d0 = 0; d0 < D; d0 += HTP_LID_KV_BLOCK) { + lid_transpose_64x64_f16(kt[j] + d0, k_buf[j] + d0 * sizeof(__fp16), L->size_k_raw_row, nkv, kt_t0, kt_t1); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (b + j)); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) b); + for (uint32_t t = 0; t < nt; ++t) { + const HVX_Vector * q_t = (const HVX_Vector *) lctx->spad_q + t * n_head * D; + const HVX_Vector * w_t = (const HVX_Vector *) (lctx->spad_w + t * n_head * VLEN); + if (nb == 2) { + lid_compute_row2((float *) (o_buf[0] + t * L->size_o_row), (float *) (o_buf[1] + t * L->size_o_row), + q_t, w_t, kt[0], kt[1], + (const __fp16 *) (m_buf[0] + t * L->size_m_row), (const __fp16 *) (m_buf[1] + t * L->size_m_row), + D, n_head); + } else { + lid_compute_row((float *) (o_buf[0] + t * L->size_o_row), q_t, w_t, kt[0], + (const __fp16 *) (m_buf[0] + t * L->size_m_row), D, n_head); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) b); + + for (uint32_t j = 0; j < nb; ++j) { + const uint32_t kv0 = (b + j) * HTP_LID_KV_BLOCK; + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - kv0); + + dma_queue_push(dma_q, dma_make_data(d_base + kv0 * sizeof(float), o_buf[j]), + nb1, L->size_o_row, nkv * sizeof(float), nt); + + if (b + j + HTP_LID_N_BUF < b1) { + const uint32_t kv2 = kv0 + HTP_LID_N_BUF * HTP_LID_KV_BLOCK; + const uint32_t nkv2 = MIN(HTP_LID_KV_BLOCK, n_kv - kv2); + + dma_queue_push(dma_q, dma_make_data(k_buf[j], k_base + kv2 * nbk2), + L->size_k_raw_row, nbk2, k_row_size, nkv2); + dma_queue_push(dma_q, dma_make_data(m_buf[j], m_base + kv2 * sizeof(__fp16)), + L->size_m_row, nbm1, nkv2 * sizeof(__fp16), nt); + } + } + } + + for (uint32_t i = 0; i < MIN(b1 - b0, HTP_LID_N_BUF); ++i) { + dma_queue_pop(dma_q); + } + + (void) nth; +} + +// ----- HMX path: K is the activation, Q the weights, head sum by a second HMX pass ----- +// +// Channel c = t * n_head + h of a token tile. Per kv row tile (32 kv) and token tile u (32 tokens): +// mm1: S[kv, c] = relu(k . q_c) * |w_c| (convert scale |w_c|, pos clamp) +// mm2: O[kv, t] = sum_c S[kv, c] * G[c, t], G[c, t] = sign(w_c) for c of token t +// O tiles hold f16 (kv pair, token) words; gather one token column, add the mask, store f32. +// +// A run preps the Q sets of up to n_qsets (stream, token tile) pairs, then pipelines all their kv rows. +// Row x of a run is kv x % kv_seg of set x / kv_seg (kv_seg = n_kv padded to 64), so a block of Bc rows can +// span sets: it is split in segments, one per set. HMX computes block g while HVX preps K of g + 1 and extracts O of g - 1. + +struct lid_hmx_set { + uint32_t is; + uint32_t im3; + uint32_t t0; + uint32_t nt; + uint32_t n_ch; + uint32_t n_u; +}; + +struct htp_lid_hmx_context { + const struct htp_ops_context * octx; + const struct htp_lid_kernel_params * kp; + struct htp_lid_hmx_vtcm_layout L; + + uint32_t Bc; + uint32_t kv_seg; // rows per set: n_kv padded to 64 + uint32_t n_blocks; // per run + uint32_t n_tile; // tokens per token tile + uint32_t n_tt; // token tiles per stream + + uint32_t n_qsets; // Q sets per buffer + uint32_t qbuf; // Q buffer of the run (two buffers: the next run's Q is prepped during this run) + + uint32_t tile0; // first (stream, token tile) of the run + uint32_t n_run; // sets in the run + uint32_t n_qthr; // threads with Q work in the run + uint32_t nt_run; // tokens of each set in the run + uint32_t n_u_run; // token tiles of 32 of each set in the run + int32_t g; // current stage + + struct lid_hmx_set sets[HTP_LID_HMX_MAX_QSETS]; + + // next run, Q prepped in the stages of this one + bool nx_on; + uint32_t nx_n_run; + struct fastdiv_values nx_stg_div; // stages of this run: n_blocks + 1 + struct lid_hmx_set nx_sets[HTP_LID_HMX_MAX_QSETS]; + + __fp16 * qw; + __fp16 * gw; + uint8_t * sc; + uint8_t * sc_id; + __fp16 * s; + float * w; + uint8_t * qs; + uint8_t * k_raw[2]; + __fp16 * k_t[2]; + __fp16 * o[2]; + uint8_t * m[4]; + uint8_t * out[2]; +}; + +#define LID_HMX_MAX_SEGS HTP_LID_HMX_MAX_QSETS + +// Rows of one set in a block: valid rows [x0, x0 + nv) of the block are kv [kv0, kv0 + nv) of the set +struct lid_hmx_seg { + uint32_t set; + uint32_t x0; + uint32_t kv0; + uint32_t nv; +}; + +static inline struct lid_hmx_set lid_hmx_set_init(const struct htp_lid_hmx_context * hctx, uint32_t set) { + const struct htp_ops_context * octx = hctx->octx; + const uint32_t ti = hctx->tile0 + set; + + struct lid_hmx_set S; + S.is = fastdiv(ti, &hctx->kp->div_n_tt); + S.im3 = fastmodulo(S.is, octx->src[3]->ne[3], &hctx->kp->div_m_ne3); + S.t0 = (ti - S.is * hctx->n_tt) * hctx->n_tile; + S.nt = MIN(hctx->n_tile, octx->src[0]->ne[2] - S.t0); + S.n_ch = (S.nt * octx->src[0]->ne[1] + 31) / 32; + S.n_u = (S.nt + 31) / 32; + return S; +} + +// Segments of block g with valid rows. Returns their count, n_rows gets the block rows (multiple of 64). +static inline uint32_t lid_hmx_segs(const struct htp_lid_hmx_context * hctx, uint32_t g, struct lid_hmx_seg * seg, uint32_t * n_rows) { + const uint32_t n_kv = hctx->octx->src[1]->ne[2]; + const uint32_t ks = hctx->kv_seg; + const uint32_t xa = g * hctx->Bc; + const uint32_t xb = MIN(xa + hctx->Bc, hctx->n_run * ks); + + uint32_t n = 0; + for (uint32_t x = xa; x < xb;) { + const uint32_t set = fastdiv(x, &hctx->kp->div_kv_seg); + const uint32_t kv = x - set * ks; + const uint32_t end = MIN(xb, (set + 1) * ks); + if (kv < n_kv) { + seg[n].set = set; + seg[n].x0 = x - xa; + seg[n].kv0 = kv; + seg[n].nv = MIN(n_kv, kv + (end - x)) - kv; + n++; + } + x = end; + } + *n_rows = xb - xa; + return n; +} + +typedef struct { + const __fp16 * qw; + const __fp16 * gw; + const uint8_t * sc; + uint32_t rt0; + uint32_t n_rt; +} lid_hmx_job_seg_t; + +typedef struct { + lid_hmx_job_seg_t seg[LID_HMX_MAX_SEGS]; + uint32_t n_seg; + + const __fp16 * k_t; + const uint8_t * sc_id; + __fp16 * s; + __fp16 * o; + uint32_t n_ch; + uint32_t n_u; + uint32_t n_dot; + uint32_t n_head; + uint32_t n_sg; +} lid_hmx_job_t; + +// acc += sum of n tile products, a and b advance one tile per step +static inline void lid_hmx_dot(const __fp16 * a, const __fp16 * b, uint32_t n) { + for (; n > 32; n -= 32) { + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") :: "r"(65535), "r"(a), "r"(b)); + a += 32 * HMX_FP16_TILE_N_ELMS; + b += 32 * HMX_FP16_TILE_N_ELMS; + } + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") :: "r"(n * HMX_FP16_TILE_SIZE - 1), "r"(a), "r"(b)); +} + +// Per segment: channel tile outer, row tile inner, one bias load per channel tile. S holds [n_rt][n_sg] tiles. +static void lid_hmx_worker(const htp_thread_t * th, void * data) { + const lid_hmx_job_t * job = (const lid_hmx_job_t *) data; + + const size_t dot_stride = (size_t) job->n_dot * HMX_FP16_TILE_N_ELMS; + const size_t s_stride = (size_t) job->n_sg * HMX_FP16_TILE_N_ELMS; + const uint32_t nh = job->n_head; + + for (uint32_t i = 0; i < job->n_seg; ++i) { + const lid_hmx_job_seg_t * sg = &job->seg[i]; + const uint32_t n_rt = sg->n_rt; + const __fp16 * kt = job->k_t + sg->rt0 * dot_stride; + __fp16 * ot = job->o + sg->rt0 * job->n_u * HMX_FP16_TILE_N_ELMS; + + for (uint32_t u = 0; u < job->n_u; ++u) { + const uint32_t j0 = u * nh; + const uint32_t j1 = MIN(j0 + nh, job->n_ch); + + for (uint32_t c = j0; c < j1; ++c) { + const __fp16 * qw = sg->qw + c * dot_stride; + __fp16 * st = job->s + (c - j0) * HMX_FP16_TILE_N_ELMS; + + asm volatile(HMX_SET_BIAS("%0") :: "r"(sg->sc + c * 256)); + for (uint32_t rt = 0; rt < n_rt; ++rt) { + asm volatile(HMX_CLRACC_F16()); + lid_hmx_dot(kt + rt * dot_stride, qw, job->n_dot); + asm volatile("mxmem(%0, %1):after:pos.hf = acc\n" :: "r"(st + rt * s_stride), "r"(0) : "memory"); + } + } + + asm volatile(HMX_SET_BIAS("%0") :: "r"(job->sc_id)); + for (uint32_t rt = 0; rt < n_rt; ++rt) { + asm volatile(HMX_CLRACC_F16()); + lid_hmx_dot(job->s + rt * s_stride, sg->gw + j0 * HMX_FP16_TILE_N_ELMS, j1 - j0); + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") :: "r"(ot + (rt * job->n_u + u) * HMX_FP16_TILE_N_ELMS), "r"(0) : "memory"); + } + } + } +} + +// Thread slices: K rows by row pairs of the padded block, mask / output rows by token. +// Block 0 is prepped together with Q: its K rows go to the threads without Q work. +// K rows are split only in stages on all n_hvx threads. +static inline void lid_hmx_rows(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g, uint32_t n_rows, + uint32_t * r0, uint32_t * r1) { + uint32_t per; + if (g == 0 && hctx->n_qthr < nth) { + if (ith < hctx->n_qthr) { + *r0 = *r1 = 0; + return; + } + ith -= hctx->n_qthr; + per = (n_rows + nth - hctx->n_qthr - 1) / (nth - hctx->n_qthr); + } else { + per = fastdiv(n_rows + hctx->octx->n_hvx - 1, &hctx->octx->n_hvx_div); + } + per = hex_round_up(per, 2); + *r0 = MIN(ith * per, n_rows); + *r1 = MIN(*r0 + per, n_rows); +} + +// Split over n_hvx; a stage on one thread has one token, which this split also gives to thread 0 +static inline void lid_hmx_toks(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nt, uint32_t * t0, uint32_t * t1) { + const uint32_t per = fastdiv(nt + hctx->octx->n_hvx - 1, &hctx->octx->n_hvx_div); + *t0 = MIN(ith * per, nt); + *t1 = MIN((ith + 1) * per, nt); +} + +// Push (or only count, push == false) the K row and mask DMAs of block g for this thread, one per segment +static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g, bool push) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * m = octx->src[3]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(hctx, g, seg, &n_rows); + + uint32_t r0, r1, ta, tb; + lid_hmx_rows(hctx, ith, nth, g, n_rows, &r0, &r1); + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + + dma_queue * dma_q = th->dma_q; + const uint32_t k_row_size = hctx->kp->k_row_size; + + uint32_t n = 0; + for (uint32_t i = 0; i < n_seg; ++i) { + const struct lid_hmx_set S = hctx->sets[seg[i].set]; + + const uint32_t ra = MAX(r0, seg[i].x0); + const uint32_t rb = MIN(r1, seg[i].x0 + seg[i].nv); + if (ra < rb) { + if (push) { + dma_queue_push(dma_q, dma_make_data(hctx->k_raw[g % 2] + ra * L->size_k_raw_row, + k->data + S.is * k->nb[3] + (seg[i].kv0 + ra - seg[i].x0) * k->nb[2]), + L->size_k_raw_row, k->nb[2], k_row_size, rb - ra); + } + n++; + } + if (ta < tb) { + if (push) { + dma_queue_push(dma_q, dma_make_data(hctx->m[g % 4] + ta * L->size_m_row + seg[i].x0 * sizeof(__fp16), + m->data + S.im3 * m->nb[3] + (S.t0 + ta) * m->nb[1] + seg[i].kv0 * sizeof(__fp16)), + L->size_m_row, m->nb[1], seg[i].nv * sizeof(__fp16), tb - ta); + } + n++; + } + } + return n; +} + +static inline uint32_t lid_hmx_push_in(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g) { + return lid_hmx_in(hctx, th, ith, nth, g, true); +} + +// Row pair (a, b) quants of two blocks (64 values each, bytes 0..63) to the two tile vectors, scaled by s0, s1 +static inline void lid_q8_0_pair_store(HVX_Vector * restrict dst, HVX_Vector a, HVX_Vector b, HVX_Vector s0, HVX_Vector s1, bool two) { + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(Q6_V_lo_W(Q6_W_vshuff_VVR(b, a, -1))); + dst[0] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_lo_W(qh)), s0); + if (two) { + dst[16] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_hi_W(qh)), s1); + } +} + +// Raw Q8_0 rows [r0, r1) to activation tiles, dequant fused with the row pair interleave. Rows >= nv are zero. +// Per row pair and two blocks: interleave the quants of both rows as bytes, widen, then scale by the (d_r, d_r+1) word. +static inline void lid_hmx_prep_k_q8_0(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + + for (uint32_t r = r0; r < r1; r += 2) { + const uint8_t * p0 = k_raw + r * row_stride; + const uint8_t * p1 = p0 + row_stride; + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + if (n_dot == 4) { + // D = 128: the row is 136 bytes, two aligned loads; scales at halfwords 0, 17, 34, 51 + const HVX_Vector a0 = v0 ? ((const HVX_Vector *) p0)[0] : zero; + const HVX_Vector a1 = v0 ? ((const HVX_Vector *) p0)[1] : zero; + const HVX_Vector b0 = v1 ? ((const HVX_Vector *) p1)[0] : zero; + const HVX_Vector b1 = v1 ? ((const HVX_Vector *) p1)[1] : zero; + + const HVX_VectorPair ds = Q6_W_vshuff_VVR(b0, a0, -SIZEOF_FP16); + + lid_q8_0_pair_store(dst, + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(a0, a0, 2), Q6_V_valign_VVR(a0, a0, 4)), + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(b0, b0, 2), Q6_V_valign_VVR(b0, b0, 4)), + hvx_vec_repl4(Q6_V_lo_W(ds)), hvx_vec_repl4(Q6_V_vror_VR(Q6_V_lo_W(ds), 17 * 4)), true); + lid_q8_0_pair_store(dst + 32, + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(a1, a0, 70), Q6_V_valign_VVR(a1, a0, 72)), + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(b1, b0, 70), Q6_V_valign_VVR(b1, b0, 72)), + hvx_vec_repl4(Q6_V_vror_VR(Q6_V_hi_W(ds), 2 * 4)), hvx_vec_repl4(Q6_V_vror_VR(Q6_V_hi_W(ds), 19 * 4)), true); + continue; + } + + for (uint32_t i = 0; i < n_dot; i += 2) { + const uint8_t * pa = p0 + i * sizeof(block_q8_0); + const uint8_t * pb = p1 + i * sizeof(block_q8_0); + + // quants of blocks i, i + 1 in bytes 0..63, scales d_i at byte 0 and d_i+1 at byte 34 + const HVX_Vector a = v0 ? Q6_V_vmux_QVV(q32, *(const HVX_UVector *) (pa + 2), *(const HVX_UVector *) (pa + 4)) : zero; + const HVX_Vector b = v1 ? Q6_V_vmux_QVV(q32, *(const HVX_UVector *) (pb + 2), *(const HVX_UVector *) (pb + 4)) : zero; + const HVX_Vector da = v0 ? *(const HVX_UVector *) pa : zero; + const HVX_Vector db = v1 ? *(const HVX_UVector *) pb : zero; + const HVX_Vector ds = Q6_V_lo_W(Q6_W_vshuff_VVR(db, da, -SIZEOF_FP16)); + + lid_q8_0_pair_store(dst + i * 16, a, b, hvx_vec_repl4(ds), hvx_vec_repl4(Q6_V_vror_VR(ds, 2 * sizeof(block_q8_0))), i + 1 < n_dot); + } + } +} + +// Raw Q4_0 rows [r0, r1) to activation tiles, same as Q8_0: per row pair and two blocks, one load per row. +// Rows >= nv are zero. +static inline void lid_hmx_prep_k_q4_0(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred lo16 = Q6_Q_or_QQ(Q6_Q_vsetq_R(16), Q6_Q_and_QQn(Q6_Q_vsetq_R(48), q32)); + + for (uint32_t r = r0; r < r1; r += 2) { + const uint8_t * p0 = k_raw + r * row_stride; + const uint8_t * p1 = p0 + row_stride; + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + for (uint32_t i = 0; i < n_dot; i += 2) { + // blocks i, i + 1: scales d_i at byte 0 and d_i+1 at byte 18 + const HVX_Vector da = v0 ? *(const HVX_UVector *) (p0 + i * sizeof(block_q4_0)) : zero; + const HVX_Vector db = v1 ? *(const HVX_UVector *) (p1 + i * sizeof(block_q4_0)) : zero; + const HVX_Vector ds = Q6_V_lo_W(Q6_W_vshuff_VVR(db, da, -SIZEOF_FP16)); + + lid_q8_0_pair_store(dst + i * 16, lid_q4_0_pair_quants(da, q32, lo16), lid_q4_0_pair_quants(db, q32, lo16), + hvx_vec_repl4(ds), hvx_vec_repl4(Q6_V_vror_VR(ds, 2 * sizeof(block_q4_0))), i + 1 < n_dot); + } + } +} + +// F16 rows [r0, r1) to activation tiles: row pairs interleaved. Rows >= nv are zero. +static inline void lid_hmx_prep_k_f16(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + + for (uint32_t r = r0; r < r1; r += 2) { + const HVX_Vector * p0 = (const HVX_Vector *) (k_raw + r * row_stride); + const HVX_Vector * p1 = (const HVX_Vector *) (k_raw + (r + 1) * row_stride); + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + for (uint32_t i = 0; i < n_dot; i += 2) { + const HVX_VectorPair p = Q6_W_vshuff_VVR(v1 ? p1[i / 2] : zero, v0 ? p0[i / 2] : zero, -SIZEOF_FP16); + dst[i * 16] = Q6_V_lo_W(p); + if (i + 1 < n_dot) { + dst[(i + 1) * 16] = Q6_V_hi_W(p); + } + } + } +} + +// K rows of block g (this thread's rows) to activation tiles, one piece per set; set padding rows are zero. +// Not inlined: keeps the stage's registers out of the prep loops. +static __attribute__((noinline)) void lid_hmx_prep_k(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const struct htp_tensor * k = hctx->octx->src[1]; + + const uint32_t n_kv = k->ne[2]; + const uint32_t ks = hctx->kv_seg; + const uint32_t xa = g * hctx->Bc; + const uint32_t n_rows = MIN(hctx->Bc, hctx->n_run * ks - xa); + + uint32_t r0, r1; + lid_hmx_rows(hctx, ith, nth, g, n_rows, &r0, &r1); + + uint8_t * k_raw = hctx->k_raw[g % 2]; + __fp16 * k_t = hctx->k_t[g % 2]; + const bool f32 = k->type == HTP_TYPE_F32; + const bool q8 = k->type == HTP_TYPE_Q8_0; + const bool q4 = k->type == HTP_TYPE_Q4_0; + + for (uint32_t r = r0; r < r1;) { + const uint32_t x = xa + r; + const uint32_t set = fastdiv(x, &hctx->kp->div_kv_seg); + const uint32_t kv = x - set * ks; + const uint32_t end = MIN(r1, (set + 1) * ks - xa); + const uint32_t nv = r + (kv < n_kv ? n_kv - kv : 0); + + if (q8) { + lid_hmx_prep_k_q8_0(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } else if (q4) { + lid_hmx_prep_k_q4_0(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } else { + if (f32) { + for (uint32_t i = r; i < end; ++i) { + lid_convert_row_f32_to_f16(k_raw + i * L->size_k_raw_row, k->ne[0]); + } + } + lid_hmx_prep_k_f16(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } + r = end; + } +} + +// O column of each token (64 kv per gather) + mask -> f32 rows, then one DMA out per segment +static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * dst = octx->dst; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + uint32_t ta, tb; + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + if (ta >= tb) { + return; + } + + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(hctx, g, seg, &n_rows); + + struct htp_thread_trace * tr = th->trace; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) g); + + const uint32_t n_u = hctx->n_u_run; + const uint32_t mu = (n_u + 1) * HMX_FP16_TILE_SIZE - 1; + + int32_t offs[32] __attribute__((aligned(128))); + for (int l = 0; l < 32; l++) { + offs[l] = (l / 16) * n_u * HMX_FP16_TILE_SIZE + (l % 16) * 128; + } + const HVX_Vector v_offs = *(const HVX_Vector *) offs; + + HVX_Vector * gbuf = (HVX_Vector *) (hctx->qs + ith * L->size_qs + 2 * 32 * L->size_qs_row + 32 * L->size_qh_row); + + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector ninf = Q6_V_vsplat_R(0xFF800000); + const HVX_Vector one = Q6_Vh_vsplat_R(0x3C00); + const HVX_Vector hinf = Q6_Vh_vsplat_R(0xFC00); + + const __fp16 * o_base = hctx->o[g % 2]; + const uint8_t * m_buf = hctx->m[g % 4]; + uint8_t * o_buf = hctx->out[g % 2]; + + for (uint32_t i = 0; i < n_seg; ++i) { + const uint32_t q0 = seg[i].x0 / 64; + const uint32_t q1 = (seg[i].x0 + seg[i].nv + 63) / 64; + + for (uint32_t t = ta; t < tb; ++t) { + const uint32_t u = t / 32; + const HVX_Vector off = Q6_Vw_vadd_VwVw(v_offs, Q6_V_vsplat_R((t % 32) * 4)); + + for (uint32_t q = q0; q < q1; ++q) { + const __fp16 * base = o_base + (2 * q * n_u + u) * HMX_FP16_TILE_N_ELMS; + Q6_vgather_ARMVw(gbuf + q - q0, (size_t) base, mu, off); + } + hvx_gather_sync(gbuf + q1 - q0 - 1); + + const HVX_Vector * mv = (const HVX_Vector *) (m_buf + t * L->size_m_row); + HVX_Vector * ov = (HVX_Vector *) (o_buf + t * L->size_o_row); + + for (uint32_t q = q0; q < q1; ++q) { + const HVX_VectorPair s = hvx_vec_f16_to_f32(gbuf[q - q0]); + + // qf32 math does not keep inf: add a finite mask, then put -inf back + const HVX_Vector m_f16 = mv[q]; + const HVX_VectorPred m_inf = Q6_Q_vcmp_eq_VhVh(m_f16, hinf); + const HVX_VectorPair m_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, zero, m_f16)); + const HVX_VectorPair f_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, one, zero)); + + const HVX_Vector acc0 = HVX_OP_ADD_F32(Q6_V_lo_W(s), Q6_V_lo_W(m_f32)); + const HVX_Vector acc1 = HVX_OP_ADD_F32(Q6_V_hi_W(s), Q6_V_hi_W(m_f32)); + ov[2 * q + 0] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_lo_W(f_f32), zero), ninf, acc0); + ov[2 * q + 1] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_hi_W(f_f32), zero), ninf, acc1); + } + } + + const struct lid_hmx_set S = hctx->sets[seg[i].set]; + dma_queue_push(th->dma_q, + dma_make_data(dst->data + S.is * dst->nb[3] + (S.t0 + ta) * dst->nb[1] + seg[i].kv0 * sizeof(float), + o_buf + ta * L->size_o_row + seg[i].x0 * sizeof(float)), + dst->nb[1], L->size_o_row, seg[i].nv * sizeof(float), tb - ta); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) g); +} + +// Number of output DMAs of block g for this thread +static inline uint32_t lid_hmx_n_out(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { + uint32_t ta, tb; + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + if (ta >= tb) { + return 0; + } + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + return lid_hmx_segs(hctx, g, seg, &n_rows); +} + +static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth); + +// Pipeline stage g (HMX runs block g meanwhile): prep block g + 1, extract block g - 1 +static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth) { + const int32_t g = hctx->g; + const int32_t n = (int32_t) hctx->n_blocks; + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + // FIFO order per thread: OUT(g - 3), IN(g + 1), OUT(g - 2). When this stage preps next run Q, also pop + // OUT(g - 2) so the queue is empty for the Q DMAs; else leave it in flight for one more stage. + const int32_t lag = hctx->nx_on ? 2 : 3; + + uint32_t n_pop = 0; + if (lag == 3 && g - 3 >= 0) { + n_pop += lid_hmx_n_out(hctx, ith, nth, g - 3); + } + if (g >= 0 && g + 1 < n) { + n_pop += lid_hmx_in(hctx, th, ith, nth, g + 1, false); + } + if (lag == 2 && g - 2 >= 0) { + n_pop += lid_hmx_n_out(hctx, ith, nth, g - 2); + } + for (uint32_t i = 0; i < n_pop; ++i) { + dma_queue_pop(dma_q); + } + + if (hctx->nx_on && g >= 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g); + lid_hmx_q_chunk(hctx, th, ith, nth); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g); + } + if (g + 2 < n) { + lid_hmx_push_in(hctx, th, ith, nth, g + 2); + } + if (g + 1 < n) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1)); + lid_hmx_prep_k(hctx, ith, nth, g + 1); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1)); + } + if (g - 1 >= 0) { + lid_hmx_extract(hctx, th, ith, nth, g - 1); + } + if (g == n) { + uint32_t n_tail = lid_hmx_n_out(hctx, ith, nth, n - 1); + if (lag == 3 && n >= 2) { + n_tail += lid_hmx_n_out(hctx, ith, nth, n - 2); + } + for (uint32_t i = 0; i < n_tail; ++i) { + dma_queue_pop(dma_q); + } + } +} + +static void lid_hmx_stage_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + lid_hmx_stage((const struct htp_lid_hmx_context *) data, th, ith, nth); +} + +// Push the Q rows of channel tile c of set S, one DMA per token. Returns the number of DMA descriptors. +static uint32_t lid_hmx_push_q(const struct htp_lid_hmx_context * hctx, dma_queue * dma_q, uint8_t * qs, + const struct lid_hmx_set * S, uint32_t c) { + const struct htp_tensor * q = hctx->octx->src[0]; + const uint32_t nh = q->ne[1]; + const uint32_t ch = c * 32; + const uint32_t nr = MIN(32, S->nt * nh - ch); + + // channel ch + r is (token t, head h); a DMA ends at the token end, so the next one starts at h 0 + uint32_t t = fastdiv(ch, &hctx->kp->div_n_head); + uint32_t h = ch - t * nh; + uint32_t n = 0; + for (uint32_t r = 0; r < nr; ++t, h = 0) { + const uint32_t cnt = MIN(nh - h, nr - r); + dma_queue_push(dma_q, dma_make_data(qs + r * hctx->L.size_qs_row, + q->data + S->is * q->nb[3] + (S->t0 + t) * q->nb[2] + h * q->nb[1]), + hctx->L.size_qs_row, q->nb[1], q->ne[0] * sizeof(float), cnt); + r += cnt; + n++; + } + return n; +} + +// Scales (|w| per channel, f16 in the low half of each word) and G tile (sign(w) at row c, column token(c) % 32) +// of one channel tile. w points at the tile's 32 channels, the first one is (token t, head h). +static inline void lid_hmx_build_sc_g(uint8_t * restrict sc, __fp16 * restrict gt, const float * restrict w, + uint32_t t, uint32_t h, uint32_t nr, uint32_t nh) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector wv = Q6_V_vmux_QVV(Q6_Q_vsetq2_R(nr * sizeof(float)), *(const HVX_Vector *) w, zero); + + HVX_Vector * scv = (HVX_Vector *) sc; + scv[0] = hvx_vec_f32_to_f16_shuff(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x7FFFFFFF)), zero); + scv[1] = zero; + + // sign(w) as f16 bits in the low half of each word, then packed: row r in halfword r + const HVX_VectorPred nz = Q6_Q_vcmp_gt_VuwVuw(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x7FFFFFFF)), zero); + const HVX_Vector sgn = Q6_V_vor_VV(Q6_V_vsplat_R(0x3C00), Q6_Vw_vasr_VwR(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x80000000)), 16)); + const HVX_Vector sv = Q6_Vh_vpacke_VwVw(zero, Q6_V_vmux_QVV(nz, sgn, zero)); + + // token column of each row; -1 for missing rows (only rows 0..31 are used) + int16_t tcol[64] __attribute__((aligned(128))) = { 0 }; + for (uint32_t r = 0; r < 32; ++r) { + tcol[r] = r < nr ? (int16_t) (t % 32) : -1; + if (++h == nh) { + h = 0; + t++; + } + } + const HVX_Vector tv = *(const HVX_Vector *) tcol; + + // halfword 2n and 2n + 1 of a G vector are column n + static const int16_t col[64] __attribute__((aligned(128))) = { + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, + 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, + 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, + }; + const HVX_Vector cv = *(const HVX_Vector *) col; + + // vector p holds rows 2p, 2p + 1 + HVX_Vector * gv = (HVX_Vector *) gt; + for (uint32_t p = 0; p < 16; ++p) { + const HVX_Vector tp = hvx_vec_repl4(Q6_V_vror_VR(tv, p * 4)); + const HVX_Vector sp = hvx_vec_repl4(Q6_V_vror_VR(sv, p * 4)); + gv[p] = Q6_V_vmux_QVV(Q6_Q_vcmp_eq_VhVh(cv, tp), sp, zero); + } +} + +// Q buffer slot of set +static inline uint32_t lid_hmx_slot(const struct htp_lid_hmx_context * hctx, uint32_t qbuf, uint32_t set) { + return qbuf * hctx->n_qsets + set; +} + +// Scales and G of channel tile c of a Q slot (needs the slot's w) +static __attribute__((noinline)) void lid_hmx_q_scg(const struct htp_lid_hmx_context * hctx, uint32_t slot, const struct lid_hmx_set * S, uint32_t c) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t nh = hctx->octx->src[0]->ne[1]; + const uint32_t ch = c * 32; + const uint32_t t = fastdiv(ch, &hctx->kp->div_n_head); + + lid_hmx_build_sc_g(hctx->sc + (slot * L->n_ch + c) * 256, hctx->gw + (slot * L->n_ch + c) * HMX_FP16_TILE_N_ELMS, + hctx->w + slot * (L->size_w / sizeof(float)) + ch, t, ch - t * nh, MIN(32, S->nt * nh - ch), nh); +} + +// Staged f32 Q rows of channel tile c to f16, then to the weight tiles of a Q slot +static __attribute__((noinline)) void lid_hmx_q_tiles(const struct htp_lid_hmx_context * hctx, uint32_t slot, const struct lid_hmx_set * S, uint32_t c, + const uint8_t * qs, uint8_t * qh) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t D = hctx->octx->src[0]->ne[0]; + const uint32_t nh = hctx->octx->src[0]->ne[1]; + const uint32_t nr = MIN(32, S->nt * nh - c * 32); + + for (uint32_t r = 0; r < nr; ++r) { + hvx_copy_f16_f32_aa(qh + r * L->size_qh_row, qs + r * L->size_qs_row, D); + } + + // missing channels must be zero (finite): zero the tiles, then scatter only the valid rows + __fp16 * qw = hctx->qw + (slot * L->n_ch + c) * L->n_dot * HMX_FP16_TILE_N_ELMS; + if (nr < 32) { + for (uint32_t i = 0; i < L->n_dot * 16; ++i) { + ((HVX_Vector *) qw)[i] = Q6_V_vzero(); + } + } + hmx_interleave_rows_to_tiles(qw, (const __fp16 *) qh, nr, D, L->size_qh_row / sizeof(__fp16), 0, hex_round_up(nr, 2)); + hvx_gather_sync(qw); +} + +static inline uint32_t lid_hmx_find_valid_gc(const struct htp_lid_hmx_context * hctx, uint32_t gc, uint32_t gc1) { + const struct fastdiv_values * div_n_ch = &hctx->kp->div_n_ch; + const uint32_t n_ch = hctx->L.n_ch; + while (gc < gc1) { + const uint32_t set = fastdiv(gc, div_n_ch); + const uint32_t c = gc - set * n_ch; + if (c < hctx->nx_sets[set].n_ch) { + return gc; + } + gc++; + } + return gc1; +} + +static inline uint32_t lid_hmx_q_chunk_push(const struct htp_lid_hmx_context * hctx, dma_queue * dma_q, + uint8_t * qs, uint32_t qb, uint32_t gc) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * w = octx->src[2]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t nh = octx->src[0]->ne[1]; + + const uint32_t set = fastdiv(gc, &hctx->kp->div_n_ch); + const uint32_t c = gc - set * L->n_ch; + const struct lid_hmx_set * S = &hctx->nx_sets[set]; + + const uint32_t slot = lid_hmx_slot(hctx, qb, set); + const uint32_t ch = c * 32; + const uint32_t ta = fastdiv(ch, &hctx->kp->div_n_head); + const uint32_t tb = fastdiv(MIN(ch + 32, S->nt * nh) - 1, &hctx->kp->div_n_head) + 1; + + float * wd = hctx->w + slot * (L->size_w / sizeof(float)); + dma_queue_push(dma_q, dma_make_data(wd + ta * nh, w->data + S->is * w->nb[3] + (S->t0 + ta) * w->nb[1]), + nh * sizeof(float), w->nb[1], nh * sizeof(float), tb - ta); + return 1 + lid_hmx_push_q(hctx, dma_q, qs, S, c); +} + +// Stage g share of the next run's Q prep (the DMA queue of the thread is empty here) +static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + const uint32_t n_tot = hctx->nx_n_run * L->n_ch; + const uint32_t a = fastdiv(hctx->g * n_tot, &hctx->nx_stg_div); + const uint32_t b = fastdiv((hctx->g + 1) * n_tot, &hctx->nx_stg_div); + const uint32_t per = fastdiv(b - a + octx->n_hvx - 1, &octx->n_hvx_div); + const uint32_t gc0 = MIN(a + ith * per, b); + const uint32_t gc1 = MIN(gc0 + per, b); + const uint32_t qb = hctx->qbuf ^ 1; + + dma_queue * dma_q = th->dma_q; + uint8_t * qs[2]; + qs[0] = hctx->qs + ith * L->size_qs; + qs[1] = qs[0] + 32 * L->size_qs_row; + uint8_t * qh = qs[0] + 2 * 32 * L->size_qs_row; + + uint32_t gc_cur = lid_hmx_find_valid_gc(hctx, gc0, gc1); + if (gc_cur >= gc1) { + return; + } + + const uint32_t gc_next = lid_hmx_find_valid_gc(hctx, gc_cur + 1, gc1); + uint32_t gc_prefetch = gc1; + + uint32_t n_dma[2] = {0, 0}; + n_dma[0] = lid_hmx_q_chunk_push(hctx, dma_q, qs[0], qb, gc_cur); + if (gc_next < gc1) { + n_dma[1] = lid_hmx_q_chunk_push(hctx, dma_q, qs[1], qb, gc_next); + gc_prefetch = lid_hmx_find_valid_gc(hctx, gc_next + 1, gc1); + } + + uint32_t slot = 0; + while (gc_cur < gc1) { + for (uint32_t i = 0; i < n_dma[slot]; ++i) { + dma_queue_pop(dma_q); + } + + const uint32_t set = fastdiv(gc_cur, &hctx->kp->div_n_ch); + const uint32_t c = gc_cur - set * L->n_ch; + const struct lid_hmx_set * S = &hctx->nx_sets[set]; + const uint32_t qslot = lid_hmx_slot(hctx, qb, set); + + lid_hmx_q_scg(hctx, qslot, S, c); + lid_hmx_q_tiles(hctx, qslot, S, c, qs[slot], qh); + + if (gc_prefetch < gc1) { + n_dma[slot] = lid_hmx_q_chunk_push(hctx, dma_q, qs[slot], qb, gc_prefetch); + gc_prefetch = lid_hmx_find_valid_gc(hctx, gc_prefetch + 1, gc1); + } else { + n_dma[slot] = 0; + } + + gc_cur = lid_hmx_find_valid_gc(hctx, gc_cur + 1, gc1); + slot ^= 1; + } +} + +// Q rows to weight tiles, per channel scales |w| and G tiles for all sets of the run, then stage -1 (prep block 0). +// Q work goes to the first threads (thread 0 starts first), the K rows of block 0 to the others. +static void lid_hmx_qprep_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_hmx_context * hctx = (struct htp_lid_hmx_context *) data; + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * w = octx->src[2]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + const uint32_t nh = q->ne[1]; + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + const uint32_t n_tot = hctx->n_run * L->n_ch; + const uint32_t per = fastdiv(n_tot + octx->n_hvx - 1, &octx->n_hvx_div); + const uint32_t gc0 = MIN(ith * per, n_tot); + const uint32_t gc1 = MIN(gc0 + per, n_tot); + + uint8_t * qs = hctx->qs + ith * L->size_qs; + uint8_t * qh = qs + 2 * 32 * L->size_qs_row; + + // all input DMAs in flight at once: w of the sets, Q of the first channel tile, then K and mask of block 0 + uint32_t n_dma = 0; + uint32_t first = gc1; + const struct fastdiv_values * div_n_ch = &hctx->kp->div_n_ch; + for (uint32_t set = fastdiv(gc0, div_n_ch); gc0 < gc1 && set <= fastdiv(gc1 - 1, div_n_ch); ++set) { + const struct lid_hmx_set S = hctx->sets[set]; + + const uint32_t c0 = MAX(gc0, set * L->n_ch) - set * L->n_ch; + const uint32_t c1 = MIN(MIN(gc1, (set + 1) * L->n_ch) - set * L->n_ch, S.n_ch); + if (c0 >= c1) { + continue; + } + first = MIN(first, set * L->n_ch + c0); + + const uint32_t ta = fastdiv(c0 * 32, &hctx->kp->div_n_head); + const uint32_t tb = fastdiv(MIN(c1 * 32, S.nt * nh) - 1, &hctx->kp->div_n_head) + 1; + float * wd = hctx->w + lid_hmx_slot(hctx, hctx->qbuf, set) * (L->size_w / sizeof(float)); + dma_queue_push(dma_q, dma_make_data(wd + ta * nh, w->data + S.is * w->nb[3] + (S.t0 + ta) * w->nb[1]), + nh * sizeof(float), w->nb[1], nh * sizeof(float), tb - ta); + n_dma++; + } + + if (first < gc1) { + const uint32_t set = fastdiv(first, div_n_ch); + const struct lid_hmx_set S = hctx->sets[set]; + n_dma += lid_hmx_push_q(hctx, dma_q, qs, &S, first - set * L->n_ch); + } + + uint32_t n_in = lid_hmx_push_in(hctx, th, ith, nth, 0); + + for (uint32_t i = 0; i < n_dma; ++i) { + dma_queue_pop(dma_q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) gc0); + + for (uint32_t gc = first; gc < gc1; ++gc) { + const uint32_t set = fastdiv(gc, div_n_ch); + const uint32_t c = gc - set * L->n_ch; + const struct lid_hmx_set S = hctx->sets[set]; + if (c >= S.n_ch) { + continue; + } + + uint32_t n_q = 0; + if (gc != first) { + n_q = lid_hmx_push_q(hctx, dma_q, qs, &S, c) + n_in; + n_in = 0; + } + + // scales and G while the Q rows land + const uint32_t slot = lid_hmx_slot(hctx, hctx->qbuf, set); + lid_hmx_q_scg(hctx, slot, &S, c); + + for (uint32_t i = 0; i < n_q; ++i) { + dma_queue_pop(dma_q); + } + + lid_hmx_q_tiles(hctx, slot, &S, c, qs, qh); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) gc0); + + for (uint32_t i = 0; i < n_in; ++i) { + dma_queue_pop(dma_q); + } + + lid_hmx_stage(hctx, th, ith, nth); +} + +// Run start when its Q was prepped in the previous run: block 0 K and mask, then stage -1 +static void lid_hmx_first_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_lid_hmx_context * hctx = (const struct htp_lid_hmx_context *) data; + + const uint32_t n_in = lid_hmx_push_in(hctx, th, ith, nth, 0); + for (uint32_t i = 0; i < n_in; ++i) { + dma_queue_pop(th->dma_q); + } + lid_hmx_stage(hctx, th, ith, nth); +} + +static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kernel_params * kparams) { + const struct htp_tensor * q = octx->src[0]; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_stream = q->ne[3]; + const uint32_t n_hvx = kparams->n_hvx; + const uint32_t n_qsets = kparams->n_qsets; + + struct htp_lid_hmx_context hctx; + memset(&hctx, 0, sizeof(hctx)); + hctx.octx = octx; + hctx.kp = kparams; + hctx.Bc = kparams->kv_block; + hctx.kv_seg = kparams->kv_seg; + hctx.n_tile = kparams->n_tokens_tile; + hctx.n_tt = kparams->n_tt; + hctx.n_qsets = n_qsets; + htp_lid_hmx_vtcm_layout_build(&hctx.L, D, n_head, hctx.n_tile, hctx.Bc, n_qsets * kparams->n_qbuf, n_hvx, + kparams->k_row_size); + + uint8_t * const base = octx->ctx->vtcm_base; + const struct htp_lid_hmx_vtcm_layout * L = &hctx.L; + + hctx.qw = VTCM_LAYOUT_PTR(__fp16, base, L->off_qw); + hctx.gw = VTCM_LAYOUT_PTR(__fp16, base, L->off_g); + hctx.sc = VTCM_LAYOUT_PTR(uint8_t, base, L->off_sc); + hctx.sc_id = VTCM_LAYOUT_PTR(uint8_t, base, L->off_sc_id); + hctx.s = VTCM_LAYOUT_PTR(__fp16, base, L->off_s); + hctx.w = VTCM_LAYOUT_PTR(float, base, L->off_w); + hctx.qs = VTCM_LAYOUT_PTR(uint8_t, base, L->off_qs); + for (int i = 0; i < 2; i++) { + hctx.k_raw[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_k_raw[i]); + hctx.k_t[i] = VTCM_LAYOUT_PTR(__fp16, base, L->off_k_t[i]); + hctx.o[i] = VTCM_LAYOUT_PTR(__fp16, base, L->off_o[i]); + hctx.out[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_out[i]); + } + for (int i = 0; i < 4; i++) { + hctx.m[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_m[i]); + } + + hmx_init_column_scales(hctx.sc_id, Q6_V_vsplat_R(0x3c00)); // identity 1.0 f16 + + hmx_queue_t hmx_q = octx->ctx->hmx_queue; + lid_hmx_job_t job[2]; + + const uint32_t n_tiles = n_stream * hctx.n_tt; + + for (uint32_t tile0 = 0, r = 0; tile0 < n_tiles; tile0 += n_qsets, ++r) { + hctx.tile0 = tile0; + hctx.n_run = MIN(n_qsets, n_tiles - tile0); + hctx.qbuf = r & (kparams->n_qbuf - 1); // n_qbuf is 1 or 2 + + for (uint32_t i = 0; i < hctx.n_run; ++i) { + hctx.sets[i] = lid_hmx_set_init(&hctx, i); + } + + // all sets of a run have the same token count: one set per run, or all tokens in one tile + const struct lid_hmx_set S0 = hctx.sets[0]; + hctx.nt_run = S0.nt; + hctx.n_u_run = S0.n_u; + hctx.n_blocks = hctx.n_run == n_qsets ? kparams->n_kv_blocks : (hctx.n_run * hctx.kv_seg + hctx.Bc - 1) / hctx.Bc; + + const int32_t n = (int32_t) hctx.n_blocks; + const uint32_t n_tot = hctx.n_run * L->n_ch; + const uint32_t per = fastdiv(n_tot + n_hvx - 1, &octx->n_hvx_div); + + hctx.n_qthr = (n_tot + per - 1) / per; + hctx.g = -1; + + const bool q_ready = hctx.nx_on; + if (q_ready) { + hctx.n_qthr = 0; + hctx.nx_on = false; + work_queue_run(octx->ctx->hvx_queue, lid_hmx_first_thread, &hctx, n_hvx); + } else { + work_queue_run(octx->ctx->hvx_queue, lid_hmx_qprep_thread, &hctx, n_hvx); + } + + // with two Q buffers the stages of this run prep the Q of the next one + if (kparams->n_qbuf == 2 && tile0 + n_qsets < n_tiles) { + struct htp_lid_hmx_context nx = hctx; + nx.tile0 = tile0 + n_qsets; + hctx.nx_n_run = MIN(n_qsets, n_tiles - nx.tile0); + for (uint32_t i = 0; i < hctx.nx_n_run; ++i) { + hctx.nx_sets[i] = lid_hmx_set_init(&nx, i); + } + hctx.nx_stg_div = init_fastdiv_values(hctx.n_blocks + 1); + hctx.nx_on = true; + } + + for (int32_t g = 0; g < n; ++g) { + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(&hctx, g, seg, &n_rows); + + lid_hmx_job_t * j = &job[g % 2]; + #pragma clang loop vectorize(disable) + for (uint32_t i = 0; i < n_seg; ++i) { + const uint32_t slot = lid_hmx_slot(&hctx, hctx.qbuf, seg[i].set); + j->seg[i].qw = hctx.qw + slot * L->n_ch * L->n_dot * HMX_FP16_TILE_N_ELMS; + j->seg[i].gw = hctx.gw + slot * L->n_ch * HMX_FP16_TILE_N_ELMS; + j->seg[i].sc = hctx.sc + slot * L->n_ch * 256; + j->seg[i].rt0 = seg[i].x0 / 32; + j->seg[i].n_rt = (seg[i].nv + 31) / 32; + } + j->n_seg = n_seg; + j->k_t = hctx.k_t[g % 2]; + j->sc_id = hctx.sc_id; + j->s = hctx.s; + j->o = hctx.o[g % 2]; + j->n_ch = S0.n_ch; + j->n_u = S0.n_u; + j->n_dot = L->n_dot; + j->n_head = n_head; + j->n_sg = L->n_sg; + hmx_queue_push(hmx_q, hmx_queue_make_desc(lid_hmx_worker, j)); + + // stage g has work only if there is a next block, a previous one or next run Q work + hctx.g = g; + if (g + 1 < n || g >= 1 || hctx.nx_on) { + work_queue_run(octx->ctx->hvx_queue, lid_hmx_stage_thread, &hctx, n_hvx); + } + + hmx_queue_pop(hmx_q); + } + + // last stage only extracts; with one token only thread 0 has work (same token split for nth 1) + hctx.g = n; + const uint32_t n_last = hctx.nt_run == 1 && !hctx.nx_on ? 1 : n_hvx; + work_queue_run(octx->ctx->hvx_queue, lid_hmx_stage_thread, &hctx, n_last); + } + + return HTP_STATUS_OK; +} + +int op_lightning_indexer(struct htp_ops_context * octx) { + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * m = octx->src[3]; + + // no multi-device split yet, device 0 does all the work + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx != 0) { + return HTP_STATUS_OK; + } + + const struct htp_lid_kernel_params * kparams = (const struct htp_lid_kernel_params *) octx->kernel_params; + + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->kernel_type == HTP_LID_KERNEL_HMX) { + return lid_hmx_run(octx, kparams); + } + + struct htp_lid_context lctx; + lctx.octx = octx; + lctx.kp = kparams; + lctx.n_blocks = kparams->n_kv_blocks; + lctx.blocks_per_hvx = kparams->blocks_per_hvx; + + htp_lid_vtcm_layout_build(&lctx.L, q->ne[0], q->ne[1], kparams->n_tokens_tile, kparams->n_hvx, kparams->k_row_size); + + uint8_t * const base = octx->ctx->vtcm_base; + + lctx.spad_q = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_q); + lctx.spad_w = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_w); + lctx.spad_qs = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_qs); + lctx.spad_k = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_k); + lctx.spad_kt = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_kt); + lctx.spad_m = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_m); + lctx.spad_o = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_o); + + const uint32_t n_tokens = q->ne[2]; + const uint32_t n_stream = q->ne[3]; + const uint32_t n_tile = kparams->n_tokens_tile; + const uint32_t n_hvx = kparams->n_hvx; + + for (uint32_t is = 0; is < n_stream; ++is) { + lctx.is = is; + lctx.im3 = fastmodulo(is, m->ne[3], &kparams->div_m_ne3); + + for (uint32_t t0 = 0; t0 < n_tokens; t0 += n_tile) { + lctx.t0 = t0; + lctx.nt = MIN(n_tile, n_tokens - t0); + + work_queue_run(octx->ctx->hvx_queue, lid_prep_thread, &lctx, MIN(n_hvx, lctx.nt * q->ne[1])); + work_queue_run(octx->ctx->hvx_queue, lid_compute_thread, &lctx, n_hvx); + } + } + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h new file mode 100644 index 000000000..14b3503f6 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h @@ -0,0 +1,213 @@ +#ifndef HTP_LIGHTNING_INDEXER_OPS_H +#define HTP_LIGHTNING_INDEXER_OPS_H + +#include <stdint.h> +#include <stddef.h> +#include <stdbool.h> + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define HTP_LID_KV_BLOCK 64 // kv rows per block (HVX and HMX Bc) +#define HTP_LID_TOKENS_MAX 32 // max tokens per Q tile (HVX splat path) +#define HTP_LID_HMX_TOKENS_MAX 512 // HMX can keep more tokens; VTCM decides +#define HTP_LID_HMX_KV_BLOCK_MAX 2048 +#define HTP_LID_HMX_MAX_QSETS 8 // streams in one HMX pipeline run +#define HTP_LID_N_BUF 4 // K / mask / output buffers per thread (two blocks computed, two in flight) + +#ifndef HMX_FP16_TILE_N_COLS +#define HMX_FP16_TILE_N_COLS 32 +#endif +#ifndef HMX_FP16_TILE_SIZE +#define HMX_FP16_TILE_SIZE 2048 +#endif + +enum htp_lid_kernel_type { + HTP_LID_KERNEL_HVX = 0, + HTP_LID_KERNEL_HMX = 1, +}; + +struct htp_lid_kernel_params { + uint8_t kernel_type; // enum htp_lid_kernel_type + uint32_t n_hvx; + uint32_t n_tokens_tile; // tokens per Q tile + uint32_t n_kv_blocks; // HVX: per stream, HMX: per run of n_qsets sets + uint32_t blocks_per_hvx; + uint32_t vtcm_size; + uint32_t k_row_size; // K row bytes + uint32_t kv_block; // HMX: kv rows per block (multiple of 64) + uint32_t n_qsets; // HMX: Q sets kept in VTCM (streams run in one pipeline) + uint32_t n_qbuf; // HMX: Q buffers, 2: the next run's Q is prepped during the current run + uint32_t kv_seg; // HMX: rows per set, n_kv padded to 64 + uint32_t n_tt; // HMX: token tiles per stream + + struct fastdiv_values div_n_head; + struct fastdiv_values div_m_ne3; // mask streams (broadcast) + struct fastdiv_values div_kv_seg; // HMX + struct fastdiv_values div_n_tt; // HMX + struct fastdiv_values div_n_ch; // HMX: channel tiles per Q set +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large"); +#else +_Static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large"); +#endif + +// VTCM layout shared by host (fit check) and device (buffer offsets). +// Shared regions: Q splats (one f16 vector per q element) and head weight splats for the current token tile. +// Per-HVX regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch. +struct htp_lid_vtcm_layout { + size_t off_q; + size_t off_w; + size_t off_qs; + size_t off_k; + size_t off_kt; + size_t off_m; + size_t off_o; + + size_t size_q_row; // f16 Q row, padded + size_t size_qs_row; // f32 Q staging row, padded + size_t size_k_row; // f16 K row, padded + size_t size_k_raw_row; + size_t size_m_row; // f16 mask row of one kv block, padded + size_t size_o_row; // f32 output row of one kv block, padded + + size_t size_qs; // per thread + size_t size_k_buf; // one K buffer + size_t size_kt; // per thread: two transposed K blocks, two 64x64 scratch tiles, Q8_0 block scales + size_t size_m_buf; // one mask buffer + size_t size_o_buf; // one output buffer + + size_t total_bytes; +}; + +static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_hvx, size_t k_row_size) { + L->size_q_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_qs_row = hex_round_up(D * sizeof(float), 128); + L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128); + L->size_m_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(uint16_t), 128); + L->size_o_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(float), 128); + + L->size_qs = 2 * (n_head * L->size_qs_row + hex_round_up(n_head * sizeof(float), 128)) + L->size_q_row; + L->size_k_buf = HTP_LID_KV_BLOCK * L->size_k_raw_row; + L->size_kt = (2 * hex_round_up(D, HTP_LID_KV_BLOCK) + 2 * HTP_LID_KV_BLOCK + D / 32) * 128; + L->size_m_buf = n_tok * L->size_m_row; + L->size_o_buf = n_tok * L->size_o_row; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_q, n_tok * n_head * D * 128); + VTCM_LAYOUT_ALLOC(off, off_w, n_tok * n_head * 128); + VTCM_LAYOUT_ALLOC(off, off_qs, n_hvx * L->size_qs); + VTCM_LAYOUT_ALLOC(off, off_k, n_hvx * HTP_LID_N_BUF * L->size_k_buf); + VTCM_LAYOUT_ALLOC(off, off_kt, n_hvx * L->size_kt); + VTCM_LAYOUT_ALLOC(off, off_m, n_hvx * HTP_LID_N_BUF * L->size_m_buf); + VTCM_LAYOUT_ALLOC(off, off_o, n_hvx * HTP_LID_N_BUF * L->size_o_buf); + + L->total_bytes = off; +} + +static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_hvx, size_t k_row_size) { + struct htp_lid_vtcm_layout L; + htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_hvx, k_row_size); + return L.total_bytes; +} + +// HMX path. K is the activation (kv rows), Q the weights (one output channel per (token, head)). +// mm1: S = relu(K Q^T) * |w| per channel (HMX convert scale + pos clamp) +// mm2: O = S G, G[(t,h), t] = sign(w[t,h]). O is [kv, token], f16. +// Q weights, G, scales and w are per Q set (one per stream when all tokens fit one tile). +struct htp_lid_hmx_vtcm_layout { + size_t off_qw; // Q weight tiles [n_qsets][n_ch][n_dot] + size_t off_g; // G weight tiles [n_qsets][n_ch] + size_t off_sc; // per channel scales, 256 bytes per channel tile, [n_qsets][n_ch] + size_t off_sc_id; // identity scales + size_t off_s; // S tiles of one block and token tile, [n_rt][n_sg] + size_t off_w; // w of the token tile, [n_qsets][n_tok][n_head] f32 + size_t off_qs; // per thread: f32 and f16 staging for 32 Q rows, gather scratch + size_t off_k_raw[2]; // K rows, [Bc][size_k_row] + size_t off_k_t[2]; // K tiles [n_rt][n_dot] + size_t off_o[2]; // O tiles [n_rt][n_u] + size_t off_m[4]; // mask rows [n_tok][size_m_row] + size_t off_out[2]; // f32 output rows [n_tok][size_o_row] + + size_t size_qs_row; // f32 Q row, padded + size_t size_qh_row; // f16 Q row, padded + size_t size_k_row; // f16 K row, padded + size_t size_k_raw_row; + size_t size_m_row; + size_t size_o_row; + size_t size_qs; // per thread + size_t size_w; // per Q set + + uint32_t n_ch; // channel tiles: ceil(n_tok * n_head / 32) + uint32_t n_u; // token tiles: ceil(n_tok / 32) + uint32_t n_dot; // D / 32 + uint32_t n_rt; // kv row tiles per block: Bc / 32 + uint32_t n_sg; // S tiles per (row tile, token tile): min(n_head, n_ch) + + size_t total_bytes; +}; + +static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout * L, + size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_hvx, size_t k_row_size) { + L->n_ch = (uint32_t) ((n_tok * n_head + 31) / 32); + L->n_u = (uint32_t) ((n_tok + 31) / 32); + L->n_dot = (uint32_t) (D / 32); + L->n_rt = (uint32_t) (Bc / 32); + L->n_sg = (uint32_t) (n_head < L->n_ch ? n_head : L->n_ch); + + L->size_qs_row = hex_round_up(D * sizeof(float), 128); + L->size_qh_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128); + L->size_m_row = hex_round_up(Bc * sizeof(uint16_t), 128); + L->size_o_row = hex_round_up(Bc * sizeof(float), 128); + L->size_qs = 2 * 32 * L->size_qs_row + 32 * L->size_qh_row + hex_round_up(Bc / 64 * 128, 128); + L->size_w = hex_round_up(n_tok * n_head * sizeof(float), 128); + + const size_t tile = HMX_FP16_TILE_SIZE; + + // tile regions first: they need 2 KB alignment + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_qw, n_qsets * L->n_ch * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_g, n_qsets * L->n_ch * tile); + VTCM_LAYOUT_ALLOC(off, off_s, (size_t) L->n_rt * L->n_sg * tile); + VTCM_LAYOUT_ALLOC(off, off_k_t[0], (size_t) L->n_rt * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_k_t[1], (size_t) L->n_rt * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_o[0], (size_t) L->n_rt * L->n_u * tile); + VTCM_LAYOUT_ALLOC(off, off_o[1], (size_t) L->n_rt * L->n_u * tile); + VTCM_LAYOUT_ALLOC(off, off_sc, n_qsets * L->n_ch * 256); + VTCM_LAYOUT_ALLOC(off, off_sc_id, 256); + VTCM_LAYOUT_ALLOC(off, off_w, n_qsets * L->size_w); + VTCM_LAYOUT_ALLOC(off, off_qs, n_hvx * L->size_qs); + for (int i = 0; i < 2; i++) { + VTCM_LAYOUT_ALLOC(off, off_k_raw[i], Bc * L->size_k_raw_row); + } + for (int i = 0; i < 4; i++) { + VTCM_LAYOUT_ALLOC(off, off_m[i], n_tok * L->size_m_row); + } + for (int i = 0; i < 2; i++) { + VTCM_LAYOUT_ALLOC(off, off_out[i], n_tok * L->size_o_row); + } + + L->total_bytes = off; +} + +static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_hvx, size_t k_row_size) { + struct htp_lid_hmx_vtcm_layout L; + htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_hvx, k_row_size); + return L.total_bytes; +} + +#ifdef __cplusplus +} +#endif + +#endif /* HTP_LIGHTNING_INDEXER_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index fcb0a7202..ad457d1a6 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -20,6 +20,7 @@ #include <string.h> #include <stdatomic.h> +#include "htp-thread.h" #include "hex-utils.h" #include "dma-queue.h" #include "hmx-queue.h" @@ -28,12 +29,13 @@ #include "ggml-common.h" #include "hex-bitmap.h" #include "htp-ctx.h" +#include "htp-dcache.h" #include "htp-ops.h" -#include "htp-tensor.h" #include "htp_iface.h" #include "work-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "allreduce-ops.h" +#include "dsv4-hc-ops.h" #include "htp-fence.h" #define HMX_QUEUE_CAPACITY 128 @@ -42,8 +44,7 @@ #define WORK_QUEUE_STACK_SIZE 16384 #define MAIN_THREAD_STACK_SIZE 32768 -_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS, - "work-queue thread cap must be >= HTP_MAX_NTHREADS"); +atomic_uint htp_thread_count = 0; struct htp_handle { struct htp_context * ctx; @@ -287,15 +288,15 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) { } static void vtcm_acquire(struct htp_context * ctx) { - if (!ctx->vtcm_valid) { + if (!atomic_load(&ctx->vtcm_valid)) { int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u); if (err != 0) { FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err); abort(); } - ctx->vtcm_needs_release = false; - ctx->vtcm_valid = true; + atomic_store(&ctx->vtcm_needs_release, false); + atomic_store(&ctx->vtcm_valid, true); // Drop the priority to make sure we get the release callback from other GGML-HTP and QNN-HTP sessions HAP_compute_res_update_priority(ctx->vtcm_rctx, ctx->thread_prio + 10); @@ -303,22 +304,33 @@ static void vtcm_acquire(struct htp_context * ctx) { } static void vtcm_release(struct htp_context * ctx) { - if (ctx->vtcm_valid) { - ctx->vtcm_valid = false; - ctx->vtcm_needs_release = false; + if (atomic_load(&ctx->vtcm_valid)) { + atomic_store(&ctx->vtcm_valid, false); + atomic_store(&ctx->vtcm_needs_release, false); HAP_compute_res_release_cached(ctx->vtcm_rctx); } } static int vtcm_release_callback(unsigned int rctx, void * state) { struct htp_context * ctx = (struct htp_context *) state; - ctx->vtcm_needs_release = true; + atomic_store(&ctx->vtcm_needs_release, true); return 0; } -static int vtcm_alloc(struct htp_context * ctx) { +static unsigned int vtcm_query_size(void) { unsigned int vtcm_size = 8 * 1024 * 1024; // 8MB default - HAP_compute_res_query_VTCM(0, &vtcm_size, NULL, NULL, NULL); + unsigned int vtcm_avail; + compute_res_vtcm_page_t total_layout, avail_layout; + // some images (e.g. FuSa cDSP) fail the query when the layout out-params are NULL + HAP_compute_res_query_VTCM(0, &vtcm_size, &total_layout, &vtcm_avail, &avail_layout); + return vtcm_size; +} + +static int vtcm_alloc(struct htp_context * ctx, uint64_t size_override) { + unsigned int vtcm_size = vtcm_query_size(); + if (size_override) { + vtcm_size = (unsigned int) size_override; + } compute_res_attr_t attr; HAP_compute_res_attr_init(&attr); @@ -345,8 +357,8 @@ static int vtcm_alloc(struct htp_context * ctx) { ctx->vtcm_base = (uint8_t *) vtcm_ptr; ctx->vtcm_size = vtcm_size; ctx->vtcm_rctx = rctx; - ctx->vtcm_valid = false; - ctx->vtcm_needs_release = false; + atomic_store(&ctx->vtcm_valid, false); + atomic_store(&ctx->vtcm_needs_release, false); return 0; } @@ -363,12 +375,91 @@ static void htp_main_thread(void * context); static void htp_packet_callback(dspqueue_t queue, int error, void * context); static void htp_error_callback(dspqueue_t queue, int error, void * context); -AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem) { +// Non-BURST modes leave DCVS enabled and keep the HTP awake. +// BURST pins all corners to MAX and disables DCVS. +// Corner constants are from HAP_power.h (HAP_dcvs_voltage_corner_t / HAP_dcvs_exp_vcorner_t). +struct htp_power_profile { + boolean dcvs_enable; + uint32_t bus_min_corner; + uint32_t bus_max_corner; + uint32_t bus_target_corner; + uint32_t core_min_corner; + uint32_t core_max_corner; + uint32_t core_target_corner; + boolean sleep_disable; + uint32_t hmx_min_corner; // only used when __HVX_ARCH__ >= 75 + uint32_t hmx_max_corner; + uint32_t hmx_target_corner; + uint32_t hmx_perf_mode; +}; + +#define HTP_POWER_PROFILE_ENTRY(min_c, max_c, hmx_c) \ + { \ + .dcvs_enable = TRUE, \ + .bus_min_corner = (min_c), \ + .bus_max_corner = (max_c), \ + .bus_target_corner = (max_c), \ + .core_min_corner = (min_c), \ + .core_max_corner = (max_c), \ + .core_target_corner = (max_c), \ + .sleep_disable = TRUE, \ + .hmx_min_corner = (hmx_c), \ + .hmx_max_corner = (hmx_c), \ + .hmx_target_corner = (hmx_c), \ + .hmx_perf_mode = HAP_CLK_PERF_HIGH, \ + } + +static const struct htp_power_profile htp_power_profiles[HTP_POWER_MODE_COUNT] = { + [HTP_POWER_MODE_LOW_POWER_SAVER] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS2, HAP_DCVS_VCORNER_SVS2, HAP_DCVS_EXP_VCORNER_MIN_SVS), + [HTP_POWER_MODE_POWER_SAVER] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS, HAP_DCVS_VCORNER_SVS_PLUS, HAP_DCVS_EXP_VCORNER_SVS), + [HTP_POWER_MODE_HIGH_POWER_SAVER] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS_PLUS, HAP_DCVS_VCORNER_NOM, HAP_DCVS_EXP_VCORNER_SVS_L2), + [HTP_POWER_MODE_LOW_BALANCED] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM, HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_EXP_VCORNER_NOM), + [HTP_POWER_MODE_BALANCED] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_VCORNER_TURBO, HAP_DCVS_EXP_VCORNER_NOM_L1), + [HTP_POWER_MODE_HIGH_PERFORMANCE] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_TURBO, HAP_DCVS_VCORNER_TURBO_PLUS, HAP_DCVS_EXP_VCORNER_TUR_L2), + [HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_VCORNER_TURBO_PLUS, HAP_DCVS_EXP_VCORNER_TUR_L2), + [HTP_POWER_MODE_BURST] = { + .dcvs_enable = FALSE, + .bus_min_corner = HAP_DCVS_VCORNER_MAX, + .bus_max_corner = HAP_DCVS_VCORNER_MAX, + .bus_target_corner = HAP_DCVS_VCORNER_MAX, + .core_min_corner = HAP_DCVS_VCORNER_MAX, + .core_max_corner = HAP_DCVS_VCORNER_MAX, + .core_target_corner = HAP_DCVS_VCORNER_MAX, + .sleep_disable = TRUE, + .hmx_min_corner = HAP_DCVS_EXP_VCORNER_MAX, + .hmx_max_corner = HAP_DCVS_EXP_VCORNER_MAX, + .hmx_target_corner = HAP_DCVS_EXP_VCORNER_MAX, + .hmx_perf_mode = HAP_CLK_PERF_HIGH, + }, +}; + +#undef HTP_POWER_PROFILE_ENTRY + +AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t vtcm_size, uint64_t max_vmem, uint32_t power_mode) { struct htp_handle * h = (struct htp_handle *) handle; if (!h) { return AEE_EBADPARM; } + if (power_mode >= HTP_POWER_MODE_COUNT) { + FARF(ERROR, "invalid power-mode %u, falling back to burst", (unsigned) power_mode); + power_mode = HTP_POWER_MODE_BURST; + } + const struct htp_power_profile * pwr = &htp_power_profiles[power_mode]; + FARF(ALWAYS, "power-mode %u : dcvs-enable %d bus[%u,%u,%u] core[%u,%u,%u] sleep-disable %d hmx[%u,%u,%u]", + (unsigned) power_mode, (int) pwr->dcvs_enable, + (unsigned) pwr->bus_min_corner, (unsigned) pwr->bus_max_corner, (unsigned) pwr->bus_target_corner, + (unsigned) pwr->core_min_corner, (unsigned) pwr->core_max_corner, (unsigned) pwr->core_target_corner, + (int) pwr->sleep_disable, + (unsigned) pwr->hmx_min_corner, (unsigned) pwr->hmx_max_corner, (unsigned) pwr->hmx_target_corner); + if (h->ctx) { FARF(ERROR, "Queue already open"); return AEE_EITEMBUSY; @@ -400,17 +491,23 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp qurt_sysenv_max_hthreads_t hw_threads; qurt_sysenv_get_max_hw_threads(&hw_threads); uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + uint32_t hw_nth = hw_threads.max_hthreads; + if (hw_nth > HTP_MAX_NTHREADS) { + hw_nth = HTP_MAX_NTHREADS; + } if (n_hvx == 0) { n_hvx = hw_nhvx; } - if (n_hvx > hw_threads.max_hthreads) { - n_hvx = hw_threads.max_hthreads; + if (n_hvx > HTP_MAX_NHVX) { + n_hvx = HTP_MAX_NHVX; } - if (n_hvx > HTP_MAX_NTHREADS) { - n_hvx = HTP_MAX_NTHREADS; + if (n_hvx > hw_nth - 1) { + n_hvx = hw_nth - 1; } + uint32_t n_threads = hw_nth; + // layout segments of our contiguous block // 1. htp_context : sits at the base (block is 4K-aligned via memalign below) @@ -429,13 +526,13 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp size_t wq_align = work_queue_alignof(); size_t offset_wq = hex_align_up(offset, wq_align); size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); - offset = offset_wq + size_wq; + offset = offset_wq + size_wq; // 4. dma_queue size_t dma_align = dma_queue_alignof(); size_t offset_dma = hex_align_up(offset, dma_align); size_t size_dma = 0; - for (uint32_t i = 0; i < n_hvx; i++) { + for (uint32_t i = 0; i < n_threads; i++) { size_dma = hex_align_up(size_dma, dma_queue_alignof()); size_dma += dma_queue_sizeof(256); } @@ -470,7 +567,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp ctx->max_vmem = max_vmem; ctx->dsp_queue = dsp_queue; - err = vtcm_alloc(ctx); + err = vtcm_alloc(ctx, vtcm_size); if (err != AEE_SUCCESS) { FARF(ERROR, "Unable to allocate VTCM"); htp_iface_stop(handle); @@ -499,17 +596,17 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp request.type = HAP_power_set_DCVS_v3; request.dcvs_v3.set_dcvs_enable = TRUE; - request.dcvs_v3.dcvs_enable = FALSE; + request.dcvs_v3.dcvs_enable = pwr->dcvs_enable; request.dcvs_v3.set_bus_params = TRUE; - request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.bus_params.min_corner = pwr->bus_min_corner; + request.dcvs_v3.bus_params.max_corner = pwr->bus_max_corner; + request.dcvs_v3.bus_params.target_corner = pwr->bus_target_corner; request.dcvs_v3.set_core_params = TRUE; - request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.core_params.min_corner = pwr->core_min_corner; + request.dcvs_v3.core_params.max_corner = pwr->core_max_corner; + request.dcvs_v3.core_params.target_corner = pwr->core_target_corner; request.dcvs_v3.set_sleep_disable = TRUE; - request.dcvs_v3.sleep_disable = TRUE; + request.dcvs_v3.sleep_disable = pwr->sleep_disable; #if (__HEXAGON_ARCH__ >= 79) HAP_set_dcvs_v3_protected_bus_corners(&request, 1); @@ -537,10 +634,10 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp request.hmx_v2.set_power = TRUE; request.hmx_v2.power_up = TRUE; request.hmx_v2.set_clock = TRUE; - request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH; + request.hmx_v2.target_corner = pwr->hmx_target_corner; + request.hmx_v2.min_corner = pwr->hmx_min_corner; + request.hmx_v2.max_corner = pwr->hmx_max_corner; + request.hmx_v2.perf_mode = pwr->hmx_perf_mode; FARF(ALWAYS, "Setting HMX clock\n"); err = HAP_power_set((void *) ctx, &request); if (err != AEE_SUCCESS) { @@ -566,16 +663,9 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp } #endif - ctx->hmx_enabled = n_hmx; - ctx->hmx_queue = NULL; - if (n_hmx) { - void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); - ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]); - } - FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); - - ctx->n_threads = n_hvx; - ctx->n_threads_div = init_fastdiv_values(ctx->n_threads); + ctx->n_threads = n_threads; + ctx->n_hvx = n_hvx; + ctx->n_hvx_div = init_fastdiv_values(ctx->n_hvx); // Initialize DMA queues uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma); @@ -587,11 +677,21 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp dma_ptr_curr += size_dma_q; } - ctx->ddr_spad_size = 512 * 1024; // 512 KB + ctx->ddr_spad_size = HTP_DDR_SPAD_SIZE; ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size); + htp_thread_reset(); + void * wq_ptr = (void *) ((uintptr_t) block + offset_wq); - ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + ctx->hvx_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE, ctx->dma, ctx->trace); + + ctx->hmx_enabled = n_hmx; + ctx->hmx_queue = NULL; + if (n_hmx) { + void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); + ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, ctx->dma, ctx->trace); + } + FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); ctx->main_stack = NULL; ctx->main_thread = 0; @@ -617,7 +717,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp } FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n", - sess_id, hw_nhvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio); + sess_id, ctx->n_hvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio); return AEE_SUCCESS; } @@ -642,7 +742,7 @@ AEEResult htp_iface_stop(remote_handle64 handle) { return err; } - work_queue_free(ctx->work_queue); + work_queue_free(ctx->hvx_queue); for (int i = 0; i < ctx->n_threads; i++) { dma_queue_free(ctx->dma[i]); @@ -678,22 +778,24 @@ AEEResult htp_iface_hwinfo(remote_handle64 handle, uint32_t * n_threads, uint32_ qurt_sysenv_get_max_hw_threads(&hw_threads); uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + uint32_t n_threads_val = hw_threads.max_hthreads; + if (n_threads_val > HTP_MAX_NTHREADS) { + n_threads_val = HTP_MAX_NTHREADS; + } + uint32_t n_hvx_val = hw_nhvx; - if (n_hvx_val > hw_threads.max_hthreads) { - n_hvx_val = hw_threads.max_hthreads; + if (n_hvx_val > HTP_MAX_NHVX) { + n_hvx_val = HTP_MAX_NHVX; } - if (n_hvx_val > HTP_MAX_NTHREADS) { - n_hvx_val = HTP_MAX_NTHREADS; + if (n_hvx_val > n_threads_val - 1) { + n_hvx_val = n_threads_val - 1; } - // for now we force n_threads == n_hvx - *n_threads = n_hvx_val; + *n_threads = n_threads_val; *n_hvx = n_hvx_val; *n_hmx = 1; - uint32_t vtcm_sz = 8 * 1024 * 1024; // 8MB default fallback - HAP_compute_res_query_VTCM(0, (unsigned int *)&vtcm_sz, NULL, NULL, NULL); - *vtcm_size = vtcm_sz; + *vtcm_size = vtcm_query_size(); return AEE_SUCCESS; } @@ -756,7 +858,7 @@ static int op_fence(struct htp_ops_context * octx) { atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; if (mode == 1) { - htp_flush_dirty_ranges(ctx); + htp_dcache_flush_dirty_ranges(ctx); htp_mdev_group_barrier(octx); @@ -842,6 +944,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_NORM: case HTP_OP_RMS_NORM: case HTP_OP_RMS_NORM_MUL: + case HTP_OP_RMS_NORM_MUL_ADD: case HTP_OP_SCALE: case HTP_OP_CLAMP: case HTP_OP_LEAKY_RELU: @@ -851,6 +954,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_UNARY_SIGMOID: case HTP_OP_UNARY_SILU: case HTP_OP_UNARY_GELU: + case HTP_OP_UNARY_GELU_QUICK: case HTP_OP_UNARY_GELU_ERF: case HTP_OP_UNARY_NEG: case HTP_OP_UNARY_EXP: @@ -859,7 +963,10 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_UNARY_LOG: case HTP_OP_UNARY_RELU: case HTP_OP_UNARY_STEP: + case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: case HTP_OP_L2_NORM: + case HTP_OP_HADAMARD: return op_unary(octx); case HTP_OP_GLU_SWIGLU: @@ -868,7 +975,9 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_GLU_GEGLU: case HTP_OP_GLU_GEGLU_QUICK: case HTP_OP_GLU_GEGLU_ERF: - return op_activations(octx); + case HTP_OP_GLU_REGLU: + case HTP_OP_GLU_SIGMOID: + return op_glu(octx); case HTP_OP_SOFTMAX: return op_softmax(octx); @@ -913,6 +1022,9 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_CUMSUM: return op_cumsum(octx); + case HTP_OP_TOP_K_MASK: + return op_top_k_mask(octx); + case HTP_OP_FILL: return op_fill(octx); @@ -928,18 +1040,46 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_IM2COL: return op_im2col(octx); + case HTP_OP_IM2COL_3D: + return op_im2col_3d(octx); + + case HTP_OP_CONV_2D: + case HTP_OP_CONV_3D: + return op_conv_2d(octx); + + case HTP_OP_CONV_2D_DW: + return op_conv_2d_dw(octx); + case HTP_OP_ROLL: return op_roll(octx); + case HTP_OP_POOL_2D: + return op_pool_2d(octx); + + case HTP_OP_POOL_1D: + return op_pool_1d(octx); + + case HTP_OP_DSV4_HC_PRE: + return op_dsv4_hc_pre(octx); + + case HTP_OP_DSV4_HC_POST: + return op_dsv4_hc_post(octx); + case HTP_OP_CONCAT: return op_concat(octx); case HTP_OP_GATED_DELTA_NET: return op_gated_delta_net(octx); + case HTP_OP_LIGHTNING_INDEXER: + return op_lightning_indexer(octx); + case HTP_OP_TRI: return op_unary(octx); + case HTP_OP_UPSCALE: + return op_upscale(octx); + case HTP_OP_INVALID: break; } @@ -1100,10 +1240,10 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) { memcpy(octx->op_params, op->params, sizeof(octx->op_params)); memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params)); - octx->flags = op->flags; - octx->op = op->opcode; - octx->n_threads = octx->ctx->n_threads; - octx->n_threads_div = octx->ctx->n_threads_div; + octx->flags = op->flags; + octx->op = op->opcode; + octx->n_hvx = octx->ctx->n_hvx; + octx->n_hvx_div = octx->ctx->n_hvx_div; FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags); @@ -1122,7 +1262,7 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs src->ne[0], src->ne[1], src->ne[2], src->ne[3]); } - htp_tensor_flush_all(octx->ctx, octx->src, HTP_OP_MAX_INPUTS); + htp_dcache_flush_dirty_tensors(octx->ctx, octx->src, HTP_OP_MAX_INPUTS); // Prep output tensors for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { @@ -1138,7 +1278,7 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } - htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); + htp_dcache_mark_dirty_tensors(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); htp_mdev_group_barrier(octx); @@ -1148,8 +1288,6 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs octx->src0_spad.src = NULL; octx->src1_spad.src = NULL; - octx->src2_spad.src = NULL; - octx->src3_spad.src = NULL; octx->dst_spad.src = NULL; return octx->status; @@ -1167,7 +1305,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops; - const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc); + const uint32_t tr_size = ctx->n_threads * req->n_traces * sizeof(struct htp_trace_desc); if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) { FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size); @@ -1190,7 +1328,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r memset(ctx->trace, 0, sizeof(ctx->trace)); if (ctx->profiler == HTP_PROF_TRACE) { struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size); - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + for (uint32_t t = 0; t < ctx->n_threads; t++) { ctx->trace[t].events = &trace_events[t * req->n_traces]; ctx->trace[t].max_events = req->n_traces; } @@ -1210,14 +1348,11 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r prep_tensors(ctx, bufs, tens, n_tens); struct htp_ops_context *octx = &ctx->octx; - memset(octx, 0, sizeof(*octx)); - octx->n_threads = ctx->n_threads; - octx->n_threads_div = ctx->n_threads_div; - octx->ctx = ctx; + htp_ops_context_reset(octx, ctx); mdev_group_init(ctx, req); - work_queue_wakeup(ctx->work_queue); + work_queue_wakeup(ctx->hvx_queue); if (ctx->hmx_queue) { hmx_queue_wakeup(ctx->hmx_queue); } @@ -1250,7 +1385,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r hmx_queue_suspend(ctx->hmx_queue); hmx_queue_flush(ctx->hmx_queue); } - work_queue_suspend(ctx->work_queue); + work_queue_suspend(ctx->hvx_queue); // Flush remaining dirty tensors at the end of the batch htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); @@ -1273,7 +1408,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r rsp.cycles_stop = batch_prof.cycles_stop; if (ctx->profiler == HTP_PROF_TRACE) { - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + for (uint32_t t = 0; t < ctx->n_threads; t++) { rsp.n_traces[t] = ctx->trace[t].count; } } @@ -1299,7 +1434,7 @@ static void process_ops(struct htp_context * ctx) { vtcm_acquire(ctx); - while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) { + while (!atomic_load(&ctx->vtcm_needs_release) && !atomic_load(&ctx->killed)) { struct htp_opbatch_req req; uint32_t r_size = sizeof(req); diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 9dfd35649..2d0750954 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -16,6 +16,7 @@ #include "hvx-dump.h" #include "hvx-arith.h" #include "hvx-reduce.h" +#include "hvx-copy.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -27,8 +28,8 @@ typedef struct { float *dst; - dma_addr_t src2_addr; - size_t src2_bytes; + dma_addr_t bias_addr; + size_t bias_bytes; dma_addr_t act_dma_addr; dma_addr_t weight; dma_queue * weight_dma; @@ -36,9 +37,14 @@ typedef struct { int k; int n; int act_stride; + uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1 int weight_stride; + int weight_type; int dst_stride; - uint32_t src2_stride; + int dst_cols; + uint32_t bias_stride; + int tile_size; + int aligned_tile_size; int ne02; int ne03; int ne12; @@ -47,15 +53,15 @@ typedef struct { size_t src0_nb3; size_t act_nb2; size_t act_nb3; - size_t src2_nb2; - size_t src2_nb3; + size_t bias_nb2; + size_t bias_nb3; size_t dst_nb2; size_t dst_nb3; int r2; int r3; struct fastdiv_values div_r2; struct fastdiv_values div_r3; -} hmx_mm_f16_f32_batched_params_t; +} hmx_mm_batched_params_t; static bool htp_matmul_has_extended_weight(const struct htp_ops_context * octx, uint32_t n_weights) { for (uint32_t i = 0; i < n_weights; ++i) { @@ -88,8 +94,13 @@ struct htp_mm_context { const void * restrict vy, uint32_t valid_rows, const float * restrict sz); + void (*vec_dot_32x2)(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx, + const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, + const float * restrict sz0, const float * restrict sz1); + // Precomputed values - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t src0_row_start; uint32_t src0_row_end; uint32_t src0_row_size_padded; @@ -104,12 +115,12 @@ struct htp_mm_context { struct fastdiv_values mm_div_ne11; // Precomputed block-parallel quantization values - uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_ib_first[HTP_MAX_NHVX]; + uint32_t quant_ib_last[HTP_MAX_NHVX]; + uint32_t quant_r[HTP_MAX_NHVX]; + uint32_t quant_c[HTP_MAX_NHVX]; uint32_t n_quant_tasks; - uint32_t n_quant_rows_per_thread; + uint32_t n_quant_rows_per_hvx; // Fields for scattered mapping & HMX support in MUL_MAT_ID const uint32_t * matrix_row_counts; @@ -118,37 +129,32 @@ struct htp_mm_context { // Dynamic VTCM pointers allocated sequentially uint8_t * vtcm_src0; - uint8_t * vtcm_src1; - uint8_t * vtcm_src2; - uint8_t * vtcm_src3; + uint8_t * vtcm_act; + uint8_t * vtcm_bias; uint8_t * vtcm_dst; uint8_t * vtcm_act_raw; // Cached strides uint32_t vtcm_src0_stride; - uint32_t vtcm_src1_stride; - uint32_t vtcm_src2_stride; - uint32_t vtcm_src3_stride; + uint32_t vtcm_act_stride; uint32_t vtcm_act_raw_stride; // Cached thread offsets/sizes - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_src1_size_per_thread; - uint32_t vtcm_src2_size_per_thread; - uint32_t vtcm_src3_size_per_thread; - uint32_t vtcm_dst_size_per_thread; + uint32_t vtcm_src0_size_per_hvx; + uint32_t vtcm_act_size_per_hvx; + uint32_t vtcm_dst_size_per_hvx; }; static int htp_mm_init_context( struct htp_ops_context * octx, const struct htp_mm_kernel_params * kparams ) { - if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, (uint32_t) kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } if (kparams->n_hmx) { - if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) { + if (kparams->n_act_hvx <= 0 || kparams->n_act_hvx > (int32_t) octx->n_hvx) { return HTP_STATUS_INVAL_PARAMS; } } @@ -190,6 +196,7 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { const struct htp_tensor * restrict src1 = octx->src[1]; \ const struct htp_tensor * restrict src2 = octx->src[2]; \ const struct htp_tensor * restrict dst = octx->dst; \ + const struct htp_tensor * restrict act = src1; \ \ const uint32_t ne00 = src0->ne[0]; \ const uint32_t ne01 = src0->ne[1]; \ @@ -229,8 +236,8 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { #define htp_matmul_preamble \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; \ + dma_queue * dma_q = th->dma_q; \ + uint32_t src0_nrows_per_hvx = mmctx->src0_nrows_per_hvx; \ htp_matmul_tensors_preamble; @@ -241,319 +248,301 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { #include "hvx-mm-kernels-float.h" #include "hmx-mm-kernels-tiled.h" -// Specialized repacked matmul macros -#define MATMUL_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ -static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ - htp_matmul_preamble; \ - \ - const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ - const uint32_t src1_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); \ - const uint32_t cur_m_start = mmctx->cur_m_start; \ - \ - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ - const uint32_t n_prefetch = kparams->n_prefetch; \ - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ - \ - const size_t dst_row_size = nb1; \ - const size_t src1_row_size = nb11; \ - const size_t src1_stride = mmctx->vtcm_src1_stride; \ - const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0; \ - \ - uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; \ - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ - uint8_t * restrict src1_data = mmctx->vtcm_src1; \ - \ - const dma_addr_t src0_row = src0->data; \ - \ - const uint32_t tile_size = TILE_SIZE; \ - const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ - \ - uint32_t n_k_tiles_w = ne00 / 32; \ - uint32_t n_k_tiles_a = ne10 / 32; \ - uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ - uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ - \ - uint32_t ct_start = src0_start_row / 32; \ - uint32_t ct_end = (src0_end_row + 31) / 32; \ - \ - uint32_t push_ct = ct_start; \ - if (src0_start_row < src0_end_row) { \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - } \ - } \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ - const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; \ - \ - int valid_rows = (int)ne0 - (int)(ct * 32); \ - valid_rows = MIN(32, MAX(0, valid_rows)); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - uint32_t ir1 = 0; \ - for (; ir1 + 1 < src1_nrows; ir1 += 2) { \ - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \ - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \ - float * restrict dst_row0 = (float *) (dst->data + ((cur_m_start + ir1+0) * dst_row_size)); \ - float * restrict dst_row1 = (float *) (dst->data + ((cur_m_start + ir1+1) * dst_row_size)); \ - \ - float * dst_ptr0 = &dst_row0[ct * 32]; \ - float * dst_ptr1 = &dst_row1[ct * 32]; \ - \ - const float * src2_ptr0 = NULL; \ - const float * src2_ptr1 = NULL; \ - if (src2) { \ - const float * restrict src2_row0 = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1+0) * src2_stride)); \ - const float * restrict src2_row1 = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1+1) * src2_stride)); \ - src2_ptr0 = &src2_row0[ct * 32]; \ - src2_ptr1 = &src2_row1[ct * 32]; \ - } \ - DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, src2_ptr0, src2_ptr1); \ - } \ - \ - for (; ir1 < src1_nrows; ++ir1) { \ - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \ - float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); \ - float * dst_ptr = &dst_row[ct * 32]; \ - \ - const float * src2_ptr = NULL; \ - if (src2) { \ - const float * restrict src2_row = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1) * src2_stride)); \ - src2_ptr = &src2_row[ct * 32]; \ - } \ - DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, src2_ptr); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - \ - if (push_ct < ct_end) { \ - dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - push_ct++; \ - } \ - } \ +static void hvx_mm_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; + + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); + const uint32_t cur_m_start = mmctx->cur_m_start; + + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); + + struct htp_thread_trace * tr = th->trace; + + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + + const size_t dst_row_size = nb1; + const size_t act_row_size = nb11; + const size_t act_stride = mmctx->vtcm_act_stride; + const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0; + + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; + + const dma_addr_t src0_row = src0->data; + + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); + + uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); + uint32_t tile_row_stride = n_k_tiles_w * tile_size; + uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; + + uint32_t ct_start = src0_start_row / 32; + uint32_t ct_end = (src0_end_row + 31) / 32; + + uint32_t push_ct = ct_start; + if (src0_start_row < src0_end_row) { + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } + } + + if (src0_start_row >= src0_end_row) { + return; + } + + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; + + int valid_rows = (int)ne0 - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + uint32_t ir1 = 0; + for (; ir1 + 1 < act_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (ir1+0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (ir1+1) * act_stride); + float * restrict dst_row0 = (float *) (dst->data + ((cur_m_start + ir1+0) * dst_row_size)); + float * restrict dst_row1 = (float *) (dst->data + ((cur_m_start + ir1+1) * dst_row_size)); + + float * dst_ptr0 = &dst_row0[ct * 32]; + float * dst_ptr1 = &dst_row1[ct * 32]; + + const float * src2_ptr0 = NULL; + const float * src2_ptr1 = NULL; + if (src2) { + const float * restrict src2_row0 = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1+0) * src2_stride)); + const float * restrict src2_row1 = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1+1) * src2_stride)); + src2_ptr0 = &src2_row0[ct * 32]; + src2_ptr1 = &src2_row1[ct * 32]; + } + mmctx->vec_dot_32x2(ne10, dst_ptr0, dst_ptr1, w_tile, act_col0, act_col1, valid_rows, src2_ptr0, src2_ptr1); + } + + for (; ir1 < act_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + ir1 * act_stride); + float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); + float * dst_ptr = &dst_row[ct * 32]; + + const float * src2_ptr = NULL; + if (src2) { + const float * restrict src2_row = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1) * src2_stride)); + src2_ptr = &src2_row[ct * 32]; + } + mmctx->vec_dot_32x1(ne10, dst_ptr, w_tile, act_col, valid_rows, src2_ptr); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); + + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } } -#define MATVEC_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X1) \ -static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ - htp_matmul_preamble; \ - \ - const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ - \ - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ - const uint32_t n_prefetch = kparams->n_prefetch; \ - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ - \ - const size_t dst_row_size = nb1; \ - const size_t src1_row_size = nb11; \ - const size_t src1_stride = mmctx->vtcm_src1_stride; \ - \ - uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; \ - uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ - uint8_t * src1_data = mmctx->vtcm_src1; \ - \ - float * tmp = (float *) vtcm_dst_ptr; \ - \ - const dma_addr_t src0_row = src0->data; \ - \ - const uint8_t * restrict src1_col = (const uint8_t *) src1_data; \ - float * restrict dst_col = (float *) dst->data; \ - \ - const uint32_t tile_size = TILE_SIZE; \ - const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ - \ - uint32_t n_k_tiles_w = ne00 / 32; \ - uint32_t n_k_tiles_a = ne10 / 32; \ - uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ - uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ - \ - uint32_t ct_start = src0_start_row / 32; \ - uint32_t ct_end = (src0_end_row + 31) / 32; \ - \ - uint32_t push_ct = ct_start; \ - if (src0_start_row < src0_end_row) { \ - if (src2) { \ - float * vtcm_src2_ptr = (float *) mmctx->vtcm_src2 + src0_start_row; \ - const dma_addr_t src2_addr = src2->data + src0_start_row * sizeof(float); \ - int slice_size = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; \ - if (slice_size > 0) { \ - dma_queue_push(dma_q, dma_make_data(vtcm_src2_ptr, src2_addr), \ - slice_size * sizeof(float), slice_size * sizeof(float), slice_size * sizeof(float), 1); \ - dma_queue_pop_nowait(dma_q); \ - } \ - } \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - } \ - } \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ - const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; \ - \ - float * dst_ptr = &tmp[ct * 32 - src0_start_row]; \ - int valid_rows = (int)ne0 - (int)(ct * 32); \ - valid_rows = MIN(32, MAX(0, valid_rows)); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - \ - if (push_ct < ct_end) { \ - dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - push_ct++; \ - } \ - } \ - \ - int copy_cnt = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; \ - if (copy_cnt > 0) { \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); \ - if (src2) { \ - hvx_add_f32_uaa((uint8_t *) &dst_col[src0_start_row], \ - (const uint8_t *) tmp, \ - (const uint8_t *) ((const float *) mmctx->vtcm_src2 + src0_start_row), \ - copy_cnt); \ - } else { \ - hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); \ - } \ +static void hvx_mv_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; + + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); + + struct htp_thread_trace * tr = th->trace; + + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + + const size_t dst_row_size = nb1; + const size_t act_row_size = nb11; + const size_t act_stride = mmctx->vtcm_act_stride; + + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * act_data = mmctx->vtcm_act; + + float * tmp = (float *) vtcm_dst_ptr; + + const dma_addr_t src0_row = src0->data; + + const uint8_t * restrict act_col = (const uint8_t *) act_data; + float * restrict dst_col = (float *) dst->data; + + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); + + uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); + uint32_t tile_row_stride = n_k_tiles_w * tile_size; + uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; + + uint32_t ct_start = src0_start_row / 32; + uint32_t ct_end = (src0_end_row + 31) / 32; + + uint32_t push_ct = ct_start; + if (src0_start_row < src0_end_row) { + if (src2) { + float * vtcm_bias_ptr = (float *) mmctx->vtcm_bias + src0_start_row; + const dma_addr_t src2_addr = src2->data + src0_start_row * sizeof(float); + int slice_size = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; + if (slice_size > 0) { + dma_queue_push(dma_q, dma_make_data(vtcm_bias_ptr, src2_addr), + slice_size * sizeof(float), slice_size * sizeof(float), slice_size * sizeof(float), 1); + dma_queue_pop_nowait(dma_q); + } + } + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } + } + + if (src0_start_row >= src0_end_row) { + return; + } + + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; + + float * dst_ptr = &tmp[ct * 32 - src0_start_row]; + int valid_rows = (int)ne0 - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + mmctx->vec_dot_32x1(ne10, dst_ptr, w_tile, act_col, valid_rows, NULL); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); + + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } + + int copy_cnt = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; + if (copy_cnt > 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); + if (src2) { + hvx_add_f32_uuu((uint8_t *) &dst_col[src0_start_row], + (const uint8_t *) tmp, + (const uint8_t *) ((const float *) mmctx->vtcm_bias + src0_start_row), + copy_cnt); + } else { + hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); + } } -#define MATMUL_NX_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ -static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_mm_context * mmctx = data; \ - struct htp_ops_context * octx = mmctx->octx; \ - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ - const uint32_t n_weights = kparams->n_weights; \ - \ - const struct htp_tensor * restrict act = octx->src[n_weights]; /* x */ \ - const uint32_t ne10 = act->ne[0]; \ - const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; \ - const size_t src1_stride = mmctx->vtcm_src1_stride; \ - \ - uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ - uint8_t * restrict src1_data = mmctx->vtcm_src1; \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t n_prefetch = kparams->n_prefetch; \ - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ - \ - const uint32_t tile_size = TILE_SIZE; \ - const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ - uint32_t n_k_tiles_a = ne10 / 32; \ - uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ - \ - for (uint32_t widx = 0; widx < n_weights; widx++) { \ - const struct htp_tensor * restrict src_w = octx->src[widx]; \ - const struct htp_tensor * restrict dst = octx->dsts[widx]; \ - if (!src_w || !dst) continue; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - \ - const uint32_t ne00 = src_w->ne[0]; \ - const uint32_t ne01 = src_w->ne[1]; \ - const size_t dst_row_size = dst->nb[1]; \ - const dma_addr_t src_w_row = src_w->data; \ - \ - uint32_t n_k_tiles_w = ne00 / 32; \ - uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ - \ - uint32_t src0_start_row = 0; \ - uint32_t src0_end_row = ne01; \ - if (octx->ctx->mdev.count > 1) { \ - const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); \ - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, \ - octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); \ - src0_start_row = range.start; \ - src0_end_row = range.start + range.count; \ - } \ - \ - const uint32_t nrows = src0_end_row - src0_start_row; \ - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); \ - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); \ - \ - const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith; \ - const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row); \ - if (start_row >= end_row) continue; \ - \ - uint32_t ct_start = start_row / 32; \ - uint32_t ct_end = (end_row + 31) / 32; \ - \ - uint32_t push_ct = ct_start; \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_q, dma_make_data(vtcm_weight_ptr + d * tile_row_transfer_size_aligned, \ - src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - } \ - \ - for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ - const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; \ - int valid_rows = (int)ne01 - (int)(ct * 32); \ - valid_rows = MIN(32, MAX(0, valid_rows)); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - uint32_t ir1 = 0; \ - for (; ir1 + 1 < src1_nrows; ir1 += 2) { \ - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \ - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \ - \ - float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); \ - float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); \ - float * dst_ptr0 = &dst_row0[ct * 32]; \ - float * dst_ptr1 = &dst_row1[ct * 32]; \ - \ - DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, NULL, NULL); \ - } \ - \ - for (; ir1 < src1_nrows; ++ir1) { \ - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \ - float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); \ - float * dst_ptr = &dst_row[ct * 32]; \ - DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - \ - if (push_ct < ct_end) { \ - dma_queue_push(dma_q, dma_make_data(w_tile, src_w_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - push_ct++; \ - } \ - } \ - } \ +static void hvx_mm_nx_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_mm_context * mmctx = data; + struct htp_ops_context * octx = mmctx->octx; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_weights = kparams->n_weights; + + const struct htp_tensor * restrict act = octx->src[n_weights]; /* x */ + const uint32_t ne10 = act->ne[0]; + const uint32_t act_nrows = act->ne[1] * act->ne[2] * act->ne[3]; + const size_t act_stride = mmctx->vtcm_act_stride; + + uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; + + struct htp_thread_trace * tr = th->trace; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + + for (uint32_t widx = 0; widx < n_weights; widx++) { + const struct htp_tensor * restrict src_w = octx->src[widx]; + const struct htp_tensor * restrict dst = octx->dsts[widx]; + if (!src_w || !dst) continue; + dma_queue * dma_q = th->dma_q; + + const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); + const uint32_t ne00 = src_w->ne[0]; + const uint32_t ne01 = src_w->ne[1]; + const size_t dst_row_size = dst->nb[1]; + const dma_addr_t src_w_row = src_w->data; + + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src_w->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; + + uint32_t src0_start_row = 0; + uint32_t src0_end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_start_row = range.start; + src0_end_row = range.start + range.count; + } + + const uint32_t nrows = src0_end_row - src0_start_row; + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); + + const uint32_t start_row = src0_start_row + src0_nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + src0_nrows_per_hvx, src0_end_row); + if (start_row >= end_row) continue; + + uint32_t ct_start = start_row / 32; + uint32_t ct_end = (end_row + 31) / 32; + + uint32_t push_ct = ct_start; + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_weight_ptr + d * tile_row_transfer_size_aligned, + src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } + + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; + int valid_rows = (int)dst->ne[0] - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + uint32_t ir1 = 0; + for (; ir1 + 1 < act_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (ir1+0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (ir1+1) * act_stride); + + float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); + float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); + float * dst_ptr0 = &dst_row0[ct * 32]; + float * dst_ptr1 = &dst_row1[ct * 32]; + + mmctx->vec_dot_32x2(ne10, dst_ptr0, dst_ptr1, w_tile, act_col0, act_col1, valid_rows, NULL, NULL); + } + + for (; ir1 < act_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + ir1 * act_stride); + float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); + float * dst_ptr = &dst_row[ct * 32]; + mmctx->vec_dot_32x1(ne10, dst_ptr, w_tile, act_col, valid_rows, NULL); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); + + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src_w_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } + } } -MATMUL_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) -MATMUL_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) -MATMUL_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) -MATMUL_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1) -MATMUL_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1) -MATMUL_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1) -MATMUL_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) -MATMUL_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) -MATMUL_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) - -static void hvx_mm_transfer_src1_dma( +static void hvx_mm_transfer_act_dma( struct htp_ops_context * octx, const struct htp_mm_kernel_params * kparams, - const struct htp_tensor * src1, + const struct htp_tensor * act, uint8_t * dst_base, size_t dst_row_size, uint32_t m_start, @@ -564,22 +553,22 @@ static void hvx_mm_transfer_src1_dma( } dma_queue * dma_q = octx->ctx->dma[0]; - const uint32_t ne0 = src1->ne[0]; - const size_t elem_size = (src1->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + const uint32_t ne0 = act->ne[0]; + const size_t elem_size = (act->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); const size_t row_bytes = ne0 * elem_size; - const size_t src1_nb1 = src1->nb[1]; - const dma_addr_t src_base = src1->data; + const size_t act_nb1 = act->nb[1]; + const dma_addr_t act_base = act->data; - const bool is_contiguous = (src1->nb[2] == src1->ne[1] * src1_nb1) && - (src1->nb[3] == src1->ne[2] * src1->nb[2]); + const bool is_contiguous = (act->nb[2] == act->ne[1] * act_nb1) && + (act->nb[3] == act->ne[2] * act->nb[2]); if (is_contiguous) { - const dma_addr_t src_addr = src_base + m_start * src1_nb1; + const dma_addr_t src_addr = act_base + m_start * act_nb1; dma_queue_push(dma_q, dma_make_data(dst_base, src_addr), - dst_row_size, src1_nb1, row_bytes, m_rows); + dst_row_size, act_nb1, row_bytes, m_rows); dma_queue_pop(dma_q); } else { - const uint32_t ne12_ne1 = src1->ne[2] * src1->ne[1]; + const uint32_t ne12_ne1 = act->ne[2] * act->ne[1]; const bool use_fastdiv = kparams->div_ne12_ne1.mp != 0; for (uint32_t ir = 0; ir < m_rows; ++ir) { const uint32_t ir1 = m_start + ir; @@ -588,19 +577,19 @@ static void hvx_mm_transfer_src1_dma( i13 = fastdiv(ir1, &kparams->div_ne12_ne1); const uint32_t rem = ir1 - i13 * ne12_ne1; i12 = fastdiv(rem, &kparams->div_ne1); - i11 = rem - i12 * src1->ne[1]; + i11 = rem - i12 * act->ne[1]; } else { i13 = ne12_ne1 ? ir1 / ne12_ne1 : 0; const uint32_t rem = ir1 - i13 * ne12_ne1; - i12 = src1->ne[1] ? rem / src1->ne[1] : 0; - i11 = rem - i12 * src1->ne[1]; + i12 = act->ne[1] ? rem / act->ne[1] : 0; + i11 = rem - i12 * act->ne[1]; } - const dma_addr_t row_src = src_base + (i11 * src1->nb[1] + - i12 * src1->nb[2] + - i13 * src1->nb[3]); + const dma_addr_t row_src = act_base + (i11 * act->nb[1] + + i12 * act->nb[2] + + i13 * act->nb[3]); uint8_t * row_dst = dst_base + ir * dst_row_size; dma_queue_push(dma_q, dma_make_data(row_dst, row_src), - dst_row_size, src1_nb1, row_bytes, 1); + dst_row_size, act_nb1, row_bytes, 1); dma_queue_pop(dma_q); } } @@ -623,25 +612,25 @@ static void hvx_mm_transfer_src1_dma( } #define QUANTIZE_IMPL(name, log_name, kernel_fn, dst_row_size_expr) \ -static void name(unsigned int nth, unsigned int ith, void * data) { \ +static void name(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ (void) nth; \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ const struct htp_tensor * src = mmctx->act; \ const uint32_t ne0 = src->ne[0]; \ const uint32_t nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; \ - const uint32_t nrows_per_thread = mmctx->n_quant_rows_per_thread; \ + const uint32_t nrows_per_hvx = mmctx->n_quant_rows_per_hvx; \ \ - const uint32_t ir_first = nrows_per_thread * ith; \ + const uint32_t ir_first = nrows_per_hvx * ith; \ if (ir_first >= nrows) { \ return; \ } \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + struct htp_thread_trace * tr = th->trace; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ \ - uint8_t * restrict dst = mmctx->vtcm_src1; \ - const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); \ + uint8_t * restrict dst = mmctx->vtcm_act; \ + const uint32_t ir_last = MIN(ir_first + nrows_per_hvx, nrows); \ const size_t raw_row_size = mmctx->vtcm_act_raw_stride; \ const size_t dst_row_size = (dst_row_size_expr); \ \ @@ -652,28 +641,30 @@ static void name(unsigned int nth, unsigned int ith, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ } -QUANTIZE_IMPL(quantize_f32_q8_0_tiled, "quantize-f32-q8_0_tiled", quantize_f32_q8_0_tiled_kernel, htp_mm_q8_0_tiled_row_size(ne0)) -QUANTIZE_IMPL(quantize_f32_q8_1_tiled, "quantize-f32-q8_1_tiled", quantize_f32_q8_1_tiled_kernel, htp_mm_q8_1_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f32_q8_0_tiled, "quantize-f32-q8_0_tiled", quantize_f32_q8_0_tiled_kernel, htp_mm_q8_0_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f32_q8_1_tiled, "quantize-f32-q8_1_tiled", quantize_f32_q8_1_tiled_kernel, htp_mm_q8_1_tiled_row_size(ne0)) QUANTIZE_IMPL(quantize_f32_q8_1_s16_tiled, "quantize-f32-q8_1_s16_tiled", quantize_f32_q8_1_s16_tiled_kernel, htp_mm_q8_1_tiled_row_size(ne0)) -QUANTIZE_IMPL(quantize_f32_f32, "quantize-f32-f32", quantize_f32_f32_kernel, mmctx->vtcm_src1_stride) -QUANTIZE_IMPL(quantize_f32_f16, "quantize-f32-f16", quantize_f32_f16_kernel, mmctx->vtcm_src1_stride) -QUANTIZE_IMPL(quantize_f16_f16, "quantize-f16-f16", quantize_f16_f16_kernel, mmctx->vtcm_src1_stride) +QUANTIZE_IMPL(quantize_f32_f16_tiled, "quantize-f32-f16_tiled", quantize_f32_f16_tiled_kernel, htp_mm_f16_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f16_f16_tiled, "quantize-f16-f16_tiled", quantize_f16_f16_tiled_kernel, htp_mm_f16_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f32_f32, "quantize-f32-f32", quantize_f32_f32_kernel, mmctx->vtcm_act_stride) +QUANTIZE_IMPL(quantize_f32_f16, "quantize-f32-f16", quantize_f32_f16_kernel, mmctx->vtcm_act_stride) +QUANTIZE_IMPL(quantize_f16_f16, "quantize-f16-f16", quantize_f16_f16_kernel, mmctx->vtcm_act_stride) -static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_0_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; quantize_f32_q8_0_tiled_block_kernel( (const float *) mmctx->vtcm_act_raw, - mmctx->vtcm_src1, + mmctx->vtcm_act, NULL, src->ne[0], mmctx->quant_ib_first[ith], @@ -687,21 +678,21 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_1_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; quantize_f32_q8_1_tiled_block_kernel( (const float *) mmctx->vtcm_act_raw, - mmctx->vtcm_src1, + mmctx->vtcm_act, NULL, src->ne[0], mmctx->quant_ib_first[ith], @@ -715,21 +706,21 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_1_s16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; quantize_f32_q8_1_s16_tiled_block_kernel( (const float *) mmctx->vtcm_act_raw, - mmctx->vtcm_src1, + mmctx->vtcm_act, NULL, src->ne[0], mmctx->quant_ib_first[ith], @@ -743,178 +734,208 @@ static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -// q8_1 for weight types with offsets (q8_1_s16 for Q2_K), otherwise q8_0 -static inline worker_callback_t htp_mm_act_quant_row_func(int weight_type) { +static void quantize_f32_f16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_mm_context * mmctx = data; + if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { + return; + } + struct htp_ops_context * octx = mmctx->octx; + struct htp_thread_trace * tr = th->trace; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); + + const struct htp_tensor * src = mmctx->act; + + quantize_f32_f16_tiled_block_kernel( + (const float *) mmctx->vtcm_act_raw, + mmctx->vtcm_act, + NULL, + src->ne[0], + mmctx->quant_ib_first[ith], + mmctx->quant_ib_last[ith], + mmctx->vtcm_act_raw_stride, + htp_mm_f16_tiled_row_size(src->ne[0]), + mmctx->quant_r[ith], + mmctx->quant_c[ith] + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); +} + +static void quantize_f16_f16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_mm_context * mmctx = data; + if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { + return; + } + struct htp_ops_context * octx = mmctx->octx; + struct htp_thread_trace * tr = th->trace; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); + + const struct htp_tensor * src = mmctx->act; + + quantize_f16_f16_tiled_block_kernel( + (const __fp16 *) mmctx->vtcm_act_raw, + mmctx->vtcm_act, + NULL, + src->ne[0], + mmctx->quant_ib_first[ith], + mmctx->quant_ib_last[ith], + mmctx->vtcm_act_raw_stride, + htp_mm_f16_tiled_row_size(src->ne[0]), + mmctx->quant_r[ith], + mmctx->quant_c[ith] + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); +} + +// q8_1 for weight types with offsets (q8_1_s16 for Q2_K), f16_tiled for F16, otherwise q8_0 +static inline work_queue_func_t htp_mm_act_quant_row_func(int weight_type, int act_type) { + if (weight_type == HTP_TYPE_F16) { + return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled : quantize_f32_f16_tiled; + } if (weight_type == HTP_TYPE_Q2_K) { return quantize_f32_q8_1_s16_tiled; } return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } -static inline worker_callback_t htp_mm_act_quant_block_func(int weight_type) { +static inline work_queue_func_t htp_mm_act_quant_block_func(int weight_type, int act_type) { + if (weight_type == HTP_TYPE_F16) { + return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled_block : quantize_f32_f16_tiled_block; + } if (weight_type == HTP_TYPE_Q2_K) { return quantize_f32_q8_1_s16_tiled_block; } return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; } -MATVEC_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x1) -MATVEC_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x1) -MATVEC_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x1) -MATVEC_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x1) -MATVEC_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x1) -MATVEC_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x1) -MATVEC_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x1) -MATVEC_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x1) -MATVEC_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x1) - -MATMUL_NX_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) -MATMUL_NX_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) -MATMUL_NX_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) - -#define MATMUL_4D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ -static void hvx_mm_4d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ - htp_matmul_preamble; \ - \ - const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ - const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); \ - const uint32_t cur_m_start = mmctx->cur_m_start; \ - \ - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ - const uint32_t n_prefetch = kparams->n_prefetch; \ - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ - \ - const size_t dst_row_size = nb1; \ - const size_t src1_stride = mmctx->vtcm_src1_stride; \ - \ - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ - uint8_t * restrict src1_data = mmctx->vtcm_src1; \ - \ - const uint32_t tile_size = TILE_SIZE; \ - const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ - \ - const uint32_t n_k_tiles_w = ne00 / 32; \ - const uint32_t n_k_tiles_a = ne10 / 32; \ - const uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ - const uint32_t src0_slice_stride = ((ne01 + 31) / 32) * tile_row_stride; \ - \ - const uint32_t ct_start = src0_start_row / 32; \ - const uint32_t ct_end = (src0_end_row + 31) / 32; \ - \ - \ - if (src0_start_row >= src0_end_row || cur_m_rows == 0) { \ - return; \ - } \ - \ - const uint32_t total_batches = ne12 * ne13; \ - const uint32_t b_start = fastdiv(cur_m_start, &kparams->div_ne1); \ - uint32_t b_end = fastdiv(cur_m_start + cur_m_rows + ne11 - 1, &kparams->div_ne1); \ - b_end = MIN(b_end, total_batches); \ - \ - uint32_t b_grp_start = b_start; \ - while (b_grp_start < b_end) { \ - const uint32_t b3 = fastdiv(b_grp_start, &kparams->div_ne12); \ - const uint32_t b2 = b_grp_start - b3 * ne12; \ - const uint32_t i02 = fastdiv(b2, &kparams->div_r2); \ - const uint32_t i03 = fastdiv(b3, &kparams->div_r3); \ - \ - uint32_t b_grp_end = b_grp_start + 1; \ - while (b_grp_end < b_end) { \ - const uint32_t cur_b3 = fastdiv(b_grp_end, &kparams->div_ne12); \ - const uint32_t cur_b2 = b_grp_end - cur_b3 * ne12; \ - const uint32_t cur_i02 = fastdiv(cur_b2, &kparams->div_r2); \ - const uint32_t cur_i03 = fastdiv(cur_b3, &kparams->div_r3); \ - if (cur_i02 != i02 || cur_i03 != i03) { \ - break; \ - } \ - b_grp_end++; \ - } \ - \ - const uint32_t slice_idx = i03 * ne02 + i02; \ - const dma_addr_t src0_slice = src0->data + (size_t) slice_idx * src0_slice_stride; \ - \ - uint32_t push_ct = ct_start; \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_slice + (size_t) push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - } \ - \ - for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ - const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; \ - \ - int valid_rows = (int)ne0 - (int)(ct * 32); \ - valid_rows = MIN(32, MAX(0, valid_rows)); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - for (uint32_t b = b_grp_start; b < b_grp_end; b++) { \ - const uint32_t b_m_start = b * ne11; \ - const uint32_t m_first = MAX(cur_m_start, b_m_start); \ - const uint32_t m_last = MIN(cur_m_start + cur_m_rows, b_m_start + ne11); \ - if (m_first >= m_last) continue; \ - \ - const uint32_t cur_b3 = fastdiv(b, &kparams->div_ne12); \ - const uint32_t cur_b2 = b - cur_b3 * ne12; \ - uint8_t * dst_batch_base = (uint8_t *) dst->data + (size_t) cur_b2 * nb2 + (size_t) cur_b3 * nb3; \ - \ - const uint32_t chunk_m_offset = m_first - cur_m_start; \ - const uint32_t dst_m_offset = m_first - b_m_start; \ - const uint32_t batch_nrows = m_last - m_first; \ - \ - uint32_t ir1 = 0; \ - for (; ir1 + 1 < batch_nrows; ir1 += 2) { \ - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (chunk_m_offset + ir1 + 0) * src1_stride); \ - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (chunk_m_offset + ir1 + 1) * src1_stride); \ - float * restrict dst_row0 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 0) * dst_row_size); \ - float * restrict dst_row1 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 1) * dst_row_size); \ - float * dst_ptr0 = &dst_row0[ct * 32]; \ - float * dst_ptr1 = &dst_row1[ct * 32]; \ - DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, NULL, NULL); \ - } \ - for (; ir1 < batch_nrows; ++ir1) { \ - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (chunk_m_offset + ir1) * src1_stride); \ - float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); \ - float * dst_ptr = &dst_row[ct * 32]; \ - DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ - } \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - \ - if (push_ct < ct_end) { \ - dma_queue_push(dma_q, dma_make_data(w_tile, src0_slice + (size_t) push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - push_ct++; \ - } \ - } \ - b_grp_start = b_grp_end; \ - } \ - if (src2) { \ - hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + cur_m_rows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); \ - } \ +static void hvx_mm_4d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; + + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); + const uint32_t cur_m_start = mmctx->cur_m_start; + + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); + + struct htp_thread_trace * tr = th->trace; + + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + + const size_t dst_row_size = nb1; + const size_t act_stride = mmctx->vtcm_act_stride; + + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; + + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); + + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; + const uint32_t src0_slice_stride = ((ne01 + 31) / 32) * tile_row_stride; + + const uint32_t ct_start = src0_start_row / 32; + const uint32_t ct_end = (src0_end_row + 31) / 32; + + if (src0_start_row >= src0_end_row || cur_m_rows == 0) { + return; + } + + const uint32_t total_batches = ne12 * ne13; + const uint32_t b_start = fastdiv(cur_m_start, &kparams->div_ne1); + uint32_t b_end = fastdiv(cur_m_start + cur_m_rows + ne11 - 1, &kparams->div_ne1); + b_end = MIN(b_end, total_batches); + + uint32_t b_grp_start = b_start; + while (b_grp_start < b_end) { + const uint32_t b3 = fastdiv(b_grp_start, &kparams->div_ne12); + const uint32_t b2 = b_grp_start - b3 * ne12; + const uint32_t i02 = fastdiv(b2, &kparams->div_r2); + const uint32_t i03 = fastdiv(b3, &kparams->div_r3); + + uint32_t b_grp_end = b_grp_start + 1; + while (b_grp_end < b_end) { + const uint32_t cur_b3 = fastdiv(b_grp_end, &kparams->div_ne12); + const uint32_t cur_b2 = b_grp_end - cur_b3 * ne12; + const uint32_t cur_i02 = fastdiv(cur_b2, &kparams->div_r2); + const uint32_t cur_i03 = fastdiv(cur_b3, &kparams->div_r3); + if (cur_i02 != i02 || cur_i03 != i03) { + break; + } + b_grp_end++; + } + + const uint32_t slice_idx = i03 * ne02 + i02; + const dma_addr_t src0_slice = src0->data + (size_t) slice_idx * src0_slice_stride; + + uint32_t push_ct = ct_start; + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, + src0_slice + (size_t) push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } + + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; + + int valid_rows = (int)ne0 - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + for (uint32_t b = b_grp_start; b < b_grp_end; b++) { + const uint32_t b_m_start = b * ne11; + const uint32_t m_first = MAX(cur_m_start, b_m_start); + const uint32_t m_last = MIN(cur_m_start + cur_m_rows, b_m_start + ne11); + if (m_first >= m_last) continue; + + const uint32_t cur_b3 = fastdiv(b, &kparams->div_ne12); + const uint32_t cur_b2 = b - cur_b3 * ne12; + uint8_t * dst_batch_base = (uint8_t *) dst->data + (size_t) cur_b2 * nb2 + (size_t) cur_b3 * nb3; + + const uint32_t chunk_m_offset = m_first - cur_m_start; + const uint32_t dst_m_offset = m_first - b_m_start; + const uint32_t batch_nrows = m_last - m_first; + + uint32_t ir1 = 0; + for (; ir1 + 1 < batch_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (chunk_m_offset + ir1 + 0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (chunk_m_offset + ir1 + 1) * act_stride); + float * restrict dst_row0 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 0) * dst_row_size); + float * restrict dst_row1 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 1) * dst_row_size); + float * dst_ptr0 = &dst_row0[ct * 32]; + float * dst_ptr1 = &dst_row1[ct * 32]; + mmctx->vec_dot_32x2(ne10, dst_ptr0, dst_ptr1, w_tile, act_col0, act_col1, valid_rows, NULL, NULL); + } + for (; ir1 < batch_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (chunk_m_offset + ir1) * act_stride); + float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); + float * dst_ptr = &dst_row[ct * 32]; + mmctx->vec_dot_32x1(ne10, dst_ptr, w_tile, act_col, valid_rows, NULL); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); + + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_slice + (size_t) push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } + b_grp_start = b_grp_end; + } + if (src2) { + hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + cur_m_rows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); + } } -MATMUL_4D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) -MATMUL_4D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) -MATMUL_4D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) -MATMUL_4D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1) -MATMUL_4D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1) -MATMUL_4D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1) -MATMUL_4D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) -MATMUL_4D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) -MATMUL_4D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) - -static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_2d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -923,26 +944,26 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t prefetch_mask = n_prefetch - 1; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows - const uint32_t src1_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; // src1 rows + const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; - const size_t src1_row_size = nb11; + const size_t act_row_size = nb11; const size_t src0_stride = mmctx->vtcm_src0_stride; - const size_t src1_stride = mmctx->vtcm_src1_stride; + const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; const dma_addr_t src0_row = src0->data; @@ -968,21 +989,21 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - // Process src1 columns in pairs (2x2 tiling) + // Process act columns in pairs (2x2 tiling) uint32_t ir1 = 0; - for (; ir1 + 1 < src1_nrows; ir1 += 2) { - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); + for (; ir1 + 1 < act_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (ir1+0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (ir1+1) * act_stride); float * restrict dst_row0 = (float *) (dst->data + ((cur_m_start + ir1+0) * dst_row_size)); float * restrict dst_row1 = (float *) (dst->data + ((cur_m_start + ir1+1) * dst_row_size)); - mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); + mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, act_col0, act_col1); } - // Handle remaining src1 rows (fallback to 2x1) - for (; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); + // Handle remaining act rows (fallback to 2x1) + for (; ir1 < act_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + ir1 * act_stride); float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); - mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col); + mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, act_col); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); @@ -1005,45 +1026,45 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); #pragma unroll(2) - for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); + for (uint32_t ir1 = 0; ir1 < act_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + ir1 * act_stride); float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); - mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); + mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, act_col); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); } if (src2) { - hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + src1_nrows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); + hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + act_nrows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); } } -static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_2d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; - const size_t src1_row_size = nb11; + const size_t act_row_size = nb11; const size_t src0_stride = mmctx->vtcm_src0_stride; - const size_t src1_stride = mmctx->vtcm_src1_stride; + const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * src1_data = mmctx->vtcm_src1; + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * act_data = mmctx->vtcm_act; float * tmp = (float *) vtcm_dst_ptr; const dma_addr_t src0_row = src0->data; - const uint8_t * restrict src1_col = (const uint8_t *) src1_data; - float * restrict dst_col = (float *) dst->data; + const uint8_t * restrict act_col = (const uint8_t *) act_data; + float * restrict dst_col = (float *) dst->data; const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); @@ -1055,11 +1076,11 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { // Prefill vtcm with 2x src0 rows if (src0_start_row < src0_end_row) { if (src2) { - float * vtcm_src2_ptr = (float *) mmctx->vtcm_src2 + src0_start_row; + float * vtcm_bias_ptr = (float *) mmctx->vtcm_bias + src0_start_row; const dma_addr_t src2_addr = src2->data + src0_start_row * sizeof(float); int slice_size = (int)src0_end_row - (int)src0_start_row; if (slice_size > 0) { - dma_queue_push(dma_q, dma_make_data(vtcm_src2_ptr, src2_addr), + dma_queue_push(dma_q, dma_make_data(vtcm_bias_ptr, src2_addr), slice_size * sizeof(float), slice_size * sizeof(float), slice_size * sizeof(float), 1); dma_queue_pop_nowait(dma_q); } @@ -1083,7 +1104,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - mmctx->vec_dot_2x1(ne00, &tmp[ir0 - src0_start_row], ss0, ss0 + src0_stride, src1_col); + mmctx->vec_dot_2x1(ne00, &tmp[ir0 - src0_start_row], ss0, ss0 + src0_stride, act_col); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); // Prefetch next (n + vtcm_nrows) row @@ -1103,7 +1124,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { src0_stride, src0_row_size, src0_row_size, 1); const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - mmctx->vec_dot_1x1(ne00, &tmp[ir0 - src0_start_row], ss0, src1_col); + mmctx->vec_dot_1x1(ne00, &tmp[ir0 - src0_start_row], ss0, act_col); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); } @@ -1111,9 +1132,9 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { if (copy_cnt > 0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, src0_end_row); if (src2) { - hvx_add_f32_uaa((uint8_t *) &dst_col[src0_start_row], + hvx_add_f32_uuu((uint8_t *) &dst_col[src0_start_row], (const uint8_t *) tmp, - (const uint8_t *) ((const float *) mmctx->vtcm_src2 + src0_start_row), + (const uint8_t *) ((const float *) mmctx->vtcm_bias + src0_start_row), copy_cnt); } else { hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); @@ -1122,7 +1143,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_4d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1134,19 +1155,19 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; const size_t src0_stride = mmctx->vtcm_src0_stride; - const size_t src1_stride = mmctx->vtcm_src1_stride; + const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; if (src0_start_row >= src0_end_row || cur_m_rows == 0) { @@ -1208,16 +1229,16 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { uint32_t ir1 = 0; for (; ir1 + 1 < batch_nrows; ir1 += 2) { - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (chunk_m_offset + ir1 + 0) * src1_stride); - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (chunk_m_offset + ir1 + 1) * src1_stride); + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (chunk_m_offset + ir1 + 0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (chunk_m_offset + ir1 + 1) * act_stride); float * restrict dst_row0 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 0) * dst_row_size); float * restrict dst_row1 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 1) * dst_row_size); - mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); + mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, act_col0, act_col1); } for (; ir1 < batch_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (chunk_m_offset + ir1) * src1_stride); + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (chunk_m_offset + ir1) * act_stride); float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); - mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col); + mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, act_col); } } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); @@ -1253,9 +1274,9 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { const uint32_t batch_nrows = m_last - m_first; for (uint32_t ir1 = 0; ir1 < batch_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (chunk_m_offset + ir1) * src1_stride); + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (chunk_m_offset + ir1) * act_stride); float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); - mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); + mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, act_col); } } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); @@ -1271,22 +1292,22 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { #define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * mmctx->mapping_stride + (i1)] -static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_id_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_tensor * restrict ids = octx->src[2]; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert - const uint32_t src1_nrows = ne11; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t act_nrows = ne11; + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); if (src0_start_row >= src0_end_row) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1299,13 +1320,13 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { const struct mmid_row_mapping * matrix_rows = mmctx->matrix_rows; const size_t dst_row_size = nb1; - const size_t src1_row_size = htp_mm_q8_0_tiled_row_size(ne10); + const size_t act_row_size = htp_mm_q8_0_tiled_row_size(ne10); - const size_t src1_stride = mmctx->vtcm_src1_stride; + const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { const int32_t cne1 = matrix_row_counts[cur_a]; @@ -1315,12 +1336,11 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { const dma_addr_t src0_row = src0->data + cur_a * nb02; - const uint32_t tile_size = htp_mm_get_weight_tile_size(src0->type); - const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(src0->type); - const uint32_t n_k_tiles_w = ne00 / 32; - const uint32_t n_k_tiles_a = ne10 / 32; + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t ct_start = src0_start_row / 32; const uint32_t ct_end = (src0_end_row + 31) / 32; @@ -1328,13 +1348,13 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - int valid_rows = (int)ne01 - (int)(ct * 32); + int valid_rows = (int)ne0 - (int)(ct * 32); valid_rows = MIN(32, MAX(0, valid_rows)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); @@ -1343,38 +1363,38 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { const int rm1 = row_mapping.i1; // expert idx const int rm2 = row_mapping.i2; // token idx - const uint32_t ir1 = fastmodulo(rm1, ne11, &mmctx->mm_div_ne11); // src1 row idx - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (ir1 + rm2 * ne11 + 0) * src1_stride); + const uint32_t ir1 = fastmodulo(rm1, ne11, &mmctx->mm_div_ne11); // act row idx + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (ir1 + rm2 * ne11 + 0) * act_stride); float * restrict dst_row = (float *) (dst->data + (rm1 * nb1 + rm2 * nb2 + 0)); - mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, src1_col, valid_rows, NULL); + mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, act_col, valid_rows, NULL); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } } } -static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_id_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_tensor * restrict ids = octx->src[2]; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); if (src0_start_row >= src0_end_row) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1383,32 +1403,31 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { assert(ne13 % ne03 == 0); const size_t dst_row_size = nb1; - const size_t src1_row_size = htp_mm_q8_0_tiled_row_size(ne10); + const size_t act_row_size = htp_mm_q8_0_tiled_row_size(ne10); const uint32_t n_aids = src2->ne[0]; // num activated experts const uint32_t n_ids = ne02; // num experts // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert const int32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]); if (eid < 0) { continue; } - assert(eid < (int32_t) n_ids); + assert(eid >= 0 && eid < (int32_t) n_ids); const dma_addr_t src0_row = src0->data + eid * nb02; - const uint8_t * restrict src1_col = (const uint8_t *) src1_data; + const uint8_t * restrict act_col = (const uint8_t *) act_data; float * restrict dst_row = (float *) (dst->data + ie1 * nb1); - const uint32_t tile_size = htp_mm_get_weight_tile_size(src0->type); - const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(src0->type); - const uint32_t n_k_tiles_w = ne00 / 32; - const uint32_t n_k_tiles_a = ne10 / 32; + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t ct_start = src0_start_row / 32; const uint32_t ct_end = (src0_end_row + 31) / 32; @@ -1416,29 +1435,29 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - int valid_rows = (int)ne01 - (int)(ct * 32); + int valid_rows = (int)ne0 - (int)(ct * 32); valid_rows = MIN(32, MAX(0, valid_rows)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); - mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, src1_col, valid_rows, NULL); + mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, act_col, valid_rows, NULL); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } } } -static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_id_nx_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = (struct htp_mm_context *) data; struct htp_ops_context * octx = mmctx->octx; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1448,7 +1467,7 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); @@ -1456,8 +1475,8 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { const uint32_t n_aids = ids->ne[0]; const uint32_t n_ids = src0->ne[2]; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { const int32_t eid = *(const int32_t *) ((const uint8_t *) ids->data + ie1 * ids->nb[0]); @@ -1468,7 +1487,7 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t ne01 = src_w->ne[1]; uint32_t start_row = 0; @@ -1481,23 +1500,22 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { } const uint32_t nrows = end_row - start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); - const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); + const uint32_t src0_start_row = start_row + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, end_row); if (src0_start_row >= src0_end_row) continue; const dma_addr_t src0_row = src_w->data + eid * src_w->nb[2]; - const uint8_t * restrict src1_col = (const uint8_t *) src1_data; + const uint8_t * restrict act_col = (const uint8_t *) act_data; float * restrict dst_row = (float *) (dst->data + ie1 * dst->nb[1]); - const uint32_t tile_size = htp_mm_get_weight_tile_size(src_w->type); - const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(src_w->type); - const uint32_t n_k_tiles_w = src_w->ne[0] / 32; - const uint32_t n_k_tiles_a = act->ne[0] / 32; + const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); + const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t ct_start = src0_start_row / 32; const uint32_t ct_end = (src0_end_row + 31) / 32; @@ -1505,22 +1523,22 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - int valid_rows = (int)src_w->ne[1] - (int)(ct * 32); + int valid_rows = (int)dst->ne[0] - (int)(ct * 32); valid_rows = MIN(32, MAX(0, valid_rows)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); - mmctx->vec_dot_32x1(act->ne[0], &dst_row[ct * 32], w_tile, src1_col, valid_rows, NULL); + mmctx->vec_dot_32x1(act->ne[0], &dst_row[ct * 32], w_tile, act_col, valid_rows, NULL); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -1528,7 +1546,7 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_id_nx_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = (struct htp_mm_context *) data; struct htp_ops_context * octx = mmctx->octx; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1538,7 +1556,7 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); @@ -1548,10 +1566,10 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { const uint32_t * matrix_row_counts = mmctx->matrix_row_counts; const struct mmid_row_mapping * matrix_rows = mmctx->matrix_rows; - const size_t src1_stride = mmctx->vtcm_src1_stride; + const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { const int32_t cne1 = matrix_row_counts[cur_a]; @@ -1561,7 +1579,7 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t ne01 = src_w->ne[1]; uint32_t start_row = 0; @@ -1574,21 +1592,20 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { } const uint32_t nrows = end_row - start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); - const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); + const uint32_t src0_start_row = start_row + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, end_row); if (src0_start_row >= src0_end_row) continue; const dma_addr_t src0_row = src_w->data + cur_a * src_w->nb[2]; - const uint32_t tile_size = htp_mm_get_weight_tile_size(src_w->type); - const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(src_w->type); - const uint32_t n_k_tiles_w = src_w->ne[0] / 32; - const uint32_t n_k_tiles_a = act->ne[0] / 32; + const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); + const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t ct_start = src0_start_row / 32; const uint32_t ct_end = (src0_end_row + 31) / 32; @@ -1596,13 +1613,13 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - int valid_rows = (int)src_w->ne[1] - (int)(ct * 32); + int valid_rows = (int)dst->ne[0] - (int)(ct * 32); valid_rows = MIN(32, MAX(0, valid_rows)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); @@ -1612,16 +1629,16 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { const int rm2 = row_mapping.i2; const uint32_t ir1 = fastmodulo(rm1, act->ne[1], &mmctx->mm_div_ne11); - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (ir1 + rm2 * act->ne[1]) * src1_stride); + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (ir1 + rm2 * act->ne[1]) * act_stride); float * restrict dst_row = (float *) (dst->data + (rm1 * dst->nb[1] + rm2 * dst->nb[2])); - mmctx->vec_dot_32x1(act->ne[0], &dst_row[ct * 32], w_tile, src1_col, valid_rows, NULL); + mmctx->vec_dot_32x1(act->ne[0], &dst_row[ct * 32], w_tile, act_col, valid_rows, NULL); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -1629,44 +1646,74 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { } } -static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type type) { +static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type type) { switch (type) { case HTP_TYPE_Q4_0: mmctx->type = "q4_0_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q4_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q4_0_32x2; return 0; case HTP_TYPE_Q4_1: case HTP_TYPE_Q4_K: mmctx->type = "q4_1_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q4_1_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q4_1_32x2; return 0; case HTP_TYPE_Q8_0: mmctx->type = "q8_0_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q8_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q8_0_32x2; + return 0; + case HTP_TYPE_Q5_0: + mmctx->type = "q5_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q5_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q5_0_32x2; return 0; + case HTP_TYPE_Q5_1: case HTP_TYPE_Q5_K: mmctx->type = "q5_k_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q5_k_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q5_k_32x2; return 0; case HTP_TYPE_Q6_K: mmctx->type = "q6_k_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q6_k_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q6_k_32x2; return 0; case HTP_TYPE_Q3_K: mmctx->type = "q3_k_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q3_k_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q3_k_32x2; return 0; case HTP_TYPE_Q2_K: mmctx->type = "q2_k_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q2_k_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q2_k_32x2; + return 0; + case HTP_TYPE_Q2_0: + mmctx->type = "q2_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q2_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q2_0_32x2; return 0; case HTP_TYPE_IQ4_NL: mmctx->type = "iq4nl_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_iq4nl_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_iq4nl_32x2; return 0; case HTP_TYPE_MXFP4: mmctx->type = "mxfp4_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_mxfp4_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_mxfp4_32x2; + return 0; + case HTP_TYPE_Q1_0: + mmctx->type = "q1_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q1_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q1_0_32x2; + return 0; + case HTP_TYPE_F16: + mmctx->type = "f16_tiled-f16"; + mmctx->vec_dot_32x1 = tiled_vec_dot_f16_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_f16_32x2; return 0; default: return -1; @@ -1682,13 +1729,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; - mmctx->act = src1; + mmctx->act = act; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t src0_nrows = ne01; - const uint32_t src1_nrows = ne11 * ne12 * ne13; - mmctx->act_nrows = src1_nrows; + const uint32_t act_nrows = ne11 * ne12 * ne13; + mmctx->act_nrows = act_nrows; uint32_t src0_row_start = 0; uint32_t src0_row_end = src0_nrows; @@ -1708,124 +1755,81 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->src0_row_start = src0_row_start; mmctx->src0_row_end = src0_row_end; - bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 || - src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL || - src0->type == HTP_TYPE_MXFP4 || src0->type == HTP_TYPE_Q6_K || - src0->type == HTP_TYPE_Q4_K || src0->type == HTP_TYPE_Q5_K || - src0->type == HTP_TYPE_Q3_K || src0->type == HTP_TYPE_Q2_K); + const bool is_tiled = htp_tensor_is_tiled(src0); - // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); - if (is_repacked) { - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + // Compute src0_nrows_per_hvx + mmctx->src0_nrows_per_hvx = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); + if (is_tiled) { + mmctx->src0_nrows_per_hvx = hex_round_up(mmctx->src0_nrows_per_hvx, 32); } else { - mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even + mmctx->src0_nrows_per_hvx += (mmctx->src0_nrows_per_hvx & 1); // round up to even } const size_t src0_row_size = nb01; const size_t dst_row_size = nb1; - size_t src1_row_size = nb11; + size_t act_row_size = nb11; const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - size_t src1_row_size_padded; - worker_callback_t quant_task_func; - worker_callback_t matmul_job_func; + work_queue_func_t quant_task_func; + work_queue_func_t matmul_job_func; uint32_t n_quant_tasks = 1; const bool is_batched = (ne12 > 1 || ne13 > 1 || ne02 > 1 || ne03 > 1); if (is_batched) { - if (is_repacked) { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_4d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_4d_repacked_q4_1; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_4d_repacked_q8_0; break; - case HTP_TYPE_Q6_K: matmul_job_func = hvx_mm_4d_repacked_q6_k; break; - case HTP_TYPE_Q5_K: matmul_job_func = hvx_mm_4d_repacked_q5_k; break; - case HTP_TYPE_Q3_K: matmul_job_func = hvx_mm_4d_repacked_q3_k; break; - case HTP_TYPE_Q2_K: matmul_job_func = hvx_mm_4d_repacked_q2_k; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_4d_repacked_iq4nl; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_4d_repacked_mxfp4; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } else { - matmul_job_func = hvx_mm_4d; - } - } else if (src1_nrows > 1) { - if (is_repacked) { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_2d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_2d_repacked_q4_1; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_2d_repacked_q8_0; break; - case HTP_TYPE_Q6_K: matmul_job_func = hvx_mm_2d_repacked_q6_k; break; - case HTP_TYPE_Q5_K: matmul_job_func = hvx_mm_2d_repacked_q5_k; break; - case HTP_TYPE_Q3_K: matmul_job_func = hvx_mm_2d_repacked_q3_k; break; - case HTP_TYPE_Q2_K: matmul_job_func = hvx_mm_2d_repacked_q2_k; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_2d_repacked_iq4nl; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_2d_repacked_mxfp4; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } else { - matmul_job_func = hvx_mm_2d; - } + matmul_job_func = is_tiled ? hvx_mm_4d_tiled : hvx_mm_4d_flat; + } else if (act_nrows > 1) { + matmul_job_func = is_tiled ? hvx_mm_2d_tiled : hvx_mm_2d_flat; } else { - if (is_repacked) { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mv_2d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: matmul_job_func = hvx_mv_2d_repacked_q4_1; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mv_2d_repacked_q8_0; break; - case HTP_TYPE_Q5_K: matmul_job_func = hvx_mv_2d_repacked_q5_k; break; - case HTP_TYPE_Q6_K: matmul_job_func = hvx_mv_2d_repacked_q6_k; break; - case HTP_TYPE_Q3_K: matmul_job_func = hvx_mv_2d_repacked_q3_k; break; - case HTP_TYPE_Q2_K: matmul_job_func = hvx_mv_2d_repacked_q2_k; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mv_2d_repacked_iq4nl; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mv_2d_repacked_mxfp4; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } else { - matmul_job_func = hvx_mv_2d; - } + matmul_job_func = is_tiled ? hvx_mv_2d_tiled : hvx_mv_2d_flat; } bool need_quant = true; switch (kparams->kernel_type) { case HTP_MM_KERNEL_HVX_F16_F16_VTCM: - quant_task_func = (src1->type == HTP_TYPE_F32) ? quantize_f32_f16 : quantize_f16_f16; - need_quant = (src1->type == HTP_TYPE_F32); - mmctx->type = (src1->type == HTP_TYPE_F32) ? "f32-f16" : "f16-f16"; - mmctx->vec_dot_1x1 = vec_dot_f16_f16_aa_1x1; - mmctx->vec_dot_2x1 = vec_dot_f16_f16_aa_2x1; - mmctx->vec_dot_2x2 = vec_dot_f16_f16_aa_2x2; - src1_row_size = hex_round_up(ne10 * 2, 128); + quant_task_func = (act->type == HTP_TYPE_F32) ? quantize_f32_f16 : quantize_f16_f16; + need_quant = (act->type == HTP_TYPE_F32); + mmctx->type = (act->type == HTP_TYPE_F32) ? "f32-f16" : "f16-f16"; + mmctx->vec_dot_1x1 = flat_vec_dot_f16_f16_aa_1x1; + mmctx->vec_dot_2x1 = flat_vec_dot_f16_f16_aa_2x1; + mmctx->vec_dot_2x2 = flat_vec_dot_f16_f16_aa_2x2; + act_row_size = kparams->act_row_size; break; case HTP_MM_KERNEL_HVX_F32_F32_VTCM: quant_task_func = NULL; need_quant = false; mmctx->type = "f32-f32"; - mmctx->vec_dot_1x1 = vec_dot_f32_f32_aa_1x1; - mmctx->vec_dot_2x1 = vec_dot_f32_f32_aa_2x1; - mmctx->vec_dot_2x2 = vec_dot_f32_f32_aa_2x2; - src1_row_size = hex_round_up(ne10 * 4, 128); + mmctx->vec_dot_1x1 = flat_vec_dot_f32_f32_aa_1x1; + mmctx->vec_dot_2x1 = flat_vec_dot_f32_f32_aa_2x1; + mmctx->vec_dot_2x2 = flat_vec_dot_f32_f32_aa_2x2; + act_row_size = kparams->act_row_size; + break; + + case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: + quant_task_func = NULL; + need_quant = false; + mmctx->type = "bf16-f32"; + mmctx->vec_dot_1x1 = flat_vec_dot_bf16_f32_aa_1x1; + mmctx->vec_dot_2x1 = flat_vec_dot_bf16_f32_aa_2x1; + mmctx->vec_dot_2x2 = flat_vec_dot_bf16_f32_aa_2x2; + act_row_size = kparams->act_row_size; break; case HTP_MM_KERNEL_HVX_QUANT_BLOCK: case HTP_MM_KERNEL_HVX_QUANT_ROW: - default: - if (hvx_mm_init_vec_dot(mmctx, src0->type) != 0) { + default: { + if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) != 0) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (ne10 + qk - 1) / qk; - const uint32_t total_nb = src1_nrows * nb; + const uint32_t total_nb = act_nrows * nb; - if (src1_nrows < octx->n_threads && !is_batched) { - n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = htp_mm_act_quant_block_func(src0->type); + if (act_nrows < octx->n_hvx && !is_batched) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); + quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; @@ -1835,41 +1839,43 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = htp_mm_act_quant_row_func(src0->type); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); + quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } - src1_row_size = htp_mm_weight_has_offset(src0->type) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + act_row_size = kparams->act_row_size; break; + } } - const uint32_t m_chunk = (kparams->m_chunk > 0 && (uint32_t) kparams->m_chunk < src1_nrows) - ? (uint32_t) kparams->m_chunk : src1_nrows; + const uint32_t m_chunk = (kparams->m_chunk > 0 && (uint32_t) kparams->m_chunk < act_nrows) + ? (uint32_t) kparams->m_chunk : act_nrows; const uint32_t m_layout_rows = m_chunk; struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_threads, - dst_row_size, src0_row_size, src1_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false); + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_hvx, + dst_row_size, src0_row_size, act_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false); if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || kparams->kernel_type == HTP_MM_KERNEL_HVX_F32_F32_VTCM || + kparams->kernel_type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { - mmctx->vtcm_src1_size_per_thread = L.src1_bytes; + mmctx->vtcm_act_size_per_hvx = L.act_bytes; } else { - mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->n_threads_div); + mmctx->vtcm_act_size_per_hvx = fastdiv(L.act_bytes, &octx->n_hvx_div); } - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); const size_t vtcm_size = L.total_bytes; - FARF(HIGH, "matmul-%s : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu (%zu)\n", mmctx->type, - L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); + FARF(HIGH, "matmul-%s : src0-vtcm-size %zu act-vtcm-size %zu dst-vtcm-size %zu (%zu)\n", mmctx->type, + L.src0_bytes, L.act_bytes, L.dst_bytes, vtcm_size); FARF(HIGH, "matmul-%s : %ux%ux%ux%u * %ux%ux%ux%u-> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\n", mmctx->type, src0->ne[0], - src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], - dst->ne[1], dst->ne[2], dst->ne[3], src0->data, src1->data, dst->data); + src0->ne[1], src0->ne[2], src0->ne[3], act->ne[0], act->ne[1], act->ne[2], act->ne[3], dst->ne[0], + dst->ne[1], dst->ne[2], dst->ne[3], src0->data, act->data, dst->data); if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-%s : current VTCM reservation %zu is too small, needed %zu\n", mmctx->type, @@ -1878,20 +1884,16 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { } uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + mmctx->vtcm_act = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act); mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); - mmctx->vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src2); + mmctx->vtcm_bias = VTCM_LAYOUT_PTR(uint8_t, base, L.off_bias); mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); mmctx->vtcm_act_raw = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act_raw); - octx->src1_spad.src = NULL; - octx->src0_spad.src = NULL; - octx->dst_spad.src = NULL; - mmctx->vtcm_src0_stride = src0_row_size_padded; - mmctx->vtcm_src1_stride = src1_row_size; + mmctx->vtcm_act_stride = act_row_size; if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW) { - mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10); } else if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) { mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), 128); } else { @@ -1900,23 +1902,23 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - if (kparams->m_chunk > 0 && (uint32_t) kparams->m_chunk < src1_nrows) { - for (uint32_t m_start = 0; m_start < src1_nrows; m_start += m_chunk) { - const uint32_t cur_m_rows = MIN(src1_nrows - m_start, m_chunk); + if (kparams->m_chunk > 0 && (uint32_t) kparams->m_chunk < act_nrows) { + for (uint32_t m_start = 0; m_start < act_nrows; m_start += m_chunk) { + const uint32_t cur_m_rows = MIN(act_nrows - m_start, m_chunk); mmctx->cur_m_start = m_start; mmctx->cur_m_rows = cur_m_rows; if (need_quant) { - hvx_mm_transfer_src1_dma(octx, kparams, src1, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, m_start, cur_m_rows); + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, m_start, cur_m_rows); - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (ne10 + qk - 1) / qk; const uint32_t total_nb = cur_m_rows * nb; uint32_t quant_tasks; work_queue_func_t q_func; - if (cur_m_rows < octx->n_threads && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) { - quant_tasks = MIN(total_nb, octx->n_threads); - q_func = htp_mm_act_quant_block_func(src0->type); + if (cur_m_rows < octx->n_hvx && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) { + quant_tasks = MIN(total_nb, octx->n_hvx); + q_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / quant_tasks; @@ -1926,193 +1928,53 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - quant_tasks = MIN(cur_m_rows, octx->n_threads); + quant_tasks = MIN(cur_m_rows, octx->n_hvx); q_func = quant_task_func; - mmctx->n_quant_rows_per_thread = (cur_m_rows + quant_tasks - 1) / quant_tasks; + mmctx->n_quant_rows_per_hvx = (cur_m_rows + quant_tasks - 1) / quant_tasks; } mmctx->n_quant_tasks = quant_tasks; - work_queue_run(octx->ctx->work_queue, q_func, mmctx, quant_tasks); + work_queue_run(octx->ctx->hvx_queue, q_func, mmctx, quant_tasks); } else { - hvx_mm_transfer_src1_dma(octx, kparams, src1, mmctx->vtcm_src1, mmctx->vtcm_src1_stride, m_start, cur_m_rows); + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, m_start, cur_m_rows); } - work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, matmul_job_func, mmctx, octx->n_hvx); } } else { mmctx->cur_m_start = 0; - mmctx->cur_m_rows = src1_nrows; + mmctx->cur_m_rows = act_nrows; if (need_quant) { - hvx_mm_transfer_src1_dma(octx, kparams, src1, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, src1_nrows); - mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); } else { - hvx_mm_transfer_src1_dma(octx, kparams, src1, mmctx->vtcm_src1, mmctx->vtcm_src1_stride, 0, src1_nrows); + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, 0, act_nrows); } - work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads); - } - - return HTP_STATUS_OK; -} - -static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { - struct htp_mm_context * mmctx = data; - struct htp_ops_context * octx = mmctx->octx; - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - const uint32_t n_weights = kparams->n_weights; - - const struct htp_tensor * restrict act = octx->src[n_weights]; - const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; - const size_t src1_stride = mmctx->vtcm_src1_stride; - - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; - uint8_t * restrict src1_data = mmctx->vtcm_src1; - - const uint32_t n_prefetch = kparams->n_prefetch; - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - const uint32_t prefetch_mask = n_prefetch - 1; - - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - - - for (uint32_t widx = 0; widx < n_weights; widx++) { - const struct htp_tensor * restrict src_w = octx->src[widx]; - const struct htp_tensor * restrict dst = octx->dsts[widx]; - if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; - - const uint32_t ne00 = src_w->ne[0]; - const uint32_t ne01 = src_w->ne[1]; - uint32_t start_row = 0; - uint32_t end_row = ne01; - if (octx->ctx->mdev.count > 1) { - const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); - start_row = range.start; - end_row = range.start + range.count; - } - - const uint32_t nrows = end_row - start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread += (src0_nrows_per_thread & 1); - - const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); - const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - if (src0_start_row >= src0_end_row) continue; - - const size_t dst_row_size = dst->nb[1]; - const size_t src0_row_size = src_w->nb[1]; - const size_t src0_stride = hex_round_up(src0_row_size, 128); - - const dma_addr_t src0_row = src_w->data; - - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const int is0 = (ir0 - src0_start_row); - if (is0 >= (int)n_prefetch) break; - dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); - } - - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - uint32_t ir1 = 0; - for (; ir1 + 1 < src1_nrows; ir1 += 2) { - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); - float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); - float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); - mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); - } - for (; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); - mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col); - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - - const int pr0 = (ir0 + n_prefetch); - const int is0 = (pr0 - src0_start_row) & prefetch_mask; - if (pr0 < src0_end_row_x2) { - dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); - } - } - - if (src0_end_row != src0_end_row_x2) { - uint32_t ir0 = src0_end_row_x2; - const int is0 = (ir0 - src0_start_row) & prefetch_mask; - dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 1); - const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); - mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - } + work_queue_run(octx->ctx->hvx_queue, matmul_job_func, mmctx, octx->n_hvx); } -} -#define DEQUANTIZE_WORKER_LOOP_IMPL(SUFFIX) \ -static void dequantize_tiled_worker_loop_##SUFFIX(unsigned int n, unsigned int i, void *data) { \ - tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; \ - struct htp_thread_trace * tr = &state->traces[i]; \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); \ - for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { \ - int start = task_id * state->n_tiles_per_task; \ - int end = hex_smin(start + state->n_tiles_per_task, state->n_tot_tiles); \ - dequantize_tiled_weight_to_fp16_task_##SUFFIX(state, start, end); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); \ + return HTP_STATUS_OK; } -DEQUANTIZE_WORKER_LOOP_IMPL(q4_0) -DEQUANTIZE_WORKER_LOOP_IMPL(q4_1) -DEQUANTIZE_WORKER_LOOP_IMPL(iq4_nl) -DEQUANTIZE_WORKER_LOOP_IMPL(mxfp4) -DEQUANTIZE_WORKER_LOOP_IMPL(q8_0) -DEQUANTIZE_WORKER_LOOP_IMPL(q6_k) -DEQUANTIZE_WORKER_LOOP_IMPL(q5_k) -DEQUANTIZE_WORKER_LOOP_IMPL(q3_k) -DEQUANTIZE_WORKER_LOOP_IMPL(q2_k) - -static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) { - tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; - struct htp_thread_trace * tr = &state->traces[i]; +static void dequantize_tiled_worker_loop(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + tiled_dequantize_state_t * state = (tiled_dequantize_state_t *) data; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); - for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { + for (unsigned int task_id = i; task_id < (unsigned int) state->n_tasks; task_id += n) { int start = task_id * state->n_tiles_per_task; int end = hex_smin(start + state->n_tiles_per_task, state->n_tot_tiles); - convert_f16_weight_to_fp16_tiles_task(state, start, end); + state->task_fn(state, start, end); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); } -static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data) { - tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; - - struct htp_thread_trace * tr = &state->traces[i]; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, i); - - for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { - int start = task_id * state->n_tiles_per_task; - int end = hex_smin(start + state->n_tiles_per_task, state->n_tot_tiles); - quantize_f32_weight_to_fp16_tiles_task(state, start, end); - } - - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, i); -} - -static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { output_transfer_task_state_t *st = (output_transfer_task_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int start_chunk_idx = i * st->n_chunks_per_task; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx); @@ -2130,8 +1992,6 @@ static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void } typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; __fp16 * dst; dma_addr_t act_dma_addr; float * vtcm_f32_act; @@ -2141,14 +2001,13 @@ typedef struct { uint32_t k_block; uint32_t k_stride; uint32_t k_valid; - size_t vtcm_f32_act_bytes_per_thread; + size_t vtcm_f32_act_bytes_per_hvx; uint32_t dma_step_rows; uint32_t dma_step_rows_shift; + uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1 } activation_transfer_task_state_t; typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; __fp16 * dst; dma_addr_t act_dma_addr; float * vtcm_f32_act; @@ -2157,7 +2016,7 @@ typedef struct { uint32_t k_stride; uint32_t k_valid; uint32_t n_col_chunks; - struct fastdiv_values n_threads_div; + struct fastdiv_values n_hvx_div; size_t vtcm_f32_act_bytes; uint32_t dma_step_rows; uint32_t dma_step_rows_shift; @@ -2231,16 +2090,16 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( } } -static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_col_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; uint32_t n_blocks = st->k_block / 32; - uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); - uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t b_first = fastdiv(n_blocks * i, &st->n_hvx_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_hvx_div); uint32_t c_first = b_first * 32; - uint32_t c_last = b_last * 32; - uint32_t c_len = c_last - c_first; + uint32_t c_last = b_last * 32; + uint32_t c_len = c_last - c_first; if (c_len == 0) { return; @@ -2253,16 +2112,16 @@ static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsign __fp16 *dst = st->dst; - size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128); + size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_hvx_div), 128); float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes); transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( - st->ctx->dma[i], dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, + th->dma_q, dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift ); } -static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( +static void transfer_activation_chunk_to_fp16_dma_pipelined( dma_queue *dma_q, __fp16 *restrict vtcm_dst, dma_addr_t act_dma_addr, @@ -2270,7 +2129,8 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( uint32_t k_block, uint32_t k_stride, uint32_t k_valid, - float *thread_f32_act, + uint8_t *thread_act, + uint32_t act_elem_size, struct htp_thread_trace *tr, uint32_t dma_step_rows, uint32_t dma_step_rows_shift) { @@ -2280,38 +2140,56 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( const uint32_t n_steps = n_rows_padded >> dma_step_rows_shift; + const bool act_is_f16 = (act_elem_size == sizeof(__fp16)); + const size_t row_bytes = (size_t) k_block * act_elem_size; // staging row (DMA dst stride) + const size_t src_row_bytes = (size_t) k_stride * act_elem_size; // DDR row (DMA src stride) + const size_t width_bytes = (size_t) k_valid * act_elem_size; + // Push step 0 if (n_steps > 0 && n_rows > 0) { uint32_t nrows_to_fetch = hex_smin(n_rows, R); - dma_queue_push(dma_q, dma_make_data(thread_f32_act, act_dma_addr), - k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); + dma_queue_push(dma_q, dma_make_data(thread_act, act_dma_addr), + row_bytes, src_row_bytes, width_bytes, nrows_to_fetch); } // Push step 1 (if valid) if (n_steps > 1) { uint32_t next_r = R * 1; if (next_r < n_rows) { uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); - float *next_buf = thread_f32_act + 1 * R * k_block; - dma_queue_push(dma_q, dma_make_data(next_buf, act_dma_addr + (size_t) next_r * k_stride * sizeof(float)), - k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); + uint8_t *next_buf = thread_act + 1 * R * row_bytes; + dma_queue_push(dma_q, dma_make_data(next_buf, act_dma_addr + (size_t) next_r * src_row_bytes), + row_bytes, src_row_bytes, width_bytes, nrows_to_fetch); } } for (uint32_t s = 0; s < n_steps; ++s) { uint32_t r = s << dma_step_rows_shift; - float *curr_buf = thread_f32_act; + uint8_t *curr_buf = thread_act; if (r < n_rows) { - curr_buf = (float *) dma_queue_pop(dma_q).dst; + curr_buf = (uint8_t *) dma_queue_pop(dma_q).dst; } htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, r); - for (uint32_t p = 0; p < (R >> 1); ++p) { - uint32_t row_idx = r + (p << 1); - float *pair_buf = curr_buf + (p << 1) * k_block; - bool r0_valid = ((row_idx + 0) < n_rows); - bool r1_valid = ((row_idx + 1) < n_rows); + // Two copies of the pair loop so the type is resolved once per step and the + // row-pair kernels stay direct (inlinable) calls. + if (act_is_f16) { + for (uint32_t p = 0; p < (R >> 1); ++p) { + uint32_t row_idx = r + (p << 1); + const __fp16 *pair_buf = (const __fp16 *) (curr_buf + (p << 1) * row_bytes); + bool r0_valid = ((row_idx + 0) < n_rows); + bool r1_valid = ((row_idx + 1) < n_rows); + + transfer_activation_row_pair_f16_to_f16(vtcm_dst, pair_buf, pair_buf + k_block, row_idx, k_block, k_valid, r0_valid, r1_valid); + } + } else { + for (uint32_t p = 0; p < (R >> 1); ++p) { + uint32_t row_idx = r + (p << 1); + const float *pair_buf = (const float *) (curr_buf + (p << 1) * row_bytes); + bool r0_valid = ((row_idx + 0) < n_rows); + bool r1_valid = ((row_idx + 1) < n_rows); - transfer_activation_row_pair_fp32_to_fp16(vtcm_dst, pair_buf, pair_buf + k_block, row_idx, k_block, k_valid, r0_valid, r1_valid); + transfer_activation_row_pair_fp32_to_fp16(vtcm_dst, pair_buf, pair_buf + k_block, row_idx, k_block, k_valid, r0_valid, r1_valid); + } } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, r); @@ -2320,33 +2198,32 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( uint32_t next_r = next_s << dma_step_rows_shift; if (next_r < n_rows) { uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); - dma_queue_push(dma_q, dma_make_data(curr_buf, act_dma_addr + (size_t) next_r * k_stride * sizeof(float)), - k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); + dma_queue_push(dma_q, dma_make_data(curr_buf, act_dma_addr + (size_t) next_r * src_row_bytes), + row_bytes, src_row_bytes, width_bytes, nrows_to_fetch); } } } -static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { int chunk_idx = task_id * st->n_chunks_per_task; size_t chunk_size = hex_smin(st->n_tot_chunks - chunk_idx, st->n_chunks_per_task); __fp16 *dst = st->dst + chunk_idx * st->k_block; - const dma_addr_t act_dma_addr = st->act_dma_addr + (size_t) chunk_idx * st->k_stride * sizeof(float); + const dma_addr_t act_dma_addr = st->act_dma_addr + (size_t) chunk_idx * st->k_stride * st->act_elem_size; - float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_thread); - transfer_activation_chunk_fp32_to_fp16_dma_pipelined( - st->ctx->dma[i], dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift + uint8_t *thread_act = (uint8_t *) st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_hvx; + transfer_activation_chunk_to_fp16_dma_pipelined( + th->dma_q, dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_act, st->act_elem_size, tr, st->dma_step_rows, st->dma_step_rows_shift ); } } typedef struct { - struct htp_thread_trace * traces; const struct mmid_row_mapping * matrix_rows; __fp16 * dst; const float * src; @@ -2366,7 +2243,6 @@ typedef struct { } activation_transfer_gathered_task_state_t; typedef struct { - struct htp_thread_trace * traces; const struct mmid_row_mapping * matrix_rows; const __fp16 * vtcm_src; float * dst; @@ -2382,9 +2258,9 @@ typedef struct { uint32_t cne1; } output_transfer_scattered_task_state_t; -static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_gathered_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2400,9 +2276,9 @@ static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigne } } -static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_gathered_worker_flat_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2418,9 +2294,9 @@ static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, un } } -static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_scattered_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { output_transfer_scattered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2438,21 +2314,48 @@ static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned i // --- HMX Dispatchers & Entry Points --- +static inline tiled_dequant_task_fn_t hmx_get_dequant_worker_fn(int weight_type, bool is_tiled) { + switch (weight_type) { + case HTP_TYPE_Q4_0: return dequantize_tiled_weight_to_fp16_task_q4_0; + case HTP_TYPE_IQ4_NL: return dequantize_tiled_weight_to_fp16_task_iq4_nl; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: return dequantize_tiled_weight_to_fp16_task_q4_1; + case HTP_TYPE_MXFP4: return dequantize_tiled_weight_to_fp16_task_mxfp4; + case HTP_TYPE_Q8_0: return dequantize_tiled_weight_to_fp16_task_q8_0; + case HTP_TYPE_Q5_0: return dequantize_tiled_weight_to_fp16_task_q5_0; + case HTP_TYPE_Q5_1: + case HTP_TYPE_Q5_K: return dequantize_tiled_weight_to_fp16_task_q5_k; + case HTP_TYPE_Q6_K: return dequantize_tiled_weight_to_fp16_task_q6_k; + case HTP_TYPE_Q3_K: return dequantize_tiled_weight_to_fp16_task_q3_k; + case HTP_TYPE_Q2_K: return dequantize_tiled_weight_to_fp16_task_q2_k; + case HTP_TYPE_Q1_0: return dequantize_tiled_weight_to_fp16_task_q1_0; + case HTP_TYPE_Q2_0: return dequantize_tiled_weight_to_fp16_task_q2_0; + case HTP_TYPE_F16: return is_tiled ? NULL : convert_f16_weight_to_fp16_tiles_task; + case HTP_TYPE_BF16: return convert_bf16_weight_to_fp16_tiles_task; + case HTP_TYPE_F32: return quantize_f32_weight_to_fp16_tiles_task; + default: return NULL; + } +} + static void dequantize_tiled_weight_chunk_to_fp16_tiles( struct htp_context *ctx, __fp16 *vtcm_dst, const void *weight_src_ddr, int n_cols, int k_block, size_t row_stride, int weight_type, int n_k_tiles, struct fastdiv_values n_k_tiles_div, - worker_callback_t dequant_worker_fn, int n_threads) { + tiled_dequant_task_fn_t dequant_worker_fn, int n_hvx) { + + if (!dequant_worker_fn) { + return; + } - assert(n_cols % HTP_MM_HMX_TILE_N_COLS == 0); assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0); - size_t n_col_tiles = n_cols / HTP_MM_HMX_TILE_N_COLS; - size_t n_tot_tiles = n_col_tiles * n_k_tiles; + const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); + const uint32_t n_weight_tiles = k_block / htp_tensor_get_tile_k(weight_type); + const size_t n_tot_tiles = n_col_tiles * n_weight_tiles; - size_t n_tiles_per_task = (n_threads == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_threads); + size_t n_tiles_per_task = (n_hvx == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_hvx); tiled_dequantize_state_t state; state.n_tasks = (n_tot_tiles + n_tiles_per_task - 1) / n_tiles_per_task; @@ -2460,29 +2363,23 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( state.n_tiles_per_task = n_tiles_per_task; state.dst = vtcm_dst; state.src = (const uint8_t *)weight_src_ddr; + state.task_fn = dequant_worker_fn; state.n_cols = n_cols; state.k_block = k_block; state.row_stride = row_stride; state.weight_type = weight_type; state.n_k_tiles = n_k_tiles; + state.n_weight_tiles = n_weight_tiles; state.n_k_tiles_div = n_k_tiles_div; - state.traces = ctx->trace; - state.ctx = ctx; - state.tile_size = htp_mm_get_weight_tile_size(weight_type); - state.aligned_tile_size = htp_mm_get_weight_aligned_tile_size(weight_type); + state.tile_size = htp_tensor_get_tile_size(weight_type); + state.aligned_tile_size = htp_tensor_get_tile_padded_size(weight_type); - if (state.n_tasks == 1 || n_threads == 1) { - dequant_worker_fn(1, 0, &state); - } else { - int n_tasks = hex_smin((int) state.n_tasks, n_threads); - worker_pool_run_func(ctx->worker_pool, dequant_worker_fn, &state, n_tasks); - } + int n_tasks = hex_smin((int) state.n_tasks, n_hvx); + work_queue_run(ctx->hvx_queue, dequantize_tiled_worker_loop, &state, n_tasks); } typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; float * dst; const __fp16 * vtcm_src; const float * src2; @@ -2491,17 +2388,19 @@ typedef struct { uint32_t dst_stride; uint32_t src2_stride; uint32_t dst_cols; - struct fastdiv_values n_threads_div; + struct fastdiv_values n_hvx_div; } output_transfer_col_chunk_state_t; -static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_col_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { (void) n; output_transfer_col_chunk_state_t *st = (output_transfer_col_chunk_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; - uint32_t n_blocks = st->n_cols / 32; - uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); - uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + // Round up: the last block is partial when N is not 32-aligned. Its pad + // columns are dropped by the dst_cols clamp inside the store. + uint32_t n_blocks = hmx_ceil_div(st->n_cols, 32); + uint32_t b_first = fastdiv(n_blocks * i, &st->n_hvx_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_hvx_div); uint32_t c_first = b_first * 32; uint32_t c_last = b_last * 32; uint32_t c_len = c_last - c_first; @@ -2526,14 +2425,12 @@ static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned i } static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, const float *src2, const __fp16 *vtcm_src, - int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_threads) { - assert(n_cols % HTP_MM_HMX_TILE_N_COLS == 0); - + int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_hvx) { if (n_rows <= 0) return; - uint32_t n_blocks = (uint32_t)n_cols / 32; - if (n_threads > 1 && n_blocks >= (uint32_t)n_threads) { - struct fastdiv_values n_threads_div = (n_threads == (int)ctx->n_threads) ? ctx->n_threads_div : init_fastdiv_values(n_threads); + uint32_t n_blocks = hmx_ceil_div((uint32_t) n_cols, 32); + if (n_hvx > 1 && n_blocks >= (uint32_t)n_hvx) { + struct fastdiv_values n_hvx_div = (n_hvx == (int)ctx->n_hvx) ? ctx->n_hvx_div : init_fastdiv_values(n_hvx); output_transfer_col_chunk_state_t col_state; col_state.dst = dst; col_state.src2 = src2; @@ -2543,22 +2440,20 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, col_state.dst_stride = (uint32_t)dst_stride; col_state.src2_stride = src2_stride; col_state.dst_cols = (uint32_t)dst_cols; - col_state.n_threads_div = n_threads_div; - col_state.traces = ctx->trace; - col_state.ctx = ctx; + col_state.n_hvx_div = n_hvx_div; - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_threads); + work_queue_run(ctx->hvx_queue, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_hvx); return; } size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_threads); + size_t n_chunks_per_task = (n_hvx == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_hvx); n_chunks_per_task = hex_align_up(n_chunks_per_task, 2); - int actual_threads = hmx_ceil_div(n_rows, n_chunks_per_task); + int actual_hvx = hmx_ceil_div(n_rows, n_chunks_per_task); output_transfer_task_state_t state; - state.n_tasks = actual_threads; + state.n_tasks = actual_hvx; state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; @@ -2568,13 +2463,8 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, state.dst_stride = dst_stride; state.src2_stride = src2_stride; state.dst_cols = dst_cols; - state.traces = ctx->trace; - if (actual_threads <= 1) { - transfer_output_chunk_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, transfer_output_chunk_worker_fn, &state, actual_hvx); } struct activation_transfer_params { @@ -2584,12 +2474,13 @@ struct activation_transfer_params { int n_rows; int k_block; int k_stride; - int n_threads; - const struct fastdiv_values * act_threads_div; + int n_hvx; + const struct fastdiv_values * act_hvx_div; const struct fastdiv_values * k_div; int k_valid; float * vtcm_f32_act; size_t vtcm_f32_act_bytes; + uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1 }; static void transfer_activation_chunk_threaded(const struct activation_transfer_params * params) { @@ -2599,24 +2490,26 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ int n_rows = params->n_rows; int k_block = params->k_block; int k_stride = params->k_stride; - int n_threads = params->n_threads; - const struct fastdiv_values * act_threads_div = params->act_threads_div; + int n_hvx = params->n_hvx; + const struct fastdiv_values * act_hvx_div = params->act_hvx_div; const struct fastdiv_values * k_div = params->k_div; int k_valid = params->k_valid; float * vtcm_f32_act = params->vtcm_f32_act; size_t vtcm_f32_act_bytes = params->vtcm_f32_act_bytes; + // element size of the activation rows (4 = F32, 2 = F16). + const uint32_t act_elem_size = params->act_elem_size ? params->act_elem_size : (uint32_t) sizeof(float); if (n_rows <= 0) { return; } const size_t n_tasks = (n_rows + 31) >> 5; - if (n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) { + if (act_elem_size == sizeof(float) && n_hvx > 1 && k_block > 32 && n_tasks < (size_t) n_hvx) { // Calculate step rows parameters for column-chunked dma pipelining uint32_t dma_step_rows = 2; uint32_t dma_step_rows_shift = 1; if (vtcm_f32_act_bytes > 0) { - size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_hvx_div), 128); size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float); size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); if (dma_step_rows_max >= 4) { @@ -2632,26 +2525,24 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ col_state.k_block = k_block; col_state.k_stride = k_stride; col_state.k_valid = k_valid; - col_state.n_col_chunks = n_threads; - col_state.n_threads_div = *act_threads_div; + col_state.n_col_chunks = n_hvx; + col_state.n_hvx_div = *act_hvx_div; col_state.vtcm_f32_act = vtcm_f32_act; col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes; - col_state.traces = ctx->trace; - col_state.ctx = ctx; col_state.dma_step_rows = dma_step_rows; col_state.dma_step_rows_shift = dma_step_rows_shift; - worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads); + work_queue_run(ctx->hvx_queue, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_hvx); return; } assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0); size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address + size_t n_chunks_per_task = (n_hvx == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address activation_transfer_task_state_t state; - state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); + state.n_tasks = (n_hvx == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; @@ -2659,16 +2550,15 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ state.k_block = k_block; state.k_stride = k_stride; state.k_valid = k_valid; - state.traces = ctx->trace; - state.ctx = ctx; state.vtcm_f32_act = vtcm_f32_act; + state.act_elem_size = act_elem_size; - state.vtcm_f32_act_bytes_per_thread = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + state.vtcm_f32_act_bytes_per_hvx = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_hvx_div), 128); uint32_t dma_step_rows = 2; uint32_t dma_step_rows_shift = 1; - if (state.vtcm_f32_act_bytes_per_thread > 0) { - size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float); + if (state.vtcm_f32_act_bytes_per_hvx > 0) { + size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_hvx / sizeof(float); size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); if (dma_step_rows_max >= 4) { dma_step_rows = 4; @@ -2678,12 +2568,8 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ state.dma_step_rows = dma_step_rows; state.dma_step_rows_shift = dma_step_rows_shift; - int active_threads = hex_smin(n_threads, (int)state.n_tasks); - if (state.n_tasks == 1 || n_threads == 1) { - transfer_activation_chunk_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_worker_fn, &state, active_threads); - } + int active_hvx = hex_smin(n_hvx, (int)state.n_tasks); + work_queue_run(ctx->hvx_queue, transfer_activation_chunk_worker_fn, &state, active_hvx); } // --- Async HMX matmul job (for pipeline overlap) --- @@ -2697,7 +2583,7 @@ typedef struct { uint32_t n_dot_tiles; } hmx_matmul_job_t; -static void hmx_matmul_worker_fn(void * data) { +static void hmx_matmul_worker_fn(const htp_thread_t * th, void * data) { hmx_matmul_job_t * job = (hmx_matmul_job_t *) data; FARF(HIGH, "hmx-mm-job: n_row_tiles %u n_col_tiles %u n_dot_tiles %u", job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); @@ -2729,6 +2615,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, dma_addr_t weight, int m, int k, int n, int act_stride, + uint32_t act_elem_size, // 4=F32 src1, 2=F16 src1 int weight_stride, int weight_type, int k_valid, @@ -2738,9 +2625,9 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, int m_chunk, int n_chunk, int pipeline, - int n_threads, - int act_threads, - const struct fastdiv_values * act_threads_div, + int n_hvx, + int act_hvx, + const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div, int tile_size, int aligned_tile_size, @@ -2748,51 +2635,37 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - if (k % 32 != 0 || n % 32 != 0) { return -1; } - if (!hex_is_aligned(dst, VLEN) || (act_dma_addr & (VLEN - 1)) != 0) { return -1; } + const bool is_tiled = (tile_size > 0); - size_t row_stride = htp_mm_get_tiled_row_stride(weight_type, k); - if (row_stride == 0) { + if (k % 32 != 0) { return -1; } + if (!hex_is_aligned(dst, sizeof(float))) { return -1; } - worker_callback_t dequant_worker_fn = NULL; - switch (weight_type) { - case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break; - case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; - case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break; - case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break; - case HTP_TYPE_Q5_K: dequant_worker_fn = dequantize_tiled_worker_loop_q5_k; break; - case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break; - case HTP_TYPE_Q3_K: dequant_worker_fn = dequantize_tiled_worker_loop_q3_k; break; - case HTP_TYPE_Q2_K: dequant_worker_fn = dequantize_tiled_worker_loop_q2_k; break; - case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break; - case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break; - default: - return -1; + size_t row_stride = htp_tensor_get_row_size(weight_type, k); + if (row_stride == 0) { + return -1; } + tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled); + const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32); const size_t vec_dot_size = k * sizeof(__fp16); const size_t vtcm_budget = ctx->vtcm_size; - const uint32_t dma_dst_stride = is_quant ? aligned_tile_size : row_stride; - const uint32_t dma_src_stride = is_quant ? tile_size : weight_stride; - const uint32_t dma_width_bytes = is_quant ? tile_size : row_stride; + const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride; + const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; + const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; size_t m_chunk_n_rows = m_chunk; size_t n_chunk_n_cols = n_chunk; size_t vtcm_used = vtcm_size; - const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; - struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, src2_bytes); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_hvx, aligned_tile_size, src2_bytes); vtcm_used = L.total_bytes; if (vtcm_used > vtcm_budget) { @@ -2806,7 +2679,6 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, VTCM_LAYOUT_PTR(__fp16, base, L.off_weight[0]), VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_weight[1], pipeline) }; - __fp16 *vtcm_f16_act = VTCM_LAYOUT_PTR(__fp16, base, L.off_act); float *vtcm_f32_act = VTCM_LAYOUT_PTR(float, base, L.off_act_f32); __fp16 *vtcm_output = VTCM_LAYOUT_PTR(__fp16, base, L.off_dst[0]); @@ -2817,10 +2689,10 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - const bool has_src2 = (src2_bytes > 0 && src2_addr != 0); - float *vtcm_src2 = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_src2, has_src2); - if (has_src2) { - dma_queue_push(weight_dma, dma_make_data(vtcm_src2, src2_addr), hex_align_up(src2_bytes, 128), 0, src2_bytes, 1); + const bool has_bias = (src2_bytes > 0 && src2_addr != 0); + float *vtcm_bias = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_bias, has_bias); + if (has_bias) { + dma_queue_push(weight_dma, dma_make_data(vtcm_bias, src2_addr), hex_align_up(src2_bytes, 128), 0, src2_bytes, 1); dma_queue_pop(weight_dma); } @@ -2834,39 +2706,50 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, if (pipeline) { // --- Asynchronous Pipelined Loop --- hmx_matmul_job_t job_slots[2]; // persistent double-buffered job descriptors + void *vtcm_raw_bufs[4] = { + vtcm_weight_raw[0], + vtcm_weight_raw[1], + dequant_worker_fn ? vtcm_weight_raw[0] : vtcm_scratch0, + dequant_worker_fn ? vtcm_weight_raw[1] : vtcm_scratch1, + }; + void *vtcm_weight_bufs[4] = { + dequant_worker_fn ? vtcm_scratch0 : vtcm_weight_raw[0], + dequant_worker_fn ? vtcm_scratch1 : vtcm_weight_raw[1], + vtcm_scratch0, + vtcm_scratch1, + }; + void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); - void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; - void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; - struct activation_transfer_params act_params = { .ctx = ctx, .dst = vtcm_f16_act, - .act_dma_addr = act_dma_addr + mr * act_stride * sizeof(float), + .act_dma_addr = act_dma_addr + (size_t) mr * act_stride * act_elem_size, // byte offset (F16/F32) .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = act_elem_size, }; transfer_activation_chunk_threaded(&act_params); // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1) const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); - const uint32_t height_A0 = is_quant ? (n_cols_A0 / 32) * n_k_tiles : n_cols_A0; - dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[0], weight), + const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); if (1 < n_chunk_cnt) { const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); - const uint32_t height_A1 = is_quant ? (n_cols_A1 / 32) * n_k_tiles : n_cols_A1; - dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), + const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); } @@ -2883,20 +2766,20 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, // 2. dequantize A_i dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_weight_bufs[i % 2], curr_raw, + ctx, vtcm_weight_bufs[i % 4], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); // 3. push A_{i+2} (if i+2 < n_chunk_cnt) if (i + 2 < n_chunk_cnt) { - const uint32_t height_p2 = is_quant ? (n_cols_p2 / 32) * n_k_tiles : n_cols_p2; - dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_p2 * weight_stride), + const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); } // 4. submit C_i hmx_matmul_job_init(&job_slots[i % 2], (__fp16 *) vtcm_output_bufs[i % 2], - (__fp16 *) vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], + (__fp16 *) vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 4], vtcm_scales, hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS), hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS), k / HTP_MM_HMX_TILE_N_ROWS); hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[i % 2])); @@ -2907,10 +2790,10 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const size_t nc_prev = (i - 1) * n_chunk_n_cols; const size_t n_cols_prev = hex_smin(n - nc_prev, n_chunk_n_cols); float *output_chunk = dst + (mr * dst_stride + nc_prev); - const float *src2_chunk = has_src2 ? (vtcm_src2 + mr * src2_stride + nc_prev) : NULL; + const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_prev) : NULL; int chunk_dst_cols = dst_cols - (int)nc_prev; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } @@ -2920,42 +2803,47 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const size_t nc_last = (n_chunk_cnt - 1) * n_chunk_n_cols; const size_t n_cols_last = hex_smin(n - nc_last, n_chunk_n_cols); float *output_chunk = dst + (mr * dst_stride + nc_last); - const float *src2_chunk = has_src2 ? (vtcm_src2 + mr * src2_stride + nc_last) : NULL; + const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_last) : NULL; int chunk_dst_cols = dst_cols - (int)nc_last; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } else { // --- Synchronous loop (m <= 32 or fallback) --- hmx_matmul_job_t job; + void *vtcm_raw_bufs[2] = { vtcm_weight_raw[0], dequant_worker_fn ? vtcm_weight_raw[0] : vtcm_scratch0 }; + void *vtcm_weight_bufs[2] = { dequant_worker_fn ? vtcm_scratch0 : vtcm_weight_raw[0], vtcm_scratch0 }; + for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); struct activation_transfer_params act_params = { .ctx = ctx, .dst = vtcm_f16_act, - .act_dma_addr = act_dma_addr + mr * act_stride * sizeof(float), + .act_dma_addr = act_dma_addr + (size_t) mr * act_stride * act_elem_size, // byte offset (F16/F32) .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = act_elem_size, }; transfer_activation_chunk_threaded(&act_params); // A0: Pre-fetch the first weight chunk (nc = 0) if (n > 0) { const size_t n_cols = hex_smin(n, n_chunk_n_cols); - const uint32_t height = is_quant ? (n_cols / 32) * n_k_tiles : n_cols; - dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } - for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { + for (int i = 0; i < n_chunk_cnt; ++i) { + const size_t nc = (size_t) i * n_chunk_n_cols; const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); @@ -2965,29 +2853,29 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, // B: Weight Dequantize (Threaded) dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_scratch0, curr_raw, + ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); // Start weight DMA for the next chunk early const size_t nc_next = nc + n_chunk_n_cols; if (nc_next < n) { const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); - const uint32_t height_next = is_quant ? (n_cols_next / 32) * n_k_tiles : n_cols_next; - dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } // C: HMX Compute (Queue-based) - hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, vtcm_scratch0, vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); hmx_queue_pop(ctx->hmx_queue); // D: Output Store float *output_chunk = dst + (mr * dst_stride + nc); - const float *src2_chunk = has_src2 ? (vtcm_src2 + mr * src2_stride + nc) : NULL; + const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc) : NULL; int chunk_dst_cols = dst_cols - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } @@ -3013,56 +2901,41 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const int k = (int) act->ne[0]; const int k_valid = (int) act->ne[0]; const int m = (int) (act->ne[1] * act->ne[2] * act->ne[3]); - const int act_stride = (int) (act->nb[1] / sizeof(float)); + const uint32_t act_elem_size = (act->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + const int act_stride = (int) (act->nb[1] / act_elem_size); const dma_addr_t act_dma_addr = act->data; if (k % 32 != 0) { return HTP_STATUS_NO_SUPPORT; } - if ((act_dma_addr & (VLEN - 1)) != 0) { return HTP_STATUS_NO_SUPPORT; } - size_t row_stride = htp_mm_get_tiled_row_stride(weight_type, k); + size_t row_stride = htp_tensor_get_row_size(weight_type, k); if (row_stride == 0) { return HTP_STATUS_NO_SUPPORT; } - worker_callback_t dequant_worker_fn = NULL; - switch (weight_type) { - case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break; - case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; - case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break; - case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break; - case HTP_TYPE_Q5_K: dequant_worker_fn = dequantize_tiled_worker_loop_q5_k; break; - case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break; - case HTP_TYPE_Q3_K: dequant_worker_fn = dequantize_tiled_worker_loop_q3_k; break; - case HTP_TYPE_Q2_K: dequant_worker_fn = dequantize_tiled_worker_loop_q2_k; break; - case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break; - case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break; - default: - return HTP_STATUS_NO_SUPPORT; - } - const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32); + const int tile_size = kparams->tile_size; + const int aligned_tile_size = kparams->aligned_tile_size; + + const bool is_tiled = (tile_size > 0); + tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled); const size_t vtcm_budget = ctx->vtcm_size; const int m_chunk_n_rows = kparams->m_chunk; const int n_chunk_n_cols = kparams->n_chunk; const int pipeline = kparams->pipeline; - const int n_threads = octx->n_threads; - const int act_threads = kparams->n_act_threads; - const struct fastdiv_values * act_threads_div = &kparams->div_n_act_threads; + const int n_hvx = octx->n_hvx; + const int act_hvx = kparams->n_act_hvx; + const struct fastdiv_values * act_hvx_div = &kparams->div_n_act_hvx; const struct fastdiv_values * k_div = &kparams->div_ne00_padded; - const int tile_size = kparams->tile_size; - const int aligned_tile_size = kparams->aligned_tile_size; - const uint32_t dma_dst_stride = is_quant ? aligned_tile_size : row_stride; - const uint32_t dma_width_bytes = is_quant ? tile_size : row_stride; + const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride; + const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, 0); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_hvx, aligned_tile_size, 0); if (L.total_bytes > vtcm_budget) { FARF(ERROR, "hmx-mm-nx-2d: VTCM overflow: used %zu budget %zu, m %d k %d mc %d nc %d", @@ -3075,7 +2948,6 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k VTCM_LAYOUT_PTR(__fp16, base, L.off_weight[0]), VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_weight[1], pipeline) }; - __fp16 *vtcm_f16_act = VTCM_LAYOUT_PTR(__fp16, base, L.off_act); float *vtcm_f32_act = VTCM_LAYOUT_PTR(float, base, L.off_act_f32); __fp16 *vtcm_output = VTCM_LAYOUT_PTR(__fp16, base, L.off_dst[0]); @@ -3089,7 +2961,16 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k int m_start = 0; int m_rows = m; if (octx->ctx->mdev.count > 1) { - const bool can_split = htp_tensor_can_row_partition(octx->dsts[0], sizeof(float)); + bool can_split = htp_tensor_can_row_partition(act, act_elem_size); + if (can_split) { + for (uint32_t p = 0; p < n_weights; ++p) { + const struct htp_tensor * restrict dst = octx->dsts[p]; + if (!htp_tensor_can_row_partition(dst, sizeof(float))) { + can_split = false; + break; + } + } + } const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); m_start = (int) range.start; m_rows = (int) range.count; @@ -3108,26 +2989,37 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k if (pipeline) { hmx_matmul_job_t job_slots[2]; + void *vtcm_raw_bufs[4] = { + vtcm_weight_raw[0], + vtcm_weight_raw[1], + dequant_worker_fn ? vtcm_weight_raw[0] : vtcm_scratch0, + dequant_worker_fn ? vtcm_weight_raw[1] : vtcm_scratch1, + }; + void *vtcm_weight_bufs[4] = { + dequant_worker_fn ? vtcm_scratch0 : vtcm_weight_raw[0], + dequant_worker_fn ? vtcm_scratch1 : vtcm_weight_raw[1], + vtcm_scratch0, + vtcm_scratch1, + }; + void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); - void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; - void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; - struct activation_transfer_params act_params = { .ctx = ctx, .dst = vtcm_f16_act, - .act_dma_addr = act_dma_addr + mr * act_stride * sizeof(float), + .act_dma_addr = act_dma_addr + (size_t) mr * act_stride * act_elem_size, .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = act_elem_size, }; transfer_activation_chunk_threaded(&act_params); @@ -3146,17 +3038,17 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const int dst_cols = (int) dst->ne[0]; const int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); - const uint32_t dma_src_stride = is_quant ? tile_size : weight_stride; + const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); - const uint32_t height_A0 = is_quant ? (n_cols_A0 / 32) * n_k_tiles : n_cols_A0; - dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[0], weight), + const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); if (1 < n_chunk_cnt) { const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); - const uint32_t height_A1 = is_quant ? (n_cols_A1 / 32) * n_k_tiles : n_cols_A1; - dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), + const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); } @@ -3170,18 +3062,18 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_weight_bufs[i % 2], curr_raw, + ctx, vtcm_weight_bufs[i % 4], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); if (i + 2 < n_chunk_cnt) { - const uint32_t height_p2 = is_quant ? (n_cols_p2 / 32) * n_k_tiles : n_cols_p2; - dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_p2 * weight_stride), + const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); } hmx_matmul_job_init(&job_slots[i % 2], (__fp16 *) vtcm_output_bufs[i % 2], - (__fp16 *) vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], + (__fp16 *) vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 4], vtcm_scales, hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS), hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS), k / HTP_MM_HMX_TILE_N_ROWS); hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[i % 2])); @@ -3193,7 +3085,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k float *output_chunk = dst_ptr + (mr * dst_stride + nc_prev); int chunk_dst_cols = dst_cols - (int)nc_prev; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_hvx); } } } @@ -3204,28 +3096,32 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k float *output_chunk = dst_ptr + (mr * dst_stride + nc_last); int chunk_dst_cols = dst_cols - (int)nc_last; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_hvx); } } } } else { hmx_matmul_job_t job; + void *vtcm_raw_bufs[2] = { vtcm_weight_raw[0], dequant_worker_fn ? vtcm_weight_raw[0] : vtcm_scratch0 }; + void *vtcm_weight_bufs[2] = { dequant_worker_fn ? vtcm_scratch0 : vtcm_weight_raw[0], vtcm_scratch0 }; + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); struct activation_transfer_params act_params = { .ctx = ctx, .dst = vtcm_f16_act, - .act_dma_addr = act_dma_addr + mr * act_stride * sizeof(float), + .act_dma_addr = act_dma_addr + (size_t) mr * act_stride * act_elem_size, .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = act_elem_size, }; transfer_activation_chunk_threaded(&act_params); @@ -3242,16 +3138,18 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const size_t weight_stride = src_w->nb[1]; const size_t dst_stride = dst->nb[1] / sizeof(float); const int dst_cols = (int) dst->ne[0]; + const int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); - const uint32_t dma_src_stride = is_quant ? tile_size : weight_stride; + const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; if (n > 0) { const size_t n_cols = hex_smin(n, n_chunk_n_cols); - const uint32_t height = is_quant ? (n_cols / 32) * n_k_tiles : n_cols; - dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } - for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { + for (int i = 0; i < n_chunk_cnt; ++i) { + const size_t nc = (size_t) i * n_chunk_n_cols; const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); @@ -3259,25 +3157,25 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_scratch0, curr_raw, + ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); const size_t nc_next = nc + n_chunk_n_cols; if (nc_next < n) { const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); - const uint32_t height_next = is_quant ? (n_cols_next / 32) * n_k_tiles : n_cols_next; - dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } - hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, vtcm_scratch0, vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); hmx_queue_pop(ctx->hmx_queue); float *output_chunk = dst_ptr + (mr * dst_stride + nc); int chunk_dst_cols = dst_cols - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_hvx); } } } @@ -3287,88 +3185,88 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k return HTP_STATUS_OK; } -static inline dma_addr_t hmx_mm_weight_batch_data(const hmx_mm_f16_f32_batched_params_t *params, +static inline dma_addr_t hmx_mm_weight_batch_data(const hmx_mm_batched_params_t *params, int dst_b2, int dst_b3) { const size_t b2_idx = (params->r2 <= 1) ? (size_t) dst_b2 : (size_t) fastdiv((uint32_t) dst_b2, ¶ms->div_r2); const size_t b3_idx = (params->r3 <= 1) ? (size_t) dst_b3 : (size_t) fastdiv((uint32_t) dst_b3, ¶ms->div_r3); return params->weight + b2_idx * params->src0_nb2 + b3_idx * params->src0_nb3; } -static inline dma_addr_t hmx_mm_act_batch_addr(const hmx_mm_f16_f32_batched_params_t *params, +static inline dma_addr_t hmx_mm_act_batch_addr(const hmx_mm_batched_params_t *params, int dst_b2, int dst_b3) { return params->act_dma_addr + dst_b2 * params->act_nb2 + dst_b3 * params->act_nb3; } -static inline float *hmx_mm_dst_batch_ptr(const hmx_mm_f16_f32_batched_params_t *params, +static inline float *hmx_mm_dst_batch_ptr(const hmx_mm_batched_params_t *params, int dst_b2, int dst_b3) { return (float *) ((uint8_t *) params->dst + (size_t) dst_b2 * params->dst_nb2 + (size_t) dst_b3 * params->dst_nb3); } -static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, - const hmx_mm_f16_f32_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size, - const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) { +static int hmx_mm_batched_simple(struct htp_context *ctx, + const hmx_mm_batched_params_t *params, + int m_chunk, int n_chunk, int pipeline, int n_hvx, int act_hvx, int vtcm_size, + const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div) { int ret = 0; for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { - dma_addr_t cur_src2_addr = params->src2_addr ? (params->src2_addr + - b2 * params->src2_nb2 + - b3 * params->src2_nb3) : 0; + dma_addr_t cur_bias_addr = params->bias_addr ? (params->bias_addr + + b2 * params->bias_nb2 + + b3 * params->bias_nb3) : 0; ret = hmx_mm_2d_f32(ctx, params->weight_dma, hmx_mm_dst_batch_ptr(params, b2, b3), - cur_src2_addr, params->src2_bytes, + cur_bias_addr, params->bias_bytes, hmx_mm_act_batch_addr(params, b2, b3), hmx_mm_weight_batch_data(params, b2, b3), params->m, params->k, params->n, - params->act_stride, params->weight_stride * (int)sizeof(__fp16), - HTP_TYPE_F16, params->k, params->dst_stride, params->src2_stride, params->n, - m_chunk, n_chunk, pipeline, n_threads, act_threads, - act_threads_div, k_div, 0, 0, vtcm_size); + params->act_stride, params->act_elem_size, params->weight_stride, + params->weight_type, params->k, params->dst_stride, params->bias_stride, params->dst_cols, + m_chunk, n_chunk, pipeline, n_hvx, act_hvx, + act_hvx_div, k_div, params->tile_size, params->aligned_tile_size, vtcm_size); } } return ret; } -static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, - const struct fastdiv_values * act_threads_div, - const struct fastdiv_values * k_div, - int vtcm_size) { - if (params->act_stride < params->k || params->weight_stride < params->k || params->dst_stride < params->n) { return -1; } +static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t *params, + int m_chunk, int n_chunk, int pipeline, int n_hvx, int act_hvx, + const struct fastdiv_values * act_hvx_div, + const struct fastdiv_values * k_div, + int vtcm_size) { + const size_t row_stride = htp_tensor_get_row_size(params->weight_type, params->k); + if (row_stride == 0) { return -1; } + if (params->act_stride < params->k || (params->tile_size == 0 && (size_t) params->weight_stride < row_stride) || params->dst_stride < params->dst_cols) { return -1; } if (params->ne02 <= 0 || params->ne03 <= 0 || params->ne12 <= 0 || params->ne13 <= 0) { return -1; } if (params->ne12 % params->ne02 != 0 || params->ne13 % params->ne03 != 0) { return -1; } - if (params->k % 32 != 0 || params->n % 32 != 0) { return -1; } - if (!hex_is_aligned(params->dst, VLEN) || (params->act_dma_addr & (VLEN - 1)) != 0) { return -1; } + // N (the weight row count) does not have to be 32-aligned: + if (params->k % 32 != 0) { return -1; } + if (!hex_is_aligned(params->dst, sizeof(float))) { return -1; } const int group_size = params->r2; const size_t vtcm_budget = ctx->vtcm_size; // Check if the precomputed parameters are grouped or simple. // If simple, or if group_size <= 1, we use simple fallback loop. - // Grouped path is only valid if group_size > 1 and it fits within VTCM budget. - bool run_grouped = (group_size > 1 && (size_t)vtcm_size <= vtcm_budget); + // Grouped path is only valid for flat F16 if group_size > 1 and it fits within VTCM budget. + bool run_grouped = (params->weight_type == HTP_TYPE_F16 && params->tile_size == 0 && group_size > 1 && (size_t)vtcm_size <= vtcm_budget); if (!run_grouped) { - return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); + return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_hvx, act_hvx, vtcm_size, act_hvx_div, k_div); } struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - const size_t vec_dot_size = params->k * sizeof(__fp16); - size_t m_chunk_n_rows = m_chunk; size_t n_chunk_n_cols = n_chunk; - size_t vtcm_used = vtcm_size; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_threads, 0, params->src2_bytes); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_hvx, 0, params->bias_bytes); if (L.total_bytes > vtcm_budget) { FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); + return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_hvx, act_hvx, vtcm_size, act_hvx_div, k_div); } uint8_t * const base = (uint8_t *) ctx->vtcm_base; @@ -3380,10 +3278,10 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ __fp16 *vtcm_scales = VTCM_LAYOUT_PTR(__fp16, base, L.off_scales); float *vtcm_f32_act = VTCM_LAYOUT_PTR(float, base, L.off_act_f32); - const bool has_src2 = (params->src2_bytes > 0 && params->src2_addr != 0); - float *vtcm_src2 = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_src2, has_src2); - if (has_src2) { - dma_queue_push(params->weight_dma, dma_make_data(vtcm_src2, params->src2_addr), hex_align_up(params->src2_bytes, 128), 0, params->src2_bytes, 1); + const bool has_bias = (params->bias_bytes > 0 && params->bias_addr != 0); + float *vtcm_bias = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_bias, has_bias); + if (has_bias) { + dma_queue_push(params->weight_dma, dma_make_data(vtcm_bias, params->bias_addr), hex_align_up(params->bias_bytes, 128), 0, params->bias_bytes, 1); dma_queue_pop(params->weight_dma); } @@ -3395,7 +3293,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ L.total_bytes, vtcm_budget); const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); - const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); + const size_t weight_row_bytes = (size_t) params->weight_stride; htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); @@ -3417,7 +3315,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ // thrashing from HVX loads at large strides. for (int g = 0; g < group_size; ++g) { const dma_addr_t act_dma_addr = hmx_mm_act_batch_addr(params, b2_base + g, b3) + - mr * params->act_stride * sizeof(float); + (size_t) mr * params->act_stride * params->act_elem_size; __fp16 *vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride; struct activation_transfer_params act_params = { .ctx = ctx, @@ -3426,12 +3324,13 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ .n_rows = (int) n_rows, .k_block = params->k, .k_stride = params->act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = params->k, .vtcm_f32_act = vtcm_f32_act, .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = params->act_elem_size, }; transfer_activation_chunk_threaded(&act_params); } @@ -3444,7 +3343,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ } if (n_chunk_n_cols < (size_t) params->n) { const size_t n_cols_second = hex_smin((size_t) params->n - n_chunk_n_cols, n_chunk_n_cols); - dma_queue_push(weight_dma, dma_make_data(vtcm_scratch1, weight_group + params->weight_stride * sizeof(__fp16)), + const dma_addr_t second_weight_chunk = weight_group + n_chunk_n_cols * weight_row_bytes; + dma_queue_push(weight_dma, dma_make_data(vtcm_scratch1, second_weight_chunk), fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_second); } @@ -3455,12 +3355,14 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ { void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; - hmx_interleave_rows_to_tiles(vtcm_weight, (const __fp16 *) curr_raw, n_cols, params->k, params->k, 0, n_cols); + const size_t n_cols_tiled = hex_align_up(n_cols, HTP_MM_HMX_TILE_N_COLS); + + hmx_interleave_rows_to_tiles(vtcm_weight, (const __fp16 *) curr_raw, (uint32_t) n_cols, params->k, params->k, 0, (uint32_t) n_cols_tiled); const size_t nc_next = nc + n_chunk_n_cols * 2; if (nc_next < (size_t) params->n) { const size_t n_cols_next = hex_smin((size_t) params->n - nc_next, n_chunk_n_cols); - const dma_addr_t next_weight_chunk = weight_group + nc_next * params->weight_stride * sizeof(__fp16); + const dma_addr_t next_weight_chunk = weight_group + nc_next * weight_row_bytes; dma_queue_push(weight_dma, dma_make_data(curr_raw, next_weight_chunk), fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_next); @@ -3478,11 +3380,11 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ { float *output = hmx_mm_dst_batch_ptr(params, b2_base + g, b3) + mr * params->dst_stride + nc; - const float *src2_chunk = has_src2 ? (vtcm_src2 + mr * params->src2_stride + nc) : NULL; - int chunk_dst_cols = params->n - (int)nc; + const float *bias_chunk = has_bias ? (vtcm_bias + mr * params->bias_stride + nc) : NULL; + int chunk_dst_cols = params->dst_cols - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output, src2_chunk, vtcm_output, (int) n_rows, (int) n_cols, - params->dst_stride, params->src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output, bias_chunk, vtcm_output, (int) n_rows, (int) n_cols, + params->dst_stride, params->bias_stride, chunk_dst_cols, n_hvx); } } } @@ -3508,20 +3410,20 @@ static void transfer_activation_chunk_gathered_threaded( size_t nb11, size_t nb12, int cne1, - int n_threads, + int n_hvx, int k_valid) { if (n_rows <= 0) return; - int chunks_per_thread = hmx_ceil_div(n_rows, n_threads); - chunks_per_thread = hex_align_up(chunks_per_thread, 2); + int chunks_per_hvx = hmx_ceil_div(n_rows, n_hvx); + chunks_per_hvx = hex_align_up(chunks_per_hvx, 2); - int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread); + int actual_hvx = hmx_ceil_div(n_rows, chunks_per_hvx); activation_transfer_gathered_task_state_t state = { .dst = dst, .src = src, - .n_tasks = actual_threads, + .n_tasks = actual_hvx, .n_tot_chunks = n_rows, - .n_chunks_per_task = chunks_per_thread, + .n_chunks_per_task = chunks_per_hvx, .k_block = k_block, .matrix_rows = matrix_rows, .cur_a = cur_a, @@ -3533,17 +3435,12 @@ static void transfer_activation_chunk_gathered_threaded( .start_row = start_row, .cne1 = cne1, .k_valid = k_valid, - .traces = ctx->trace, }; - worker_callback_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn : + work_queue_func_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn : transfer_activation_chunk_gathered_worker_fn; - if (actual_threads <= 1) { - worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, worker_fn, &state, actual_hvx); } static void transfer_output_chunk_scattered_threaded( @@ -3559,19 +3456,19 @@ static void transfer_output_chunk_scattered_threaded( size_t dst_nb1, size_t dst_nb2, int cne1, - int n_threads) { + int n_hvx) { if (n_rows <= 0) return; - int chunks_per_thread = hmx_ceil_div(n_rows, n_threads); - chunks_per_thread = hex_align_up(chunks_per_thread, 2); + int chunks_per_hvx = hmx_ceil_div(n_rows, n_hvx); + chunks_per_hvx = hex_align_up(chunks_per_hvx, 2); - int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread); + int actual_hvx = hmx_ceil_div(n_rows, chunks_per_hvx); output_transfer_scattered_task_state_t state = { .vtcm_src = vtcm_src, .dst = dst, - .n_tasks = actual_threads, + .n_tasks = actual_hvx, .n_tot_chunks = n_rows, - .n_chunks_per_task = chunks_per_thread, + .n_chunks_per_task = chunks_per_hvx, .n_cols = n_cols, .matrix_rows = matrix_rows, .cur_a = cur_a, @@ -3580,14 +3477,9 @@ static void transfer_output_chunk_scattered_threaded( .dst_nb2 = dst_nb2, .start_row = start_row, .cne1 = cne1, - .traces = ctx->trace, }; - if (actual_threads <= 1) { - transfer_output_chunk_scattered_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_scattered_worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, transfer_output_chunk_scattered_worker_fn, &state, actual_hvx); } static int hmx_mm_id_2d_f32(struct htp_context *ctx, @@ -3602,62 +3494,46 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, size_t dst_nb1, size_t dst_nb2, int weight_stride, int weight_type, + int tile_size, + int aligned_tile_size, const struct mmid_row_mapping *matrix_rows, int cur_a, int mapping_stride, int m_start, int m_end, - int n_threads) { + int n_hvx) { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); const int cne1 = m; - const int m_padded = hex_align_up(m, 32); + const int m_core = m_end - m_start; + const int m_core_padded = hex_align_up(m_core > 0 ? m_core : 1, 32); if (k % 32 != 0 || n % 32 != 0) { return -1; } if (!hex_is_aligned(dst, VLEN) || !hex_is_aligned(activation, VLEN)) { return -1; } - size_t row_stride = htp_mm_get_tiled_row_stride(weight_type, k); + size_t row_stride = htp_tensor_get_row_size(weight_type, k); if (row_stride == 0) { return -1; } - worker_callback_t dequant_worker_fn = NULL; - switch (weight_type) { - case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break; - case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; - case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break; - case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break; - case HTP_TYPE_Q5_K: dequant_worker_fn = dequantize_tiled_worker_loop_q5_k; break; - case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break; - case HTP_TYPE_Q3_K: dequant_worker_fn = dequantize_tiled_worker_loop_q3_k; break; - case HTP_TYPE_Q2_K: dequant_worker_fn = dequantize_tiled_worker_loop_q2_k; break; - case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break; - case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break; - default: - return -1; - } - const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32); - const size_t vec_dot_size = k * sizeof(__fp16); const size_t vtcm_budget = ctx->vtcm_size; size_t vtcm_used = 0; - int tile_size = htp_mm_get_weight_tile_size(weight_type); - int aligned_tile_size = htp_mm_get_weight_aligned_tile_size(weight_type); + const bool is_tiled = (tile_size > 0); + tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled); - const uint32_t dma_dst_stride = is_quant ? aligned_tile_size : row_stride; - const uint32_t dma_src_stride = is_quant ? tile_size : weight_stride; - const uint32_t dma_width_bytes = is_quant ? tile_size : row_stride; + const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride; + const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; + const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; - const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; - const size_t weight_row_stride = is_quant ? qweight_row_stride : row_stride; + const size_t qweight_row_stride = is_tiled ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0; + const size_t weight_row_stride = is_tiled ? qweight_row_stride : row_stride; size_t size_per_n = 0, size_per_m = 0, size_per_mn = 0; htp_mm_hmx_get_2d_chunk_costs(weight_type, k, /*pipeline=*/false, aligned_tile_size, @@ -3666,10 +3542,10 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const size_t overhead = htp_mm_hmx_get_2d_overhead(/*pipeline=*/false, /*is_matmul_id=*/true); size_t m_chunk_n_rows = 0, n_chunk_n_cols = 0; if (htp_mm_hmx_compute_chunks(vtcm_budget, overhead, size_per_n, size_per_m, size_per_mn, - m_padded, n, + m_core_padded, n, /*m_block_cost=*/(size_t) n * HTP_MM_HMX_COST_W_DEQUANT, - /*n_block_cost=*/(size_t) m_padded * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used)) { - FARF(ERROR, "hmx-mm-id-2d: VTCM too small : m %d k %d n %d budget %zu", m_padded, k, n, vtcm_budget); + /*n_block_cost=*/(size_t) m_core_padded * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used)) { + FARF(ERROR, "hmx-mm-id-2d: VTCM too small : m %d k %d n %d budget %zu", m_core_padded, k, n, vtcm_budget); return -1; } @@ -3697,6 +3573,9 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); hmx_matmul_job_t job; + void *vtcm_raw_bufs[2] = { vtcm_weight, dequant_worker_fn ? vtcm_weight : vtcm_scratch0 }; + void *vtcm_weight_bufs[2] = { dequant_worker_fn ? vtcm_scratch0 : vtcm_weight, vtcm_scratch0 }; + const int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); for (size_t mr = (size_t) m_start; mr < (size_t) m_end; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin((size_t) m_end - mr, m_chunk_n_rows); @@ -3704,17 +3583,18 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, transfer_activation_chunk_gathered_threaded( ctx, vtcm_f16_act, activation, (int) mr, (int) n_rows, k, - matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_threads, k_valid); + matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_hvx, k_valid); // A0: Pre-fetch the first weight chunk (nc = 0) if (n > 0) { const size_t n_cols = hex_smin((size_t) n, n_chunk_n_cols); - const uint32_t height = is_quant ? (n_cols / 32) * n_k_tiles : n_cols; - dma_queue_push(weight_dma, dma_make_data(vtcm_weight, weight), + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } - for (size_t nc = 0; nc < (size_t) n; nc += n_chunk_n_cols) { + for (int i = 0; i < n_chunk_cnt; ++i) { + const size_t nc = (size_t) i * n_chunk_n_cols; const size_t n_cols = hex_smin((size_t) n - nc, n_chunk_n_cols); const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); @@ -3723,29 +3603,29 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, // B: Weight Dequantize (Threaded) dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_scratch0, curr_raw, + ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx ); // Start weight DMA for the next chunk early const size_t nc_next = nc + n_chunk_n_cols; if (nc_next < (size_t) n) { const size_t n_cols_next = hex_smin((size_t) n - nc_next, n_chunk_n_cols); - const uint32_t height_next = is_quant ? (n_cols_next / 32) * n_k_tiles : n_cols_next; - dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_next * weight_stride), + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } // C: HMX Compute (Queue-based) - hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, vtcm_scratch0, vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); hmx_queue_pop(ctx->hmx_queue); // D: Output Store transfer_output_chunk_scattered_threaded( ctx, dst + nc, vtcm_output, (int) mr, (int) n_rows, (int) n_cols, - matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_threads); + matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_hvx); } } @@ -3759,14 +3639,18 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k int k = (int) src0->ne[0]; int n = (int) src0->ne[1]; - const int m_total = (int) src1->ne[1]; - const int act_stride = (int)(src1->nb[1] / sizeof(float)); - const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16)); + const int m_total = (int) act->ne[1]; + const uint32_t act_elem_size = (act->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + const int act_stride = (int) (act->nb[1] / act_elem_size); int m_start = 0; int m_rows = m_total; if (octx->ctx->mdev.count > 1) { - const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)) && + htp_tensor_can_row_partition(act, act_elem_size); + if (src2 && src2->ne[1] > 1 && !htp_tensor_can_row_partition(src2, sizeof(float))) { + can_split = false; + } const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_total, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); m_start = (int) range.start; m_rows = (int) range.count; @@ -3784,65 +3668,71 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k if (src2) { src2_stride = (src2->ne[1] == 1) ? 0 : (uint32_t) (src2->nb[1] / sizeof(float)); src2_addr = src2->data + m_start * src2_stride * sizeof(float); - src2_bytes = (size_t) kparams->vtcm_src2_size; + src2_bytes = (size_t) kparams->vtcm_bias_size; src2_nb2 = (src2->ne[2] == 1) ? 0 : src2->nb[2]; src2_nb3 = (src2->ne[3] == 1) ? 0 : src2->nb[3]; } const int dst_stride = (int)(dst->nb[1] / sizeof(float)); float * dst_ptr = (float *) dst->data + m_start * dst_stride; - const dma_addr_t act_addr = src1->data + m_start * act_stride * sizeof(float); + // byte offset, act_stride is in activation elements (F16 or F32) + const dma_addr_t act_addr = act->data + (size_t) m_start * act_stride * act_elem_size; int ret = -1; - const int n_threads = kparams->n_threads; - if (kparams->kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) { - hmx_mm_f16_f32_batched_params_t batch_params = { - .dst = dst_ptr, - .src2_addr = src2_addr, - .src2_bytes = src2_bytes, - .act_dma_addr = act_addr, - .weight = src0->data, - .weight_dma = octx->ctx->dma[0], - .m = m_rows, - .k = k, - .n = n, - .act_stride = act_stride, - .weight_stride = wgt_stride, - .dst_stride = dst_stride, - .src2_stride = src2_stride, - .ne02 = ne02, - .ne03 = ne03, - .ne12 = ne12, - .ne13 = ne13, - .src0_nb2 = src0->nb[2], - .src0_nb3 = src0->nb[3], - .act_nb2 = src1->nb[2], - .act_nb3 = src1->nb[3], - .dst_nb2 = dst->nb[2], - .dst_nb3 = dst->nb[3], - .src2_nb2 = src2_nb2, - .src2_nb3 = src2_nb3, - .r2 = (ne02 > 0) ? (ne12 / ne02) : 1, - .r3 = (ne03 > 0) ? (ne13 / ne03) : 1, - .div_r2 = kparams->div_r2, - .div_r3 = kparams->div_r3, + const int n_hvx = kparams->n_hvx; + if (kparams->kernel_type == HTP_MM_KERNEL_HMX_BATCHED) { + hmx_mm_batched_params_t batch_params = { + .dst = dst_ptr, + .bias_addr = src2_addr, + .bias_bytes = src2_bytes, + .act_dma_addr = act_addr, + .weight = src0->data, + .weight_dma = octx->ctx->dma[0], + .m = m_rows, + .k = k, + .n = n, + .act_stride = act_stride, + .act_elem_size = act_elem_size, + .weight_stride = (int) src0->nb[1], + .weight_type = (int) src0->type, + .dst_stride = dst_stride, + .dst_cols = (int) dst->ne[0], + .bias_stride = src2_stride, + .tile_size = kparams->tile_size, + .aligned_tile_size = kparams->aligned_tile_size, + .ne02 = ne02, + .ne03 = ne03, + .ne12 = ne12, + .ne13 = ne13, + .src0_nb2 = src0->nb[2], + .src0_nb3 = src0->nb[3], + .act_nb2 = act->nb[2], + .act_nb3 = act->nb[3], + .dst_nb2 = dst->nb[2], + .dst_nb3 = dst->nb[3], + .bias_nb2 = src2_nb2, + .bias_nb3 = src2_nb3, + .r2 = (ne02 > 0) ? (ne12 / ne02) : 1, + .r3 = (ne03 > 0) ? (ne13 / ne03) : 1, + .div_r2 = kparams->div_r2, + .div_r3 = kparams->div_r3, }; - ret = hmx_mm_f16_f32_batched(octx->ctx, &batch_params, - kparams->m_chunk, kparams->n_chunk, - kparams->pipeline, n_threads, - kparams->n_act_threads, - &kparams->div_n_act_threads, - &kparams->div_ne00_padded, - kparams->vtcm_size); + ret = hmx_mm_batched(octx->ctx, &batch_params, + kparams->m_chunk, kparams->n_chunk, + kparams->pipeline, n_hvx, + kparams->n_act_hvx, + &kparams->div_n_act_hvx, + &kparams->div_ne00_padded, + kparams->vtcm_size); } else { ret = hmx_mm_2d_f32( octx->ctx, octx->ctx->dma[0], dst_ptr, src2_addr, src2_bytes, act_addr, src0->data, - m_rows, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0], + m_rows, k, n, act_stride, act_elem_size, (int) src0->nb[1], (int) src0->type, (int) act->ne[0], dst_stride, src2_stride, (int)dst->ne[0], - kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, - kparams->n_act_threads, - &kparams->div_n_act_threads, + kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_hvx, + kparams->n_act_hvx, + &kparams->div_n_act_hvx, &kparams->div_ne00_padded, kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size ); @@ -3855,7 +3745,7 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k return HTP_STATUS_OK; } -int op_matmul(struct htp_ops_context * octx) { +static int op_matmul_impl(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const int status = htp_mm_init_context(octx, kparams); @@ -3870,6 +3760,10 @@ int op_matmul(struct htp_ops_context * octx) { return hvx_mm_matmul(octx); } +int op_matmul(struct htp_ops_context * octx) { + return op_matmul_impl(octx); +} + static int hmx_mm_op_matmul_id( struct htp_ops_context * octx, struct htp_mm_context * mmctx @@ -3881,21 +3775,40 @@ static int hmx_mm_op_matmul_id( const int n_ids = octx->src[2]->ne[0]; const int n_as = ne02; - for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { + const bool mdev_split = (octx->ctx->mdev.count > 1) && htp_tensor_can_row_partition(dst, sizeof(float)); + if (octx->ctx->mdev.count > 1 && !mdev_split && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + uint32_t n_active = 0; + if (mdev_split) { + for (uint32_t a = 0; a < (uint32_t) n_as; ++a) { + if (matrix_row_counts[a] > 0) n_active++; + } + } + const bool expert_split = mdev_split && (n_active >= octx->ctx->mdev.count); + + uint32_t target_dev = 0; + for (uint32_t cur_a = 0; cur_a < (uint32_t) n_as; ++cur_a) { const int32_t cne1 = matrix_row_counts[cur_a]; if (cne1 == 0) continue; const int m_padded = hex_align_up(cne1, 32); int m_start = 0, m_end = m_padded; - if (octx->ctx->mdev.count > 1) { - const bool can_split = htp_tensor_mdev_data_aligned(dst) && (uint32_t) cne1 >= octx->ctx->mdev.count; + if (expert_split) { + const bool my_expert = (target_dev == octx->ctx->mdev.idx); + if (++target_dev == octx->ctx->mdev.count) { + target_dev = 0; + } + if (!my_expert) continue; + } else if (mdev_split) { + const bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count; const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); m_start = (int) range.start; m_end = (int) (range.start + range.count); } if (m_start >= m_end) continue; - int ret = hmx_mm_id_2d_f32(octx->ctx, octx->ctx->dma[0], (float*) dst->data, (float*) src1->data, + int ret = hmx_mm_id_2d_f32(octx->ctx, octx->ctx->dma[0], (float*) dst->data, (float*) act->data, src0->data + cur_a * nb02, cne1, ne00, ne01, ne10, @@ -3903,8 +3816,9 @@ static int hmx_mm_op_matmul_id( nb11, nb12, nb1, nb2, (int) src0->nb[1], (int) src0->type, + kparams->tile_size, kparams->aligned_tile_size, matrix_rows, cur_a, mmctx->mapping_stride, - m_start, m_end, (int) octx->n_threads); + m_start, m_end, (int) octx->n_hvx); if (ret != 0) { FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret); return HTP_STATUS_NO_SUPPORT; @@ -3930,15 +3844,15 @@ static int hvx_mm_matmul_id( const struct htp_tensor * restrict ids = octx->src[2]; const size_t src0_row_size = nb01; - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (ne10 + qk - 1) / qk; const uint32_t total_nb = act_nrows * nb; work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = htp_mm_act_quant_block_func(src0->type); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); + quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; @@ -3948,24 +3862,24 @@ static int hvx_mm_matmul_id( mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); - quant_task_func = htp_mm_act_quant_row_func(src0->type); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); + quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } - size_t src1_row_size = htp_mm_weight_has_offset(src0->type) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, ne10); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_threads, - 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_hvx, + 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false); const size_t vtcm_size = L.total_bytes; - FARF(HIGH, "matmul-id-%s : src0-spad-size %zu src1-spad-size %zu src2-spad-size 0 dst-spad-size %zu (%zu)\n", mmctx->type, - L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); + FARF(HIGH, "matmul-id-%s : src0-spad-size %zu act-spad-size %zu bias-spad-size 0 dst-spad-size %zu (%zu)\n", mmctx->type, + L.src0_bytes, L.act_bytes, L.dst_bytes, vtcm_size); FARF(HIGH, "matmul-id-%s : %ux%ux%ux%u * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\n", mmctx->type, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], act->ne[0], act->ne[1], act->ne[2], act->ne[3], ids->ne[0], ids->ne[1], ids->ne[2], ids->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], src0->data, - src1->data, dst->data); + act->data, dst->data); // Make sure the reserved vtcm size is sufficient if (octx->ctx->vtcm_size < vtcm_size) { @@ -3974,38 +3888,32 @@ static int hvx_mm_matmul_id( } uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + mmctx->vtcm_act = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act); mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); - mmctx->vtcm_src2 = NULL; + mmctx->vtcm_bias = NULL; mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); mmctx->vtcm_act_raw = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act_raw); - octx->src1_spad.src = NULL; - octx->src0_spad.src = NULL; - octx->src2_spad.src = NULL; - octx->dst_spad.src = NULL; - mmctx->vtcm_src0_stride = src0_row_size_padded; - mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + mmctx->vtcm_act_stride = act_row_size; + mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10); - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_src2_size_per_thread = 0; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); mmctx->cur_m_start = 0; mmctx->cur_m_rows = act_nrows; htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - hvx_mm_transfer_src1_dma(octx, kparams, src1, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, hvx_mmid_task_func, mmctx, octx->n_hvx); return HTP_STATUS_OK; } @@ -4022,18 +3930,37 @@ static int hmx_mm_op_matmul_id_nx( const struct htp_tensor * restrict act = octx->src[n_weights]; const int n_as = src0->ne[2]; + bool mdev_split = (octx->ctx->mdev.count > 1); + for (uint32_t p = 0; p < n_weights && mdev_split; ++p) { + const struct htp_tensor * restrict dst = octx->dsts[p]; + mdev_split = htp_tensor_can_row_partition(dst, sizeof(float)); + } + if (octx->ctx->mdev.count > 1 && !mdev_split && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + uint32_t n_active = 0; + if (mdev_split) { + for (uint32_t a = 0; a < (uint32_t) n_as; ++a) { + if (matrix_row_counts[a] > 0) n_active++; + } + } + const bool expert_split = mdev_split && (n_active >= octx->ctx->mdev.count); + + uint32_t target_dev = 0; for (uint32_t cur_a = 0; cur_a < (uint32_t) n_as; ++cur_a) { const int32_t cne1 = matrix_row_counts[cur_a]; if (cne1 == 0) continue; const int m_padded = hex_align_up(cne1, 32); int m_start = 0, m_end = m_padded; - if (octx->ctx->mdev.count > 1) { - bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count; - for (uint32_t p = 0; p < n_weights && can_split; ++p) { - const struct htp_tensor * restrict dst = octx->dsts[p]; - can_split = !dst || htp_tensor_mdev_data_aligned(dst); + if (expert_split) { + const bool my_expert = (target_dev == octx->ctx->mdev.idx); + if (++target_dev == octx->ctx->mdev.count) { + target_dev = 0; } + if (!my_expert) continue; + } else if (mdev_split) { + const bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count; const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); m_start = (int) range.start; m_end = (int) (range.start + range.count); @@ -4053,8 +3980,9 @@ static int hmx_mm_op_matmul_id_nx( act->nb[1], act->nb[2], dst->nb[1], dst->nb[2], (int) src_w->nb[1], (int) src_w->type, + kparams->tile_size, kparams->aligned_tile_size, matrix_rows, cur_a, mmctx->mapping_stride, - m_start, m_end, (int) octx->n_threads); + m_start, m_end, (int) octx->n_hvx); if (ret != 0) { FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret); return HTP_STATUS_NO_SUPPORT; @@ -4083,15 +4011,15 @@ static int hvx_mm_matmul_id_nx( const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; const size_t src0_row_size = src0->nb[1]; - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (act->ne[0] + qk - 1) / qk; const uint32_t total_nb = act_nrows * nb; work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = htp_mm_act_quant_block_func(src0->type); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); + quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; @@ -4101,14 +4029,14 @@ static int hvx_mm_matmul_id_nx( mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); - quant_task_func = htp_mm_act_quant_row_func(src0->type); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); + quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } - size_t src1_row_size = htp_mm_weight_has_offset(src0->type) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]); + size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads, - 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_hvx, + 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false); const size_t vtcm_size = L.total_bytes; @@ -4120,46 +4048,40 @@ static int hvx_mm_matmul_id_nx( uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); - mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + mmctx->vtcm_act = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act); mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); mmctx->vtcm_act_raw = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act_raw); - octx->src0_spad.src = NULL; - octx->src1_spad.src = NULL; - octx->src2_spad.src = NULL; - octx->src3_spad.src = NULL; - octx->dst_spad.src = NULL; - mmctx->vtcm_src0_stride = 0; - mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_act_raw_stride = hex_round_up(act->ne[0] * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + mmctx->vtcm_act_stride = act_row_size; + mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]); - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); mmctx->cur_m_start = 0; mmctx->cur_m_rows = act_nrows; - FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, src1 %d:%d:%d nrows %u, vtcm %zu/%zu, threads %d\n", + FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, act %d:%d:%d nrows %u, vtcm %zu/%zu, nhvx %d\n", src0->ne[0], src0->ne[1], src0->ne[2], mmctx->type, src0->ne[1], act->ne[0], act->ne[1], act->ne[2], act_nrows, - L.total_bytes, octx->ctx->vtcm_size, octx->n_threads); + L.total_bytes, octx->ctx->vtcm_size, octx->n_hvx); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - hvx_mm_transfer_src1_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, hvx_mmid_task_func, mmctx, octx->n_hvx); return HTP_STATUS_OK; } -static inline void scan_expert_ids_n( +static __attribute__((noinline)) uint32_t scan_expert_ids_n( const struct htp_tensor * ids, const uint32_t n_ids, uint32_t n_as, @@ -4169,15 +4091,17 @@ static inline void scan_expert_ids_n( ) { const size_t ids_nb1 = ids->nb[1]; const uint8_t * ids_data = (const uint8_t *) ids->data; + uint32_t n_skipped = 0; for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1); for (uint32_t id = 0; id < n_ids; ++id) { const int32_t i02 = row_ptr[id]; if (i02 < 0) { + n_skipped++; continue; } - assert(i02 < n_as); + assert(i02 >= 0 && i02 < (int32_t) n_as); if (matrix_rows) { matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 }; @@ -4185,9 +4109,10 @@ static inline void scan_expert_ids_n( counts[i02] += 1; } } + return n_skipped; } -static inline void scan_expert_ids( +static __attribute__((noinline)) uint32_t scan_expert_ids( const struct htp_tensor * ids, uint32_t n_ids, uint32_t n_as, @@ -4199,23 +4124,25 @@ static inline void scan_expert_ids( if (ids_nb0 == 4) { switch (n_ids) { - case 8: scan_expert_ids_n(ids, 8, n_as, counts, matrix_rows, mapping_stride); break; - case 4: scan_expert_ids_n(ids, 4, n_as, counts, matrix_rows, mapping_stride); break; - case 2: scan_expert_ids_n(ids, 2, n_as, counts, matrix_rows, mapping_stride); break; - default: scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride); break; + case 8: return scan_expert_ids_n(ids, 8, n_as, counts, matrix_rows, mapping_stride); + case 4: return scan_expert_ids_n(ids, 4, n_as, counts, matrix_rows, mapping_stride); + case 2: return scan_expert_ids_n(ids, 2, n_as, counts, matrix_rows, mapping_stride); + default: return scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride); } } else { // Strided fallback const size_t ids_nb1 = ids->nb[1]; const uint8_t * ids_data = (const uint8_t *) ids->data; + uint32_t n_skipped = 0; for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1); for (uint32_t id = 0; id < n_ids; ++id) { const int32_t i02 = *(const int32_t *) ((const uint8_t *) row_ptr + id * ids_nb0); if (i02 < 0) { + n_skipped++; continue; } - assert(i02 < n_as); + assert(i02 >= 0 && i02 < (int32_t) n_as); if (matrix_rows) { matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 }; @@ -4223,57 +4150,92 @@ static inline void scan_expert_ids( counts[i02] += 1; } } + return n_skipped; } } -int op_matmul_id(struct htp_ops_context * octx) { - htp_matmul_tensors_preamble; - - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - struct htp_mm_context mmctx_struct = {0}; - struct htp_mm_context * mmctx = &mmctx_struct; - - const int status = htp_mm_init_context(octx, kparams); - if (status != HTP_STATUS_OK) { - return status; +static __attribute__((noinline)) uint32_t count_skipped_ids_single(const struct htp_tensor * ids, uint32_t n_ids) { + const uint8_t * ids_data = (const uint8_t *) ids->data; + const size_t ids_nb0 = ids->nb[0]; + uint32_t n_skipped = 0; + for (uint32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = *(const int32_t *) (ids_data + id * ids_nb0); + if (i02 < 0) { + n_skipped++; + } } + return n_skipped; +} - struct htp_thread_trace * tr = &octx->ctx->trace[0]; - htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - - mmctx->octx = octx; - mmctx->act = src1; +// Zero rows for skipped slots (ids < 0). Matmuls only write rows that an expert owns. +static void zero_skipped_dst_rows( + struct htp_ops_context * octx, + const struct htp_tensor * ids, + uint32_t n_ids, + const struct htp_tensor * dst +) { + const uint32_t elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + uint32_t col_start = 0; + uint32_t col_count = dst->ne[0]; - const struct htp_tensor * restrict ids = octx->src[2]; - if (htp_tensor_is_extended(ids) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { - return HTP_STATUS_NO_SUPPORT; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, elem_size); + const uint32_t chunk_elems = (dst->type == HTP_TYPE_F16) ? 64 : 32; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + dst->ne[0], can_split ? chunk_elems : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + col_start = range.start; + col_count = range.count; } - const size_t src0_row_size = nb01; - const size_t dst_row_size = nb1; - - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); + if (col_count == 0) { + return; + } - const uint32_t src0_nrows = ne01; // per expert - const uint32_t src1_nrows = ne11 * ne12 * ne13; + const uint8_t * ids_data = (const uint8_t *) ids->data; + const size_t ids_nb0 = ids->nb[0]; + const size_t ids_nb1 = ids->nb[1]; + const size_t dst_nb1 = dst->nb[1]; + const size_t dst_nb2 = dst->nb[2]; + const uint32_t col_offset_bytes = col_start * elem_size; - // row groups - const int n_ids = ids->ne[0]; // n_expert_used - const int n_as = ne02; // n_expert + for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { + const uint8_t * ids_row = ids_data + iid1 * ids_nb1; + uint8_t * dst_row_group = (uint8_t *) dst->data + iid1 * dst_nb2 + col_offset_bytes; + for (uint32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = *(const int32_t *) (ids_row + id * ids_nb0); + if (i02 < 0) { + hvx_splat_u(dst_row_group + id * dst_nb1, Q6_V_vzero(), col_count, elem_size); + } + } + } +} +static int htp_mmid_prepare_mapping( + struct htp_ops_context * octx, + const struct htp_tensor * ids, + uint32_t n_ids, + uint32_t n_as, + uint32_t act_nrows, + uint8_t ** out_mapping_buf, + uint32_t ** out_matrix_row_counts, + struct mmid_row_mapping ** out_matrix_rows, + uint32_t * out_mapping_stride, + uint32_t * out_n_skipped +) { uint8_t * mapping_buf = octx->ctx->ddr_spad_base; uint32_t mapping_stride = 1; uint32_t * matrix_row_counts = (uint32_t *) mapping_buf; struct mmid_row_mapping * matrix_rows = NULL; + uint32_t n_skipped = 0; - if (src1_nrows > 1) { + if (act_nrows > 1) { const size_t matrix_row_counts_size = n_as * sizeof(uint32_t); assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size); hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]); - memset(matrix_row_counts, 0, matrix_row_counts_size); - scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0); + hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t)); + n_skipped = scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0); uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as); mapping_stride = max_count > 0 ? max_count : 1; @@ -4291,8 +4253,71 @@ int op_matmul_id(struct htp_ops_context * octx) { matrix_row_counts = (uint32_t *) mapping_buf; matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size); - memset(matrix_row_counts, 0, n_as * sizeof(uint32_t)); + hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t)); scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride); + } else { + n_skipped = count_skipped_ids_single(ids, n_ids); + } + + *out_mapping_buf = mapping_buf; + *out_matrix_row_counts = matrix_row_counts; + *out_matrix_rows = matrix_rows; + *out_mapping_stride = mapping_stride; + *out_n_skipped = n_skipped; + + return HTP_STATUS_OK; +} + +int op_matmul_id(struct htp_ops_context * octx) { + htp_matmul_tensors_preamble; + + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + + mmctx->octx = octx; + mmctx->act = act; + + const struct htp_tensor * restrict ids = octx->src[2]; + if (htp_tensor_is_extended(ids) || htp_tensor_is_extended(act) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const size_t src0_row_size = nb01; + const size_t dst_row_size = nb1; + + const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); + + const uint32_t src0_nrows = ne01; // per expert + const uint32_t act_nrows = ne11 * ne12 * ne13; + + // row groups + const int n_ids = ids->ne[0]; // n_expert_used + const int n_as = ne02; // n_expert + + uint8_t * mapping_buf = NULL; + uint32_t * matrix_row_counts = NULL; + struct mmid_row_mapping * matrix_rows = NULL; + uint32_t mapping_stride = 1; + uint32_t n_skipped = 0; + + int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows, + &mapping_buf, &matrix_row_counts, &matrix_rows, + &mapping_stride, &n_skipped); + if (map_status != HTP_STATUS_OK) { + return map_status; + } + + if (n_skipped > 0) { + zero_skipped_dst_rows(octx, ids, n_ids, dst); } mmctx->matrix_row_counts = matrix_row_counts; @@ -4300,9 +4325,9 @@ int op_matmul_id(struct htp_ops_context * octx) { mmctx->mapping_stride = mapping_stride; mmctx->mm_div_ne11 = kparams->div_ne1; mmctx->src0_row_size_padded = src0_row_size_padded; - mmctx->act_nrows = src1_nrows; + mmctx->act_nrows = act_nrows; mmctx->cur_m_start = 0; - mmctx->cur_m_rows = src1_nrows; + mmctx->cur_m_rows = act_nrows; htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); @@ -4330,11 +4355,11 @@ int op_matmul_id(struct htp_ops_context * octx) { mmctx->src0_row_start = src0_row_start; mmctx->src0_row_end = src0_row_end; - mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + mmctx->src0_nrows_per_hvx = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); + mmctx->src0_nrows_per_hvx = hex_round_up(mmctx->src0_nrows_per_hvx, 32); - if (hvx_mm_init_vec_dot(mmctx, src0->type) == 0) { - s = hvx_mm_matmul_id(octx, mmctx, src1_nrows > 1 ? hvx_mm_id : hvx_mv_id); + if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) == 0) { + s = hvx_mm_matmul_id(octx, mmctx, act_nrows > 1 ? hvx_mm_id_tiled : hvx_mv_id_tiled); } else { s = HTP_STATUS_NO_SUPPORT; } @@ -4369,7 +4394,7 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } for (uint32_t p = 0; p < n_weights; p++) { - if (octx->dsts[p] && htp_tensor_is_extended(octx->dsts[p])) { + if (htp_tensor_is_extended(octx->dsts[p])) { return HTP_STATUS_NO_SUPPORT; } } @@ -4384,38 +4409,26 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { const int n_ids = ids->ne[0]; const int n_as = src0->ne[2]; - uint8_t * mapping_buf = octx->ctx->ddr_spad_base; - uint32_t mapping_stride = 1; - uint32_t * matrix_row_counts = (uint32_t *) mapping_buf; + uint8_t * mapping_buf = NULL; + uint32_t * matrix_row_counts = NULL; struct mmid_row_mapping * matrix_rows = NULL; + uint32_t mapping_stride = 1; + uint32_t n_skipped = 0; - if (act_nrows > 1) { - const size_t matrix_row_counts_size = n_as * sizeof(uint32_t); - assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size); - - hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]); - - memset(matrix_row_counts, 0, matrix_row_counts_size); - scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0); - - uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as); - mapping_stride = max_count > 0 ? max_count : 1; - - size_t matrix_row_map_size = n_as * mapping_stride * sizeof(struct mmid_row_mapping); - const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size; + int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows, + &mapping_buf, &matrix_row_counts, &matrix_rows, + &mapping_stride, &n_skipped); + if (map_status != HTP_STATUS_OK) { + return map_status; + } - if (total_map_size > octx->ctx->ddr_spad_size) { - mapping_buf = memalign(128, total_map_size); - if (!mapping_buf) { - return HTP_STATUS_INTERNAL_ERR; + if (n_skipped > 0) { + for (uint32_t p = 0; p < n_weights; ++p) { + const struct htp_tensor * restrict d = octx->dsts[p]; + if (d) { + zero_skipped_dst_rows(octx, ids, n_ids, d); } } - - matrix_row_counts = (uint32_t *) mapping_buf; - matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size); - - memset(matrix_row_counts, 0, n_as * sizeof(uint32_t)); - scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride); } mmctx->matrix_row_counts = matrix_row_counts; @@ -4433,8 +4446,8 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { if (kparams->n_hmx) { s = hmx_mm_op_matmul_id_nx(octx, mmctx); } else { - if (hvx_mm_init_vec_dot(mmctx, src0->type) == 0) { - s = hvx_mm_matmul_id_nx(octx, mmctx, act_nrows > 1 ? hvx_mm_id_nx : hvx_mv_id_nx); + if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) == 0) { + s = hvx_mm_matmul_id_nx(octx, mmctx, act_nrows > 1 ? hvx_mm_id_nx_tiled : hvx_mv_id_nx_tiled); } else { s = HTP_STATUS_NO_SUPPORT; } @@ -4446,7 +4459,7 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { return s; } -int op_matmul_nx(struct htp_ops_context * octx) { +static int op_matmul_nx_impl(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const int status = htp_mm_init_context(octx, kparams); @@ -4466,12 +4479,6 @@ int op_matmul_nx(struct htp_ops_context * octx) { const struct htp_tensor * restrict src0 = octx->src[0]; // first weight const struct htp_tensor * restrict act = octx->src[n_weights]; // activation x - bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 || - src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL || - src0->type == HTP_TYPE_MXFP4 || src0->type == HTP_TYPE_Q4_K || - src0->type == HTP_TYPE_Q5_K || src0->type == HTP_TYPE_Q3_K || - src0->type == HTP_TYPE_Q2_K); - struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; @@ -4483,21 +4490,20 @@ int op_matmul_nx(struct htp_ops_context * octx) { mmctx->cur_m_rows = act_nrows; const size_t src0_row_size = src0->nb[1]; - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - if (hvx_mm_init_vec_dot(mmctx, src0->type) != 0) { + if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) != 0) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t qk = QK_Q8_0_TILED; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; const uint32_t nb = (act->ne[0] + qk - 1) / qk; const uint32_t total_nb = act_nrows * nb; - worker_callback_t quant_task_func; + work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = htp_mm_act_quant_block_func(src0->type); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); + quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; @@ -4507,17 +4513,15 @@ int op_matmul_nx(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); - quant_task_func = htp_mm_act_quant_row_func(src0->type); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); + quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } - const size_t src1_row_size = htp_mm_weight_has_offset(src0->type) - ? htp_mm_q8_1_tiled_row_size(act->ne[0]) - : htp_mm_q8_0_tiled_row_size(act->ne[0]); + const size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads, - 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true); + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_hvx, + 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, false, true); const size_t vtcm_size = L.total_bytes; @@ -4529,55 +4533,31 @@ int op_matmul_nx(struct htp_ops_context * octx) { uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); - mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + mmctx->vtcm_act = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act); mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); mmctx->vtcm_act_raw = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act_raw); - octx->src0_spad.src = NULL; - octx->src1_spad.src = NULL; - octx->src2_spad.src = NULL; - octx->src3_spad.src = NULL; - octx->dst_spad.src = NULL; - - mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded; - mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_act_raw_stride = hex_round_up(act->ne[0] * sizeof(float), QK_Q8_0_TILED * sizeof(float)); - - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); - - // Run fused matmul - const uint32_t n_matmul_jobs = octx->n_threads; - worker_callback_t matmul_job_func; - if (is_repacked) { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break; - case HTP_TYPE_Q5_K: matmul_job_func = hvx_mm_nx_2d_repacked_q5_k; break; - case HTP_TYPE_Q3_K: matmul_job_func = hvx_mm_nx_2d_repacked_q3_k; break; - case HTP_TYPE_Q2_K: matmul_job_func = hvx_mm_nx_2d_repacked_q2_k; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4; break; - default: - htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - return HTP_STATUS_NO_SUPPORT; - } - } else { - matmul_job_func = hvx_mm_nx_2d; - } + mmctx->vtcm_src0_stride = 0; + mmctx->vtcm_act_stride = act_row_size; + mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]); + + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - hvx_mm_transfer_src1_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, n_matmul_jobs); + work_queue_run(octx->ctx->hvx_queue, hvx_mm_nx_2d_tiled, mmctx, octx->n_hvx); return HTP_STATUS_OK; } + +int op_matmul_nx(struct htp_ops_context * octx) { + return op_matmul_nx_impl(octx); +} diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 386cb3049..7a85624cf 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -4,6 +4,7 @@ #include <stdint.h> #include <stddef.h> #include "htp-ops.h" +#include "htp-tensor.h" #include "hex-fastdiv.h" #include "hex-common.h" #include "htp-vtcm.h" @@ -19,45 +20,15 @@ extern "C" { #define HTP_MM_HMX_TILE_N_ELMS 1024 #define HTP_MM_HMX_MIN_NROWS 4 -// --- Weight Repacked Tile Sizes --- -#define HTP_MM_WEIGHT_TILE_SIZE_Q4_0 576 -#define HTP_MM_WEIGHT_TILE_SIZE_Q4_1 640 -#define HTP_MM_WEIGHT_TILE_SIZE_Q8_0 1088 -#define HTP_MM_WEIGHT_TILE_SIZE_IQ4_NL 576 -#define HTP_MM_WEIGHT_TILE_SIZE_MXFP4 544 -// Q5_K: the Q4_1 tile (640) followed by a 128-byte plane with the 5th bit of every quant, transposed so that -// plane byte l holds the eight flags of lane l: bit 2i = low nibble of nibble vector i, bit 2i+1 = high nibble -#define HTP_MM_WEIGHT_TILE_SIZE_Q5_K 768 -// Q6_K native 6-bit tile (32 rows x 32 k), vrmpy-ready: byte 4*row+b of a vector holds k = 4*group+b -// vectors 0..3: low nibbles, vector i holds group 2i (low nibble) and group 2i+1 (high nibble) -// vectors 4..5: high 2 bits, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6 -// vector 6: fp16 scales per row, d * scales[]: k 0..15 in lanes 0..31, k 16..31 in lanes 32..63 -#define HTP_MM_WEIGHT_TILE_SIZE_Q6_K 896 -// Q3_K native 3-bit tile, vrmpy-ready like Q6_K -// vectors 0..1: low 2 bits, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6 -// vector 2: bit g set where the hmask bit of group g is clear (quant = low 2 bits - 4) -// vector 3: fp16 scales per row, d * (scales[] - 32): k 0..15 in lanes 0..31, k 16..31 in lanes 32..63 -#define HTP_MM_WEIGHT_TILE_SIZE_Q3_K 512 -// Q2_K native 2-bit tile, vrmpy-ready like Q6_K -// vectors 0..1: unsigned 2-bit quants, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6 -// vector 2: fp16 scales per row, d * (scales[] & 0xF), same lanes as Q3_K vector 3 -// vector 3: fp16 offsets per row, -dmin * (scales[] >> 4), same lanes -#define HTP_MM_WEIGHT_TILE_SIZE_Q2_K 512 - -// --- Weight Repacked Aligned Tile Sizes --- -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0 640 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_1 640 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0 1152 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_IQ4_NL 640 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4 640 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q5_K 768 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K 896 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q3_K 512 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K 512 +#define HTP_MM_WEIGHT_TILE_K_Q2_0 64 +#define HTP_MM_WEIGHT_TILE_SIZE_Q2_0 HTP_TENSOR_TILE_SIZE_Q2_0 +#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0 640 // --- Activation Tiled Block Sizes (including padding) --- +#define HTP_MM_ACT_BLOCK_K 128 #define HTP_MM_ACT_TILE_SIZE_Q8_0 1152 #define HTP_MM_ACT_TILE_SIZE_Q8_1 1280 +#define HTP_MM_ACT_TILE_SIZE_F16 2048 #define HTP_MM_MAX_PREFETCH 16 @@ -74,11 +45,12 @@ enum htp_mm_kernel_type { // HMX paths HTP_MM_KERNEL_HMX_2D, - HTP_MM_KERNEL_HMX_F16_BATCHED, + HTP_MM_KERNEL_HMX_BATCHED, // HVX floating-point paths HTP_MM_KERNEL_HVX_F16_F16_VTCM, HTP_MM_KERNEL_HVX_F32_F32_VTCM, + HTP_MM_KERNEL_HVX_BF16_F32_VTCM, // HVX quantized paths HTP_MM_KERNEL_HVX_QUANT_ROW, // standard row-wise parallel quantization @@ -87,24 +59,23 @@ enum htp_mm_kernel_type { // Op-specific struct for precomputed matmul params struct htp_mm_kernel_params { - int32_t kernel_type; // enum htp_mm_kernel_type - int32_t pipeline; // 1 = pipelined execution, 0 = standard + uint8_t kernel_type; // enum htp_mm_kernel_type + uint8_t pipeline; // 1 = pipelined execution, 0 = standard + uint8_t unused; // reserved + uint8_t n_hmx; // 1 = use HMX, 0 = use HVX + + uint8_t n_hvx; // Number of threads to spawn + uint8_t n_act_hvx; // Number of threads for activation preparation + uint8_t n_prefetch; // Prefetch lookahead buffers/rows in VTCM + uint8_t n_weights; // Number of weights for fused NX + int32_t m_chunk; // Row chunk size (M chunk) int32_t n_chunk; // Col chunk size (N chunk) - int32_t n_threads; // Number of threads to spawn - int32_t n_act_threads; // Number of threads for activation preparation - int32_t n_hmx; // 1 = use HMX, 0 = use HVX - int32_t n_prefetch; // Prefetch lookahead buffers/rows in VTCM int32_t tile_size; // Weight tile size int32_t aligned_tile_size; // Aligned weight tile size (padded to 128) - int32_t src1_row_size; // Row size for quantized activation + int32_t act_row_size; // Row size for activation scratchpad int32_t vtcm_size; // Total required scratchpad size in VTCM - int32_t vtcm_src0_size; // src0 scratchpad size in VTCM - int32_t vtcm_src1_size; // src1 scratchpad size in VTCM - int32_t vtcm_src2_size; // src2 scratchpad size in VTCM (fused only) - int32_t vtcm_src3_size; // src3 scratchpad size in VTCM (fused only) - int32_t vtcm_dst_size; // dst scratchpad size in VTCM - int32_t n_weights; // Number of weights for fused NX + int32_t vtcm_bias_size; // bias scratchpad size in VTCM (fused only) // Precomputed division values struct fastdiv_values div_ne12_ne1; @@ -112,7 +83,7 @@ struct htp_mm_kernel_params { struct fastdiv_values div_r2; struct fastdiv_values div_r3; struct fastdiv_values div_ne12; - struct fastdiv_values div_n_act_threads; + struct fastdiv_values div_n_act_hvx; struct fastdiv_values div_ne00_padded; }; @@ -147,6 +118,7 @@ static inline int htp_mm_hmx_compute_chunks(size_t vtcm_total, const size_t usable = vtcm_total - overhead; size_t best_cost = SIZE_MAX; + size_t best_tail_waste = SIZE_MAX; size_t best_mn = 0; size_t best_m = 0, best_n = 0; @@ -173,12 +145,17 @@ static inline int htp_mm_hmx_compute_chunks(size_t vtcm_total, size_t mblocks = ((size_t) m + mc - 1) / mc; size_t nblocks = ((size_t) n + nc - 1) / nc; size_t cost = mblocks * m_block_cost + nblocks * n_block_cost; + size_t rem = n % nc; + size_t tail_waste = (rem == 0) ? 0 : (nc - rem); size_t mn = mc * nc; - if (cost < best_cost || (cost == best_cost && mn > best_mn)) { - best_cost = cost; - best_mn = mn; - best_m = mc; - best_n = nc; + if (cost < best_cost || + (cost == best_cost && tail_waste < best_tail_waste) || + (cost == best_cost && tail_waste == best_tail_waste && mn > best_mn)) { + best_cost = cost; + best_tail_waste = tail_waste; + best_mn = mn; + best_m = mc; + best_n = nc; } } @@ -204,98 +181,41 @@ static inline int htp_mm_hmx_compute_chunks(size_t vtcm_total, return 0; } -// --- Tile Size Helpers --- -static inline uint32_t htp_mm_get_weight_tile_size(int weight_type) { - switch (weight_type) { - case HTP_TYPE_Q4_0: - case HTP_TYPE_IQ4_NL: - return HTP_MM_WEIGHT_TILE_SIZE_Q4_0; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: - return HTP_MM_WEIGHT_TILE_SIZE_Q4_1; - case HTP_TYPE_Q8_0: - return HTP_MM_WEIGHT_TILE_SIZE_Q8_0; - case HTP_TYPE_Q5_K: - return HTP_MM_WEIGHT_TILE_SIZE_Q5_K; - case HTP_TYPE_Q6_K: - return HTP_MM_WEIGHT_TILE_SIZE_Q6_K; - case HTP_TYPE_Q3_K: - return HTP_MM_WEIGHT_TILE_SIZE_Q3_K; - case HTP_TYPE_Q2_K: - return HTP_MM_WEIGHT_TILE_SIZE_Q2_K; - case HTP_TYPE_MXFP4: - return HTP_MM_WEIGHT_TILE_SIZE_MXFP4; - default: - return 0; - } -} - -static inline uint32_t htp_mm_get_weight_aligned_tile_size(int weight_type) { - switch (weight_type) { - case HTP_TYPE_Q4_0: - case HTP_TYPE_IQ4_NL: - return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0; - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: - return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_1; - case HTP_TYPE_Q8_0: - return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0; - case HTP_TYPE_Q5_K: - return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q5_K; - case HTP_TYPE_Q6_K: - return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K; - case HTP_TYPE_Q3_K: - return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q3_K; - case HTP_TYPE_Q2_K: - return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K; - case HTP_TYPE_MXFP4: - return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4; - default: - return 0; - } -} - // weight types whose tiles carry a per-block offset (x = d * q + m): the activations need block sums (q8_1) // (Q2_K: per-16 k sums, q8_1_s16) static inline bool htp_mm_weight_has_offset(int weight_type) { return weight_type == HTP_TYPE_Q4_1 || weight_type == HTP_TYPE_Q4_K || weight_type == HTP_TYPE_Q5_K || - weight_type == HTP_TYPE_Q2_K; + weight_type == HTP_TYPE_Q2_K || weight_type == HTP_TYPE_Q2_0 || weight_type == HTP_TYPE_Q5_1; } // --- Activation/Row Size Helpers --- static inline size_t htp_mm_q8_0_tiled_row_size(uint32_t ne) { - const uint32_t ne_padded = ((ne + 127) / 128) * 128; + const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K); const uint32_t nb_32 = ne_padded / 32; return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_0; } static inline size_t htp_mm_q8_1_tiled_row_size(uint32_t ne) { - const uint32_t ne_padded = ((ne + 127) / 128) * 128; + const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K); const uint32_t nb_32 = ne_padded / 32; return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_1; } -static inline size_t htp_mm_get_tiled_row_stride(int weight_type, uint32_t k) { - uint32_t nb = (k + QK_Q4_0_TILED - 1) / QK_Q4_0_TILED; - switch (weight_type) { - case HTP_TYPE_Q4_0: - case HTP_TYPE_IQ4_NL: - case HTP_TYPE_Q4_1: - case HTP_TYPE_Q4_K: - case HTP_TYPE_Q8_0: - case HTP_TYPE_Q5_K: - case HTP_TYPE_Q6_K: - case HTP_TYPE_Q3_K: - case HTP_TYPE_Q2_K: - case HTP_TYPE_MXFP4: - return (size_t) nb * htp_mm_get_weight_tile_size(weight_type); - case HTP_TYPE_F16: - return (size_t) k * sizeof(__fp16); - case HTP_TYPE_F32: - return (size_t) k * sizeof(float); - default: - return 0; +static inline size_t htp_mm_f16_tiled_row_size(uint32_t ne) { + const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K); + const uint32_t nb_32 = ne_padded / 32; + return nb_32 * HTP_MM_ACT_TILE_SIZE_F16; +} + +static inline size_t htp_mm_tiled_act_row_size(int weight_type, uint32_t ne) { + if (weight_type == HTP_TYPE_F16) { + return htp_mm_f16_tiled_row_size(ne); } + return htp_mm_weight_has_offset(weight_type) ? htp_mm_q8_1_tiled_row_size(ne) : htp_mm_q8_0_tiled_row_size(ne); +} + +static inline size_t htp_mm_act_raw_row_size(uint32_t ne) { + return hex_round_up(ne, HTP_MM_ACT_BLOCK_K) * sizeof(float); } static inline size_t htp_mm_round_up(size_t n, size_t m) { @@ -310,11 +230,11 @@ static inline void htp_mm_hmx_get_2d_chunk_costs( int wtype, uint32_t k, bool pipeline, uint32_t aligned_tile_size, size_t * size_per_n_out, size_t * size_per_m_out, size_t * size_per_mn_out ) { - const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32); - const size_t row_stride = htp_mm_get_tiled_row_stride(wtype, k); + const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16); + const size_t row_stride = htp_tensor_get_row_size(wtype, k); const size_t vec_dot_size = k * sizeof(uint16_t); - const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; - const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype); + const size_t qweight_row_stride = is_quant ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0; *size_per_n_out = (pipeline ? 2 : 1) * (is_quant ? qweight_row_stride : row_stride) + (pipeline ? 2 * vec_dot_size : vec_dot_size); @@ -349,7 +269,7 @@ struct htp_mm_hmx_vtcm_layout { size_t off_dst[2]; // [1] is only used when pipelined size_t off_scratch[2]; // dequantization scratch pads size_t off_scales; // HMX scales (256 bytes) - size_t off_src2; // src2 bias in VTCM + size_t off_bias; // bias in VTCM // Cached sizes of regions for HMX kernel use size_t weight_area_bytes; @@ -358,25 +278,23 @@ struct htp_mm_hmx_vtcm_layout { size_t output_area_bytes; size_t scratch_bytes[2]; size_t act_head_stride; - size_t src2_bytes; + size_t bias_bytes; size_t total_bytes; }; struct htp_mm_hvx_vtcm_layout { // Byte offsets from vtcm_base for each region - size_t off_src1; // vtcm_src1 (activation) + size_t off_act; // vtcm_act (activation) size_t off_src0; // vtcm_src0 (weight/Wk) - size_t off_src2; // vtcm_src2 (Wq / fused only) - size_t off_src3; // vtcm_src3 (Wv / fused only) + size_t off_bias; // vtcm_bias (bias / fused add only) size_t off_dst; // vtcm_dst (output scratch) size_t off_act_raw; // vtcm_act_raw (raw activation DMA staging) // Cached sizes size_t src0_bytes; - size_t src1_bytes; - size_t src2_bytes; - size_t src3_bytes; + size_t act_bytes; + size_t bias_bytes; size_t dst_bytes; size_t act_raw_bytes; @@ -392,26 +310,26 @@ static inline void htp_mm_hmx_vtcm_layout_build( size_t nc, uint32_t group_size, bool pipeline, - uint32_t act_threads, + uint32_t act_hvx, uint32_t aligned_tile_size, - size_t src2_size + size_t bias_size ) { size_t off = 0; - if (kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) { + if (kernel_type == HTP_MM_KERNEL_HMX_BATCHED) { const size_t vec_dot_size = k * sizeof(uint16_t); const size_t act_head_stride = mc * k; const size_t weight_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); const size_t activation_area_size = hex_align_up(group_size * act_head_stride * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); const size_t output_area_size = hex_align_up(group_size * mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); const size_t scratch_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + const size_t min_f32_size = hex_align_up(act_hvx * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); // Group A: Permanent activation tiles and scales size_t off_group_a = 0; VTCM_LAYOUT_ALLOC(off_group_a, off_act, activation_area_size); VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data - VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_src2, hex_align_up(src2_size, HTP_MM_HMX_TILE_SIZE), src2_size > 0); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE), bias_size > 0); // Group B: Compute-only buffers (starts at off_group_a) size_t off_group_b = off_group_a; @@ -425,7 +343,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t group_b_size = off_group_b - off_group_a; // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) - const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t max_f32_size = act_hvx * 64 * k * sizeof(float); const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); size_t off_group_c = off_group_a; VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); @@ -439,19 +357,19 @@ static inline void htp_mm_hmx_vtcm_layout_build( L->scratch_bytes[0] = scratch_area_size; L->scratch_bytes[1] = scratch_area_size; L->act_head_stride = act_head_stride; - L->src2_bytes = src2_size; + L->bias_bytes = bias_size; off = off_group_a + hex_smax(group_b_size, group_c_size); } else { // HTP_MM_KERNEL_HMX_2D - const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32); - const size_t row_stride = htp_mm_get_tiled_row_stride(wtype, k); + const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16); + const size_t row_stride = htp_tensor_get_row_size(wtype, k); const size_t vec_dot_size = k * sizeof(uint16_t); - const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype); - const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + const size_t min_f32_size = hex_align_up(act_hvx * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); const size_t weight_area_size = is_quant - ? hex_align_up((nc / 32) * n_k_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) + ? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) : hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE); const size_t act_area_size = hex_align_up(mc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); const size_t output_area_size = hex_align_up(mc * nc * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE); @@ -463,7 +381,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( size_t off_group_a = 0; VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data VTCM_LAYOUT_ALLOC(off_group_a, off_act, act_area_size); - VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_src2, hex_align_up(src2_size, HTP_MM_HMX_TILE_SIZE), src2_size > 0); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE), bias_size > 0); // Group B: Compute-only buffers (starts at off_group_a) size_t off_group_b = off_group_a; @@ -477,7 +395,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t group_b_size = off_group_b - off_group_a; // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) - const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t max_f32_size = act_hvx * 64 * k * sizeof(float); const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); size_t off_group_c = off_group_a; VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); @@ -491,7 +409,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( L->scratch_bytes[0] = scratch0_size; L->scratch_bytes[1] = scratch1_size; L->act_head_stride = 0; - L->src2_bytes = src2_size; + L->bias_bytes = bias_size; off = off_group_a + hex_smax(group_b_size, group_c_size); } @@ -504,120 +422,90 @@ static inline void htp_mm_hvx_vtcm_layout_build( int kernel_type, int wtype, uint32_t ne10, // k - uint32_t src1_nrows, // m_total - uint32_t n_threads, + uint32_t act_nrows, // m_total + uint32_t n_hvx, size_t dst_row_size, size_t src0_row_size, - size_t src1_row_size, - size_t src2_row_size, + size_t act_row_size, + size_t bias_row_size, uint32_t n_prefetch, bool is_matmul_id, bool is_fused_nx ) { - (void)src1_row_size; + (void)act_row_size; size_t src0_sz = 0; - size_t src1_sz = 0; - size_t src2_sz = src2_row_size > 0 ? htp_mm_round_up(src2_row_size, 128) : 0; - size_t src3_sz = 0; + size_t act_sz = 0; + size_t bias_sz = bias_row_size > 0 ? htp_mm_round_up(bias_row_size, 128) : 0; size_t dst_sz = 0; size_t act_raw_sz = 0; - const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 || - wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL || - wtype == HTP_TYPE_MXFP4 || wtype == HTP_TYPE_Q6_K || - wtype == HTP_TYPE_Q4_K || wtype == HTP_TYPE_Q5_K || - wtype == HTP_TYPE_Q3_K || wtype == HTP_TYPE_Q2_K); - if (is_fused_nx) { - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - - size_t weight_sz_per_thread = 0; - - if (is_repack) { - uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32; - uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - - weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); - } else { - weight_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); - } - - size_t tiled_act_row_size = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - size_t act_sz = hex_round_up(tiled_act_row_size * src1_nrows, 128); - size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); - - src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer - src1_sz = act_sz; // quantized activation buffer - src2_sz = 0; - src3_sz = 0; + uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); + uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t weight_sz_per_hvx = hex_round_up(n_prefetch * tile_row_size, 128); + + size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128); + size_t raw_row_size = htp_mm_act_raw_row_size(ne10); + + src0_sz = weight_sz_per_hvx * n_hvx; // shared single-weight prefetch buffer + act_sz = q_act_sz; // quantized activation buffer + bias_sz = 0; dst_sz = 0; - act_raw_sz = hex_round_up(raw_row_size * src1_nrows, 128); + act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); } else if (is_matmul_id) { - const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); - const size_t src1_row_size_tiled = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10) - : htp_mm_q8_0_tiled_row_size(ne10); - - size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); - src1_sz = htp_mm_round_up(src1_row_size_tiled * src1_nrows, 256); - - if (is_repack) { - const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - const uint32_t n_k_tiles = ne10 / 32; - const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - src0_sz_per_thread = repacked_vtcm_size; - } + const size_t act_row_size_tiled = htp_mm_tiled_act_row_size(wtype, ne10); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + const uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); + const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t src0_sz_per_hvx = htp_mm_round_up(n_prefetch * tile_row_size, 256); - size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256); - src0_sz = src0_sz_per_thread * n_threads; + size_t raw_row_size = htp_mm_act_raw_row_size(ne10); + + src0_sz = src0_sz_per_hvx * n_hvx; dst_sz = 0; - src2_sz = 0; - src3_sz = 0; - act_raw_sz = hex_round_up(raw_row_size * src1_nrows, 128); + bias_sz = 0; + act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); } else { const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); - const size_t dst_nrows = (src1_nrows > 1) ? 0 : 1; + const size_t dst_nrows = (act_nrows > 1) ? 0 : 1; switch (kernel_type) { case HTP_MM_KERNEL_HVX_F16_F16_VTCM: { - size_t f16_src1_row_size = htp_mm_round_up(ne10 * 2, 128); - src1_sz = htp_mm_round_up(f16_src1_row_size * src1_nrows, 256); - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; - dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; - act_raw_sz = hex_round_up(hex_round_up(ne10 * sizeof(float), 128) * src1_nrows, 128); + size_t f16_act_row_size = htp_mm_round_up(ne10 * 2, 128); + act_sz = htp_mm_round_up(f16_act_row_size * act_nrows, 256); + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_hvx; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_hvx : 0; + act_raw_sz = hex_round_up(hex_round_up(ne10 * sizeof(float), 128) * act_nrows, 128); break; } - case HTP_MM_KERNEL_HVX_F32_F32_VTCM: { - size_t f32_src1_row_size = htp_mm_round_up(ne10 * 4, 128); - src1_sz = htp_mm_round_up(f32_src1_row_size * src1_nrows, 256); - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; - dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + case HTP_MM_KERNEL_HVX_F32_F32_VTCM: + case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: { + size_t f32_act_row_size = htp_mm_round_up(ne10 * 4, 128); + act_sz = htp_mm_round_up(f32_act_row_size * act_nrows, 256); + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_hvx; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_hvx : 0; act_raw_sz = 0; break; } case HTP_MM_KERNEL_HVX_QUANT_BLOCK: case HTP_MM_KERNEL_HVX_QUANT_ROW: { - size_t q_src1_row_size = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); - src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); - - src0_sz = src0_sz * n_threads; - - if (is_repack) { - uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - uint32_t n_k_tiles = ne10 / 32; - uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - src0_sz = repacked_vtcm_size * n_threads; - } - - size_t dst_slice_per_thread = (dst_nrows > 0 && src1_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0; - dst_sz = dst_slice_per_thread * n_threads; - size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); - act_raw_sz = hex_round_up(raw_row_size * src1_nrows, 128); + size_t q_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + uint32_t n_weight_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); + uint32_t tile_row_size = n_weight_tiles * aligned_tile_size; + size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); + + src0_sz = tiled_vtcm_size * n_hvx; + act_sz = htp_mm_round_up(q_act_row_size * act_nrows, 256); + + size_t dst_slice_per_hvx = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_hvx - 1) / n_hvx, 128) : 0; + dst_sz = dst_slice_per_hvx * n_hvx; + size_t raw_row_size = htp_mm_act_raw_row_size(ne10); + act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); break; } default: @@ -627,9 +515,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( // Group A: Persistent buffers across chunk compute size_t off_group_a = 0; - VTCM_LAYOUT_ALLOC(off_group_a, off_src1, src1_sz); - VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_src2, src2_sz, src2_sz > 0); - VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_src3, src3_sz, src3_sz > 0); + VTCM_LAYOUT_ALLOC(off_group_a, off_act, act_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, bias_sz, bias_sz > 0); // Group B: Compute-only buffers (starts at off_group_a) size_t off_group_b = off_group_a; @@ -643,9 +530,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( const size_t group_c_size = off_group_c - off_group_a; L->src0_bytes = src0_sz; - L->src1_bytes = src1_sz; - L->src2_bytes = src2_sz; - L->src3_bytes = src3_sz; + L->act_bytes = act_sz; + L->bias_bytes = bias_sz; L->dst_bytes = dst_sz; L->act_raw_bytes = act_raw_sz; L->total_bytes = off_group_a + hex_smax(group_b_size, group_c_size); @@ -655,12 +541,12 @@ static inline bool htp_mm_hvx_solve_vtcm_params( int kernel_type, int wtype, uint32_t ne10, - uint32_t src1_nrows, - uint32_t n_threads, + uint32_t act_nrows, + uint32_t n_hvx, size_t dst_row_size, size_t src0_row_size, - size_t src1_row_size, - size_t src2_row_size, + size_t act_row_size, + size_t bias_row_size, uint32_t n_prefetch, size_t vtcm_budget, struct htp_mm_hvx_vtcm_layout * L_out, @@ -668,17 +554,17 @@ static inline bool htp_mm_hvx_solve_vtcm_params( ) { struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, src1_nrows, n_threads, - dst_row_size, src0_row_size, src1_row_size, src2_row_size, n_prefetch, false, false + &L, kernel_type, wtype, ne10, act_nrows, n_hvx, + dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); if (L.total_bytes <= vtcm_budget) { *L_out = L; - *m_chunk_out = src1_nrows; + *m_chunk_out = act_nrows; return true; } - const size_t fixed_bytes = L.src0_bytes + L.src2_bytes + L.dst_bytes; + const size_t fixed_bytes = L.src0_bytes + L.bias_bytes + L.dst_bytes; if (vtcm_budget <= fixed_bytes) { return false; } @@ -686,9 +572,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params( const size_t avail_act = vtcm_budget - fixed_bytes; size_t row_size = 0; if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { - row_size = htp_mm_weight_has_offset(wtype) - ? htp_mm_q8_1_tiled_row_size(ne10) - : htp_mm_q8_0_tiled_row_size(ne10); + row_size = htp_mm_tiled_act_row_size(wtype, ne10); } else if (kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) { row_size = hex_round_up(ne10 * 2, 128); } else { @@ -700,30 +584,30 @@ static inline bool htp_mm_hvx_solve_vtcm_params( size_t eff_row_size = row_size; if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { - eff_row_size += hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + eff_row_size += htp_mm_act_raw_row_size(ne10); } uint32_t m_chunk = (uint32_t) (avail_act / eff_row_size); if (m_chunk > 1) { m_chunk &= ~1U; } - if (m_chunk > src1_nrows) { - m_chunk = src1_nrows; + if (m_chunk > act_nrows) { + m_chunk = act_nrows; } if (m_chunk < 1) { return false; } htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, m_chunk, n_threads, - dst_row_size, src0_row_size, src1_row_size, src2_row_size, n_prefetch, false, false + &L, kernel_type, wtype, ne10, m_chunk, n_hvx, + dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); while (m_chunk > 2 && L.total_bytes > vtcm_budget) { m_chunk -= 2; htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, m_chunk, n_threads, - dst_row_size, src0_row_size, src1_row_size, src2_row_size, n_prefetch, false, false + &L, kernel_type, wtype, ne10, m_chunk, n_hvx, + dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); } @@ -737,18 +621,18 @@ static inline bool htp_mm_hvx_solve_vtcm_params( } static inline size_t htp_mm_hmx_get_2d_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size, size_t src2_size + int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_hvx, uint32_t aligned_tile_size, size_t bias_size ) { struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_threads, aligned_tile_size, src2_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_hvx, aligned_tile_size, bias_size); return L.total_bytes; } static inline size_t htp_mm_hmx_get_batched_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_threads, size_t src2_size) { + int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_hvx, size_t bias_size) { (void)pipeline; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, wtype, k, mc, nc, group_size, false, act_threads, 0, src2_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_hvx, 0, bias_size); return L.total_bytes; } @@ -758,24 +642,24 @@ static inline bool htp_mm_hmx_solve_batched_params( uint32_t ne01_padded, uint32_t ne11, uint32_t group_size, - int n_threads, + int n_hvx, bool pipeline, - size_t src2_size, + size_t bias_size, size_t vtcm_budget, size_t * m_chunk_out, size_t * n_chunk_out, - int * act_threads_out, + int * act_hvx_out, size_t * vtcm_size_out ) { size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; + int best_act_hvx = 0; size_t best_m_chunk = 0; size_t best_n_chunk = 0; size_t best_vtcm_size = 0; - int act_threads = n_threads; - while (act_threads >= 1) { - size_t group_overhead = htp_mm_hmx_get_batched_overhead() + (src2_size > 0 ? hex_align_up(src2_size, HTP_MM_HMX_TILE_SIZE) : 0); + int act_hvx = n_hvx; + while (act_hvx >= 1) { + size_t group_overhead = htp_mm_hmx_get_batched_overhead() + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0); size_t group_size_per_n, group_size_per_m, group_size_per_mn; htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); @@ -785,31 +669,31 @@ static inline bool htp_mm_hmx_solve_batched_params( if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, - &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_threads, src2_size); + &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { + size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_hvx, bias_size); if (exact_size <= vtcm_budget) { size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_hvx > best_act_hvx)) { best_mblocks = mblocks; - best_act_threads = act_threads; + best_act_hvx = act_hvx; best_m_chunk = m_chunk_candidate; best_n_chunk = n_chunk_candidate; best_vtcm_size = exact_size; } } } - if (act_threads == 1) { - act_threads = 0; + if (act_hvx == 1) { + act_hvx = 0; } else { - act_threads /= 2; + act_hvx /= 2; } } - if (best_act_threads > 0) { + if (best_act_hvx > 0) { *m_chunk_out = best_m_chunk; *n_chunk_out = best_n_chunk; *vtcm_size_out = best_vtcm_size; - *act_threads_out = best_act_threads; + *act_hvx_out = best_act_hvx; return true; } return false; @@ -822,28 +706,28 @@ static inline bool htp_mm_hmx_solve_2d_params( uint32_t ne01_padded, uint32_t ne11_padded, uint32_t m_for_cost, - int n_threads, + int n_hvx, bool pipeline, bool is_matmul_id, uint32_t aligned_tile_size, - size_t src2_size, + size_t bias_size, size_t vtcm_budget, size_t * m_chunk_out, size_t * n_chunk_out, - int * act_threads_out, + int * act_hvx_out, size_t * vtcm_size_out ) { size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; + int best_act_hvx = 0; size_t best_m_chunk = 0; size_t best_n_chunk = 0; size_t best_vtcm_size = 0; const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded; - int act_threads = n_threads; - while (act_threads >= 1) { - size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id) + (src2_size > 0 ? hex_align_up(src2_size, HTP_MM_HMX_TILE_SIZE) : 0); + int act_hvx = n_hvx; + while (act_hvx >= 1) { + size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id) + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0); size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); @@ -854,30 +738,30 @@ static inline bool htp_mm_hmx_solve_2d_params( if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size, src2_size); + size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_hvx, aligned_tile_size, bias_size); if (exact_size <= vtcm_budget) { size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_hvx > best_act_hvx)) { best_mblocks = mblocks; - best_act_threads = act_threads; + best_act_hvx = act_hvx; best_m_chunk = m_chunk_candidate; best_n_chunk = n_chunk_candidate; best_vtcm_size = exact_size; } } } - if (act_threads == 1) { - act_threads = 0; + if (act_hvx == 1) { + act_hvx = 0; } else { - act_threads /= 2; + act_hvx /= 2; } } - if (best_act_threads > 0) { + if (best_act_hvx > 0) { *m_chunk_out = best_m_chunk; *n_chunk_out = best_n_chunk; *vtcm_size_out = best_vtcm_size; - *act_threads_out = best_act_threads; + *act_hvx_out = best_act_hvx; return true; } return false; diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c index 85f25a8eb..b77e95a90 100644 --- a/ggml/src/ggml-hexagon/htp/pad-ops.c +++ b/ggml/src/ggml-hexagon/htp/pad-ops.c @@ -13,22 +13,32 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" +#include "htp-vtcm.h" +#include "pad-ops.h" /* Circular wrap: maps any integer x into [0, n) */ -static inline uint32_t wrap_around(int32_t x, uint32_t n) { - return (uint32_t)(((x % (int32_t)n) + (int32_t)n) % (int32_t)n); +static inline uint32_t wrap_around(int32_t x, uint32_t n, const struct fastdiv_values * div_n) { + if (x >= 0) { + const uint32_t ux = (uint32_t) x; + return ux < n ? ux : fastmodulo(ux, n, div_n); + } + const uint32_t ux = (uint32_t) (-x); + const uint32_t rem = ux < n ? ux : fastmodulo(ux, n, div_n); + return rem ? (n - rem) : 0; } /* Decompose a flat dst row index into (i1, i2, i3) */ static inline void pad_decompose_row(uint32_t ir, uint32_t ne1, uint32_t ne2, + const struct htp_pad_kernel_params * kparams, uint32_t *i1, uint32_t *i2, uint32_t *i3) { - *i1 = ir % ne1; - *i2 = (ir / ne1) % ne2; - *i3 = ir / (ne1 * ne2); + *i3 = fastdiv(ir, &kparams->div_ne1_ne2); + const uint32_t rem = ir - (*i3) * (ne1 * ne2); + *i2 = fastdiv(rem, &kparams->div_ne1); + *i1 = rem - (*i2) * ne1; } /* Return non-zero if row (i1,i2,i3) falls in the non-padded interior */ @@ -62,32 +72,35 @@ static inline dma_addr_t pad_src_row_data(const struct htp_tensor * src, /* Compute the DDR src row pointer for a circular row (wrap-around indexing) */ static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src, + const struct htp_pad_kernel_params * kparams, uint32_t i1, uint32_t i2, uint32_t i3, int32_t lp1, int32_t lp2, int32_t lp3) { return (const uint8_t *) src->data - + wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1] - + wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2] - + wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3]; + + wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1] + + wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2] + + wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3]; } static inline dma_addr_t pad_circ_src_row_data(const struct htp_tensor * src, + const struct htp_pad_kernel_params * kparams, uint32_t i1, uint32_t i2, uint32_t i3, int32_t lp1, int32_t lp2, int32_t lp3) { return src->data - + wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1] - + wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2] - + wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3]; + + wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1] + + wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2] + + wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3]; } struct htp_pad_context { struct htp_ops_context * octx; + const struct htp_pad_kernel_params * kparams; int32_t lp0, rp0; int32_t lp1, rp1; int32_t lp2, rp2; int32_t lp3, rp3; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_dst_rows; uint32_t row_start; @@ -123,8 +136,8 @@ struct htp_pad_context { \ const size_t type_size = pctx->type_size; \ \ - const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \ - const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows); + const uint32_t row_start = pctx->row_start + pctx->nrows_per_hvx * ith; \ + const uint32_t row_end = MIN(row_start + pctx->nrows_per_hvx, pctx->row_start + pctx->total_dst_rows); #define htp_pad_dma_preamble \ @@ -133,26 +146,26 @@ struct htp_pad_context { const size_t dst_row_size = pctx->dst_row_size; \ const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ \ - uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \ - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \ + uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; \ + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; \ \ - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; // --------------------------------------------------------------------------- // HVX vectorized PAD kernel // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { uint32_t i1, i2, i3; - pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3); + pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3); uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; @@ -200,7 +213,7 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da // HVX + DMA PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; @@ -219,7 +232,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void dst_row_size, dst_row_size_aligned, dst_row_size, 0); uint32_t i1, i2, i3; - pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3); + pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3); const int interior = pad_is_interior(i1, i2, i3, lp1, rp1, ne1, lp2, rp2, ne2, @@ -239,14 +252,14 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void // Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops, // push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i1, i2, i3; - pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3); + pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3); const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; @@ -278,7 +291,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void const uint32_t next_row = ir + 2; if (next_row < row_end) { uint32_t ni1, ni2, ni3; - pad_decompose_row(next_row, ne1, ne2, &ni1, &ni2, &ni3); + pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &ni1, &ni2, &ni3); const int next_interior = pad_is_interior(ni1, ni2, ni3, lp1, rp1, ne1, lp2, rp2, ne2, @@ -305,20 +318,20 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void // HVX circular PAD kernel // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_circular(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { uint32_t i1, i2, i3; - pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3); + pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3); uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; - const uint8_t * src_row = pad_circ_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3); + const uint8_t * src_row = pad_circ_src_row_ptr(src, pctx->kparams, i1, i2, i3, lp1, lp2, lp3); if (nb00 == type_size) { @@ -374,7 +387,7 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, // HVX + DMA circular PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_circular_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; @@ -393,8 +406,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i dst_row_size, dst_row_size_aligned, dst_row_size, 0); uint32_t pi1, pi2, pi3; - pad_decompose_row(ir, ne1, ne2, &pi1, &pi2, &pi3); - const dma_addr_t src_data = pad_circ_src_row_data(src, pi1, pi2, pi3, lp1, lp2, lp3); + pad_decompose_row(ir, ne1, ne2, pctx->kparams, &pi1, &pi2, &pi3); + const dma_addr_t src_data = pad_circ_src_row_data(src, pctx->kparams, pi1, pi2, pi3, lp1, lp2, lp3); dma_queue_push(dma_q, dma_make_data(src_spad_cur, src_data), src_row_size_aligned, src_row_size, src_row_size, 1); @@ -404,14 +417,14 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i // Main loop: pop completed DMAs, assemble circular row in VTCM with // aligned HVX ops, push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i1, i2, i3; - pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3); + pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3); const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); @@ -455,8 +468,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i const uint32_t next_row = ir + 2; if (next_row < row_end) { uint32_t nri1, nri2, nri3; - pad_decompose_row(next_row, ne1, ne2, &nri1, &nri2, &nri3); - const dma_addr_t next_src_data = pad_circ_src_row_data(src, nri1, nri2, nri3, lp1, lp2, lp3); + pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &nri1, &nri2, &nri3); + const dma_addr_t next_src_data = pad_circ_src_row_data(src, pctx->kparams, nri1, nri2, nri3, lp1, lp2, lp3); dma_queue_push(dma_q, dma_make_data(src_spad_cur, next_src_data), src_row_size_aligned, src_row_size, src_row_size, 1); @@ -475,15 +488,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i int op_pad(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - - // Only F32 supported - size_t type_size; - switch (src0->type) { - case HTP_TYPE_F32: type_size = 4; break; - default: - FARF(ERROR, "pad-hvx: unsupported type %u\n", src0->type); - return HTP_STATUS_NO_SUPPORT; - } + const struct htp_pad_kernel_params * kparams = + (const struct htp_pad_kernel_params *) octx->kernel_params; const int32_t lp0 = octx->op_params[0]; const int32_t rp0 = octx->op_params[1]; @@ -495,11 +501,10 @@ int op_pad(struct htp_ops_context * octx) { const int32_t rp3 = octx->op_params[7]; const int32_t circular = octx->op_params[8]; - const uint32_t ne0 = dst->ne[0]; - const uint32_t ne00 = src0->ne[0]; - - const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; - const size_t dst_row_size = (size_t)ne0 * type_size; + const uint32_t total_dst_rows = kparams->total_rows; + const size_t type_size = kparams->elem_size; + const size_t src_row_size = src0->ne[0] * type_size; + const size_t dst_row_size = dst->ne[0] * type_size; uint32_t row_start = 0; uint32_t nrows = total_dst_rows; @@ -516,60 +521,68 @@ int op_pad(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - - const size_t src_row_size = (size_t)ne00 * type_size; - const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); - - // Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread - const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned); + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } - const int use_dma = (src0->nb[0] == (uint32_t)type_size) && (ne00 >= 512) && - (octx->ctx->vtcm_size >= vtcm_needed); + const uint32_t n_hvx = octx->n_hvx; - if (!use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { + if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { return HTP_STATUS_NO_SUPPORT; } - if (use_dma) { - octx->src0_spad.size_per_thread = 2 * src_row_size_aligned; - octx->dst_spad.size_per_thread = 2 * dst_row_size_aligned; - octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread; - octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread; - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; + struct htp_pad_vtcm_layout layout = { 0 }; + if (kparams->use_dma) { + htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_hvx); + + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; + octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); + octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); octx->src0_spad.src = NULL; octx->dst_spad.src = NULL; } struct htp_pad_context pctx = { - .octx = octx, + .octx = octx, + .kparams = kparams, .lp0 = lp0, .rp0 = rp0, .lp1 = lp1, .rp1 = rp1, .lp2 = lp2, .rp2 = rp2, .lp3 = lp3, .rp3 = rp3, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), - .total_dst_rows = nrows, - .row_start = row_start, - .type_size = type_size, + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), + .total_dst_rows = nrows, + .row_start = row_start, + .type_size = type_size, .src_row_size = src_row_size, - .src_row_size_aligned = src_row_size_aligned, + .src_row_size_aligned = layout.src_row_size_aligned, .dst_row_size = dst_row_size, - .dst_row_size_aligned = dst_row_size_aligned, + .dst_row_size_aligned = layout.dst_row_size_aligned, }; FARF(HIGH, "pad-hvx%s%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) pads=(%d,%d,%d,%d,%d,%d,%d,%d)\n", - circular ? "-circ" : "", - use_dma ? "-dma" : "", + circular ? "-circ" : "", + kparams->use_dma ? "-dma" : "", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3); - if (circular && use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); } - else if (circular) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); } - else if (use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); } - else { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); } + switch (kparams->kernel_type) { + case HTP_PAD_KERNEL_HVX_CIRCULAR_DMA: + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_circular_dma, &pctx, n_hvx); + break; + case HTP_PAD_KERNEL_HVX_CIRCULAR: + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_circular, &pctx, n_hvx); + break; + case HTP_PAD_KERNEL_HVX_DMA: + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_dma, &pctx, n_hvx); + break; + default: + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx, &pctx, n_hvx); + break; + } return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.h b/ggml/src/ggml-hexagon/htp/pad-ops.h new file mode 100644 index 000000000..15357e5f6 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/pad-ops.h @@ -0,0 +1,65 @@ +#ifndef HTP_PAD_OPS_H +#define HTP_PAD_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include <stdint.h> + +enum htp_pad_kernel_type { + HTP_PAD_KERNEL_HVX = 0, + HTP_PAD_KERNEL_HVX_DMA = 1, + HTP_PAD_KERNEL_HVX_CIRCULAR = 2, + HTP_PAD_KERNEL_HVX_CIRCULAR_DMA = 3, +}; + +struct htp_pad_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + size_t src_row_size_aligned; + size_t dst_row_size_aligned; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; +}; + +static inline void htp_pad_vtcm_layout_build( + struct htp_pad_vtcm_layout * L, + size_t src_row_size, + size_t dst_row_size, + uint32_t n_hvx +) { + L->src_row_size_aligned = hex_round_up(src_row_size, 128); + L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); + L->src0_bytes_per_hvx = 2 * L->src_row_size_aligned; + L->dst_bytes_per_hvx = 2 * L->dst_row_size_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + + L->total_bytes = off; +} + +struct htp_pad_kernel_params { + uint8_t kernel_type; + uint32_t n_hvx; + uint32_t total_rows; + uint32_t use_dma; + uint32_t elem_size; + uint32_t vtcm_size; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne1_ne2; + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne03; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_PAD_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.c b/ggml/src/ggml-hexagon/htp/pool-ops.c new file mode 100644 index 000000000..12cd8ea90 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/pool-ops.c @@ -0,0 +1,390 @@ +#pragma clang diagnostic ignored "-Wunused-variable" + +#include <float.h> +#include <HAP_farf.h> + +#include "hex-common.h" +#include "dma-queue.h" +#include "htp-trace.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-inverse.h" +#include "hvx-types.h" +#include "hvx-utils.h" +#include "pool-ops.h" + +#define HTP_POOL_MAX 0 +#define HTP_POOL_AVG 1 + +// Fast path: exact non-overlapping tiling (stride == kernel, no padding), kernel_x in {1,2}. +// Every window is guaranteed fully in-bounds, so this never needs boundary clamping. +static void pool_plane_hvx( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + const bool is_max = (p->pool_op == HTP_POOL_MAX); + const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area); + const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + + for (uint32_t oy = 0; oy < p->dst_y; ++oy) { + const uint32_t sy = oy * p->kernel_y; + for (uint32_t ox = 0; ox < p->dst_x; ox += VLEN_FP32) { + const uint32_t rem = p->dst_x - ox; + const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN; + HVX_Vector acc = seed; + for (uint32_t ky = 0; ky < p->kernel_y; ++ky) { + const float * row = src + (sy + ky) * p->src_x; + if (p->kernel_x == 2) { + const HVX_Vector v0 = *(const HVX_UVector *) (row + ox * 2); + const HVX_Vector v1 = *(const HVX_UVector *) (row + ox * 2 + VLEN_FP32); + const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(v1, v0, -4); + const HVX_Vector lo = Q6_V_lo_W(deinterleaved); + const HVX_Vector hi = Q6_V_hi_W(deinterleaved); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, Q6_Vsf_vmax_VsfVsf(lo, hi)) + : hvx_vec_add_f32_f32(acc, hvx_vec_add_f32_f32(lo, hi)); + } else if (p->kernel_x == 1) { + const HVX_Vector v = *(const HVX_UVector *) (row + ox); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v); + } + } + hvx_vec_store_u(dst + oy * p->dst_x + ox, nbytes, is_max ? acc : hvx_vec_mul_f32_f32(acc, scale)); + } + } +} + +// Narrow exact-tiling path. The input is staged in VTCM with one vector of +// guard space, so full-width loads are safe even when src_x is below 32. +static void pool_plane_hvx_narrow( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + const bool is_max = (p->pool_op == HTP_POOL_MAX); + const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area); + const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + + for (uint32_t oy = 0; oy < p->dst_y; ++oy) { + const uint32_t sy = oy * p->kernel_y; + HVX_Vector acc = seed; + for (uint32_t ky = 0; ky < p->kernel_y; ++ky) { + const float * row = src + (sy + ky) * p->src_x; + const HVX_Vector v0 = *(const HVX_UVector *) row; + HVX_Vector v = v0; + if (p->kernel_x == 2) { + // The second vector is zero because a narrow row has fewer + // than 32 input elements. The low lanes still contain the + // complete even/odd pairs needed by the output. + const HVX_Vector zero = Q6_V_vsplat_R(0); + const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(zero, v0, -4); + const HVX_Vector even = Q6_V_lo_W(deinterleaved); + const HVX_Vector odd = Q6_V_hi_W(deinterleaved); + v = is_max ? Q6_Vsf_vmax_VsfVsf(even, odd) + : hvx_vec_add_f32_f32(even, odd); + } + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) + : hvx_vec_add_f32_f32(acc, v); + } + hvx_vec_store_u(dst + oy * p->dst_x, p->dst_x * sizeof(float), + is_max ? acc : hvx_vec_mul_f32_f32(acc, scale)); + } +} + +static void pool_plane_global( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + const uint32_t n = p->src_x * p->src_y; + float val; + if (p->pool_op == HTP_POOL_MAX) { + val = hvx_reduce_max_f32((const uint8_t *) src, n); + } else { + val = hvx_reduce_sum_f32((const uint8_t *) src, n) * p->inv_kernel_area; + } + hvx_vec_store_u(dst, sizeof(float), hvx_vec_splat_f32(val)); +} + +static void pool_plane_block( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + for (uint32_t oy = 0; oy < p->dst_y; ++oy) { + const float * row = src + oy * p->src_x; + float * dst_row = dst + oy * p->dst_x; + for (uint32_t ox = 0; ox < p->dst_x; ++ox) { + const uint8_t * block = (const uint8_t *) (row + ox * p->kernel_x); + float val; + if (p->pool_op == HTP_POOL_MAX) { + val = hvx_reduce_max_f32(block, p->kernel_x); + } else { + val = hvx_reduce_sum_f32(block, p->kernel_x) * p->inv_kernel_area; + } + hvx_vec_store_u(dst_row + ox, sizeof(float), hvx_vec_splat_f32(val)); + } + } +} + +// General path: arbitrary kernel/stride/padding + +// Vertical padding is uniform across a whole output row (iy0 depends only on oy, not ox), +// so it collapses to one valid-ky range per row instead of a per-element check. +static inline void pool_row_bounds_y( + const struct htp_pool_2d_kernel_params * p, uint32_t oy, int32_t * iy0, uint32_t * ky_lo, uint32_t * ky_hi) { + *iy0 = (int32_t) (oy * p->stride_y) - p->pad_y; + const int32_t lo = -(*iy0); + const int32_t hi = (int32_t) p->src_y - *iy0; + *ky_lo = (uint32_t) MAX(0, lo); + *ky_hi = (uint32_t) MAX(0, MIN((int32_t) p->kernel_y, hi)); +} + +static inline void pool_pixel_boundary_vec( + const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p, + uint32_t ox, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) { + const int32_t ix0 = (int32_t) (ox * p->stride_x) - p->pad_x; + const int32_t kx_lo = MAX(0, -ix0); + const int32_t kx_hi = MIN((int32_t) p->kernel_x, (int32_t) p->src_x - ix0); + + if (kx_lo >= kx_hi || ky_lo >= ky_hi) { + HVX_Vector empty_val = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + hvx_vec_store_u(dst_row + ox, sizeof(float), empty_val); + return; + } + + const uint32_t valid_kx = (uint32_t) (kx_hi - kx_lo); + const HVX_Vector mask_identity = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + + HVX_Vector acc = mask_identity; + for (uint32_t ky = ky_lo; ky < ky_hi; ++ky) { + const float * row = src + (uint32_t) (iy0 + (int32_t) ky) * p->src_x; + for (uint32_t k = 0; k < valid_kx; k += VLEN_FP32) { + const uint32_t k_rem = valid_kx - k; + const uint32_t n = (k_rem < VLEN_FP32) ? k_rem : VLEN_FP32; + const HVX_VectorPred q = Q6_Q_vsetq_R(n * sizeof(float)); + const HVX_Vector raw = *(const HVX_UVector *) (row + ix0 + kx_lo + k); + const HVX_Vector v = Q6_V_vmux_QVV(q, raw, mask_identity); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v); + } + } + + HVX_Vector reduced = is_max ? hvx_vec_reduce_max_f32(acc) : hvx_vec_reduce_sum_f32(acc); + if (!is_max) { + HVX_Vector scale_vec; + if (p->avg_divide_count) { + const uint32_t count = (ky_hi - ky_lo) * valid_kx; + scale_vec = hvx_vec_inverse_f32(hvx_vec_splat_f32((float) count)); + } else { + scale_vec = hvx_vec_splat_f32(p->inv_kernel_area); + } + reduced = hvx_vec_mul_f32_f32(reduced, scale_vec); + } + hvx_vec_store_u(dst_row + ox, sizeof(float), reduced); +} + +static inline void pool_row_general_boundary_vec( + const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p, + uint32_t ox_start, uint32_t ox_end, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) { + for (uint32_t ox = ox_start; ox < ox_end; ++ox) { + pool_pixel_boundary_vec(src, dst_row, p, ox, iy0, ky_lo, ky_hi, is_max); + } +} + +// Vectorized interior loop. +static inline void pool_row_general_vec( + const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p, + uint32_t ox_start, uint32_t ox_end, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) { + if (ox_start >= ox_end) { + return; + } + + if (p->stride_x != 1 && p->stride_x != 2) { + pool_row_general_boundary_vec(src, dst_row, p, ox_start, ox_end, iy0, ky_lo, ky_hi, is_max); + return; + } + + const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area); + const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + + for (uint32_t ox = ox_start; ox < ox_end; ox += VLEN_FP32) { + const uint32_t rem = ox_end - ox; + const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN; + const int32_t ix0 = (int32_t) (ox * p->stride_x) - p->pad_x; + HVX_Vector acc = seed; + + for (uint32_t ky = ky_lo; ky < ky_hi; ++ky) { + const float * row = src + (uint32_t) (iy0 + (int32_t) ky) * p->src_x; + if (p->stride_x == 1) { + for (uint32_t kx = 0; kx < p->kernel_x; ++kx) { + const HVX_Vector v = *(const HVX_UVector *) (row + ix0 + (int32_t) kx); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v); + } + } else { + for (uint32_t kx = 0; kx < p->kernel_x; kx += 2) { + const HVX_Vector v0 = *(const HVX_UVector *) (row + ix0 + (int32_t) kx); + const HVX_Vector v1 = *(const HVX_UVector *) (row + ix0 + (int32_t) kx + VLEN_FP32); + const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(v1, v0, -4); + const HVX_Vector lo = Q6_V_lo_W(deinterleaved); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, lo) : hvx_vec_add_f32_f32(acc, lo); + if (kx + 1 < p->kernel_x) { + const HVX_Vector hi = Q6_V_hi_W(deinterleaved); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, hi) : hvx_vec_add_f32_f32(acc, hi); + } + } + } + } + hvx_vec_store_u(dst_row + ox, nbytes, is_max ? acc : hvx_vec_mul_f32_f32(acc, scale)); + } +} + +static void pool_plane_general( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + const bool is_max = (p->pool_op == HTP_POOL_MAX); + + for (uint32_t oy = 0; oy < p->dst_y; ++oy) { + int32_t iy0; + uint32_t ky_lo, ky_hi; + pool_row_bounds_y(p, oy, &iy0, &ky_lo, &ky_hi); + float * dst_row = dst + oy * p->dst_x; + + pool_row_general_boundary_vec(src, dst_row, p, 0, p->ox_lo, iy0, ky_lo, ky_hi, is_max); + pool_row_general_vec(src, dst_row, p, p->ox_lo, p->ox_hi, iy0, ky_lo, ky_hi, is_max); + pool_row_general_boundary_vec(src, dst_row, p, p->ox_hi, p->dst_x, iy0, ky_lo, ky_hi, is_max); + } +} + +typedef void (*pool_plane_fn_t)(const float * src, float * dst, const struct htp_pool_2d_kernel_params * p); + +struct pool_2d_context { + struct htp_ops_context * octx; + const struct htp_pool_2d_kernel_params * kparams; + struct htp_pool_vtcm_layout layout; + pool_plane_fn_t pool_plane; + uint32_t n_hvx; + uint32_t plane_start; + uint32_t plane_count; + uint32_t planes_per_hvx; +}; + +static void pool_2d_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct pool_2d_context * ctx = (struct pool_2d_context *) data; + const struct htp_pool_2d_kernel_params * p = ctx->kparams; + const struct htp_pool_vtcm_layout * L = &ctx->layout; + const struct htp_tensor * src0 = ctx->octx->src[0]; + const struct htp_tensor * dst = ctx->octx->dst; + pool_plane_fn_t pool_plane = ctx->pool_plane; + const uint32_t planes_per_hvx = ctx->planes_per_hvx; + const uint32_t first = ctx->plane_start + ith * planes_per_hvx; + const uint32_t last = MIN(first + planes_per_hvx, ctx->plane_start + ctx->plane_count); + + if (first >= last) { + return; + } + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_queue = th->dma_q; + + const uint32_t src_plane_bytes = L->src_plane_bytes; + const uint32_t dst_plane_bytes = L->dst_plane_bytes; + const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size; + const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; + + uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base; + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; + + float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; + float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) }; + + const uint32_t total = last - first; + + // Warm up the pipeline: push up to 2 initial (dummy dst, src) transfer pairs. + for (uint32_t i = 0; i < total && i < 2; ++i) { + dma_queue_push(dma_queue, + dma_make_data(dst->data, dstb2[i]), + dst_plane_bytes, dst_spad_half, + dst_plane_bytes, 0); + + const dma_addr_t src_addr = src0->data + (first + i) * src_plane_bytes; + dma_queue_push(dma_queue, + dma_make_data(srcb2[i], src_addr), + src_spad_half, src_plane_bytes, + src_plane_bytes, 1); + } + + for (uint32_t i = 0; i < total; ++i) { + const uint32_t plane = first + i; + const uint32_t buf = i & 1u; + float * srcb = srcb2[buf]; + float * dstb = dstb2[buf]; + + dma_queue_pop(dma_queue); // dst writeback from plane i - 2 (or dummy on iter 0, 1) + dma_queue_pop(dma_queue); // input for plane i + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) plane); + pool_plane(srcb, dstb, p); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) plane); + + const dma_addr_t dst_addr = dst->data + plane * dst_plane_bytes; + dma_queue_push(dma_queue, + dma_make_data(dst_addr, dstb), + dst_plane_bytes, dst_spad_half, + dst_plane_bytes, 1); + + if (i + 2 < total) { + const dma_addr_t next_src_addr = src0->data + (plane + 2) * src_plane_bytes; + dma_queue_push(dma_queue, + dma_make_data(srcb, next_src_addr), + src_spad_half, src_plane_bytes, + src_plane_bytes, 1); + } + } + + dma_queue_flush(dma_queue); + + FARF(HIGH, "pool2d-f32-dma %d/%d: %ux%ux%ux%u -> %ux%ux%ux%u (%u:%u)\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], first, last); + (void) nth; +} + +int op_pool_2d(struct htp_ops_context * octx) { + const struct htp_tensor * dst = octx->dst; + const struct htp_pool_2d_kernel_params * p = + (const struct htp_pool_2d_kernel_params *) octx->kernel_params; + + uint32_t plane_start = 0; + uint32_t plane_count = p->planes; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + plane_count, htp_tensor_mdev_data_aligned(dst) ? p->planes_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + plane_start = range.start; + plane_count = range.count; + } + if (plane_count == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_hvx = MIN(p->n_hvx, plane_count); + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t planes_per_hvx = fastdiv(plane_count + n_hvx - 1, &octx->n_hvx_div); + + pool_plane_fn_t pool_plane; + switch (p->kernel_type) { + case HTP_POOL_KERNEL_GLOBAL: pool_plane = pool_plane_global; break; + case HTP_POOL_KERNEL_BLOCK: pool_plane = pool_plane_block; break; + case HTP_POOL_KERNEL_HVX_NARROW: pool_plane = pool_plane_hvx_narrow; break; + case HTP_POOL_KERNEL_HVX: pool_plane = pool_plane_hvx; break; + default: pool_plane = pool_plane_general; break; + } + + struct pool_2d_context ctx = { + .octx = octx, + .kparams = p, + .pool_plane = pool_plane, + .n_hvx = n_hvx, + .plane_start = plane_start, + .plane_count = plane_count, + .planes_per_hvx = planes_per_hvx, + }; + htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_hvx); + work_queue_run(octx->ctx->hvx_queue, pool_2d_thread, &ctx, n_hvx); + return HTP_STATUS_OK; +} + +int op_pool_1d(struct htp_ops_context * octx) { + return op_pool_2d(octx); +} diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.h b/ggml/src/ggml-hexagon/htp/pool-ops.h new file mode 100644 index 000000000..e113bb9dd --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/pool-ops.h @@ -0,0 +1,95 @@ +#ifndef HTP_POOL_OPS_H +#define HTP_POOL_OPS_H + +#include <stdint.h> +#include <stddef.h> +#include <stdbool.h> +#include <string.h> + +#include "hex-common.h" +#include "htp-vtcm.h" + +enum htp_pool_kernel_type { + HTP_POOL_KERNEL_GENERAL = 0, + HTP_POOL_KERNEL_HVX, + HTP_POOL_KERNEL_HVX_NARROW, + HTP_POOL_KERNEL_GLOBAL, + HTP_POOL_KERNEL_BLOCK, +}; + +struct htp_pool_2d_kernel_params { + uint8_t kernel_type; + uint32_t src_x; + uint32_t src_y; + uint32_t dst_x; + uint32_t dst_y; + uint32_t kernel_x; + uint32_t kernel_y; + uint32_t stride_x; + uint32_t stride_y; + int32_t pad_x; + int32_t pad_y; + uint32_t n_hvx; + uint32_t planes; + uint32_t planes_per_chunk; + uint32_t pool_op; + uint32_t avg_divide_count; + uint32_t ox_lo; + uint32_t ox_hi; + float inv_kernel_area; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_pool_2d_kernel_params) <= 128, "htp_pool_2d_kernel_params is too large"); +#else +_Static_assert(sizeof(struct htp_pool_2d_kernel_params) <= 128, "htp_pool_2d_kernel_params is too large"); +#endif + +struct htp_pool_vtcm_layout { + size_t total_bytes; + size_t off_src; + size_t off_dst; + uint32_t src_plane_bytes; + uint32_t dst_plane_bytes; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; + size_t src_spad_half_size; + size_t dst_spad_half_size; +}; + +static inline void htp_pool_vtcm_layout_build( + struct htp_pool_vtcm_layout * L, + uint32_t src_x, + uint32_t src_y, + uint32_t dst_x, + uint32_t dst_y, + uint32_t n_hvx +) { + // Full-plane double buffering (256 bytes guard space for vector loads) + L->src_plane_bytes = src_x * src_y * sizeof(float); + L->dst_plane_bytes = dst_x * dst_y * sizeof(float); + L->src_spad_half_size = hex_round_up(L->src_plane_bytes + 256, 128); + L->dst_spad_half_size = hex_round_up(L->dst_plane_bytes, 128); + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + L->total_bytes = off; +} + +static inline bool htp_pool_solve_layout( + struct htp_pool_vtcm_layout * layout, + uint32_t src_x, + uint32_t src_y, + uint32_t dst_x, + uint32_t dst_y, + uint32_t n_hvx, + size_t vtcm_budget +) { + htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_hvx); + return layout->total_bytes <= vtcm_budget; +} + +#endif // HTP_POOL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c index 2551be225..a9e4d5c86 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.c +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c @@ -13,28 +13,26 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" +#include "repeat-ops.h" struct htp_repeat_context { - struct htp_ops_context * octx; + struct htp_ops_context * octx; + const struct htp_repeat_kernel_params * kparams; - uint32_t nr0; - uint32_t nr1; - uint32_t nr2; - uint32_t nr3; - - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_dst_rows; // ne1 * ne2 * ne3 uint32_t row_start; size_t type_size; }; -static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * data) { +static void repeat_job_per_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_repeat_context * rctx = (const struct htp_repeat_context *) data; struct htp_ops_context * octx = rctx->octx; + const struct htp_repeat_kernel_params * kparams = rctx->kparams; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; @@ -58,29 +56,30 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat const uint32_t nb2 = dst->nb[2]; const uint32_t nb3 = dst->nb[3]; - const uint32_t nr0 = rctx->nr0; - const uint32_t nr1 = rctx->nr1; - const uint32_t nr2 = rctx->nr2; - const uint32_t nr3 = rctx->nr3; + const uint32_t nr0 = kparams->nr0; const size_t row_bytes = ne00 * rctx->type_size; - const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_hvx, rctx->row_start + rctx->total_dst_rows); + if (row_start >= row_end) { + return; + } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { // Decompose flat dst row index into (i1, i2, i3) - const uint32_t i1 = dst_row % ne1; - const uint32_t i2 = (dst_row / ne1) % ne2; - const uint32_t i3 = dst_row / (ne1 * ne2); + const uint32_t dst_row_div_ne1 = fastdiv(dst_row, &kparams->div_ne1); + const uint32_t i1 = dst_row - dst_row_div_ne1 * ne1; + const uint32_t i3 = fastdiv(dst_row, &kparams->div_ne12); + const uint32_t i2 = dst_row_div_ne1 - i3 * ne2; // Map to source indices (tiling) - const uint32_t k1 = i1 % ne01; - const uint32_t k2 = i2 % ne02; - const uint32_t k3 = i3 % ne03; + const uint32_t k1 = fastmodulo(i1, ne01, &kparams->div_ne01); + const uint32_t k2 = fastmodulo(i2, ne02, &kparams->div_ne02); + const uint32_t k3 = fastmodulo(i3, ne03, &kparams->div_ne03); const uint8_t * src_row = (const uint8_t *) src->data + k1 * nb01 + k2 * nb02 + k3 * nb03; uint8_t * dst_base = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; @@ -104,15 +103,6 @@ int op_repeat(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - // Validate that dst dims are multiples of src dims - if (dst->ne[0] % src0->ne[0] != 0 || - dst->ne[1] % src0->ne[1] != 0 || - dst->ne[2] % src0->ne[2] != 0 || - dst->ne[3] % src0->ne[3] != 0) { - FARF(ERROR, "repeat: dst dims must be multiples of src dims\n"); - return HTP_STATUS_INVAL_PARAMS; - } - size_t type_size; switch (src0->type) { case HTP_TYPE_F32: type_size = 4; break; @@ -126,6 +116,13 @@ int op_repeat(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } + const struct htp_repeat_kernel_params * kparams = + (const struct htp_repeat_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; const size_t dst_row_size = dst->ne[0] * type_size; @@ -144,15 +141,12 @@ int op_repeat(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_repeat_context rctx = { .octx = octx, - .nr0 = dst->ne[0] / src0->ne[0], - .nr1 = dst->ne[1] / src0->ne[1], - .nr2 = dst->ne[2] / src0->ne[2], - .nr3 = dst->ne[3] / src0->ne[3], - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .kparams = kparams, + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_dst_rows = nrows, .row_start = row_start, .type_size = type_size, @@ -161,9 +155,9 @@ int op_repeat(struct htp_ops_context * octx) { FARF(HIGH, "repeat: (%ux%ux%ux%u) -> (%ux%ux%ux%u) nr=(%u,%u,%u,%u)\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - rctx.nr0, rctx.nr1, rctx.nr2, rctx.nr3); + kparams->nr0, kparams->nr1, kparams->nr2, kparams->nr3); - work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, repeat_job_per_hvx, &rctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.h b/ggml/src/ggml-hexagon/htp/repeat-ops.h new file mode 100644 index 000000000..1561994d4 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.h @@ -0,0 +1,27 @@ +#ifndef HTP_REPEAT_OPS_H +#define HTP_REPEAT_OPS_H + +#include "hex-fastdiv.h" +#include <stdint.h> + +struct htp_repeat_kernel_params { + uint32_t n_hvx; + uint32_t nr0; + uint32_t nr1; + uint32_t nr2; + uint32_t nr3; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne12; + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne03; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_REPEAT_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.c b/ggml/src/ggml-hexagon/htp/roll-ops.c index 9c373f56d..aa7a0ad4b 100644 --- a/ggml/src/ggml-hexagon/htp/roll-ops.c +++ b/ggml/src/ggml-hexagon/htp/roll-ops.c @@ -14,19 +14,18 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" +#include "roll-ops.h" struct htp_roll_context { struct htp_ops_context * octx; + const struct htp_roll_kernel_params * kparams; uint32_t row_start; uint32_t nrows; - uint32_t nrows_per_thread; - - struct fastdiv_values div_ne1; - struct fastdiv_values div_ne2_ne1; + uint32_t nrows_per_hvx; }; static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) { @@ -39,30 +38,32 @@ static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) { return (uint32_t) i; } -#define htp_roll_preamble \ - const struct htp_tensor * src0 = octx->src[0]; \ - const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const int32_t s0 = octx->op_params[0]; \ - const int32_t s1 = octx->op_params[1]; \ - const int32_t s2 = octx->op_params[2]; \ - const int32_t s3 = octx->op_params[3]; \ - \ - const uint32_t i0_src0 = htp_roll_wrap(-s0, ne0); \ - const uint32_t n0 = ne0 - i0_src0; +#define htp_roll_preamble \ + const struct htp_tensor * src0 = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + const struct htp_roll_kernel_params * kparams = \ + (const struct htp_roll_kernel_params *) octx->kernel_params; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ + const int32_t s0 = octx->op_params[0]; \ + const int32_t s1 = octx->op_params[1]; \ + const int32_t s2 = octx->op_params[2]; \ + const int32_t s3 = octx->op_params[3]; \ + \ + const uint32_t i0_src0 = kparams->i0_src0; \ + const uint32_t n0 = kparams->n0; #define htp_roll_dma_preamble dma_queue * q = octx->ctx->dma[0]; @@ -91,10 +92,10 @@ static inline void roll_dma_push_rows(dma_queue * q, uint32_t src_row, uint32_t nrows, uint32_t row_size, - uint32_t i0_src0) { + uint32_t i0_src0, + uint32_t n0) { const dma_addr_t dst_base = dst->data + (size_t) dst_row * row_size; const dma_addr_t src_base = src0->data + (size_t) src_row * row_size; - const uint32_t n0 = src0->ne[0] - i0_src0; roll_dma_push(q, dst_base, src_base + (size_t) i0_src0 * sizeof(float), row_size, row_size, n0 * sizeof(float), nrows); @@ -125,7 +126,7 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) { const uint32_t row_size = ne0 * sizeof(float); if (s1 == 0 && s2 == 0 && s3 == 0) { - roll_dma_push_rows(q, dst, src0, 0, 0, ne1 * ne2 * ne3, row_size, i0_src0); + roll_dma_push_rows(q, dst, src0, 0, 0, ne1 * ne2 * ne3, row_size, i0_src0, n0); dma_queue_flush(q); return HTP_STATUS_OK; } @@ -139,9 +140,9 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) { const uint32_t n2_first = ne2 - i2_src0; roll_dma_push_rows(q, dst, src0, dst_row0, src_row0, n2_first * ne1, - row_size, i0_src0); + row_size, i0_src0, n0); roll_dma_push_rows(q, dst, src0, dst_row0 + n2_first * ne1, i03 * ne2 * ne1, - i2_src0 * ne1, row_size, i0_src0); + i2_src0 * ne1, row_size, i0_src0, n0); } dma_queue_flush(q); @@ -159,9 +160,9 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) { const uint32_t src_row0 = (i03 * ne2 + i02) * ne1; roll_dma_push_rows(q, dst, src0, dst_row0, src_row0 + i1_src0, - n1_first, row_size, i0_src0); + n1_first, row_size, i0_src0, n0); roll_dma_push_rows(q, dst, src0, dst_row0 + n1_first, src_row0, - i1_src0, row_size, i0_src0); + i1_src0, row_size, i0_src0, n0); } } @@ -198,25 +199,25 @@ static int roll_dma_f32_strided(struct htp_ops_context * octx) { return HTP_STATUS_OK; } -static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void roll_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_roll_context * rctx = (struct htp_roll_context *) data; struct htp_ops_context * octx = rctx->octx; htp_roll_preamble; - const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->nrows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_hvx, rctx->row_start + rctx->nrows); if (row_start >= row_end) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t row = row_start; row < row_end; row++) { - const uint32_t i3 = fastdiv(row, &rctx->div_ne2_ne1); + const uint32_t i3 = fastdiv(row, &kparams->div_ne2_ne1); const uint32_t rem = row - i3 * ne2 * ne1; - const uint32_t i2 = fastdiv(rem, &rctx->div_ne1); + const uint32_t i2 = fastdiv(rem, &kparams->div_ne1); const uint32_t i1 = rem - i2 * ne1; const uint32_t i01 = htp_roll_wrap((int32_t) i1 - s1, ne1); @@ -245,23 +246,14 @@ static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) { int execute_op_roll_f32(struct htp_ops_context * octx) { htp_roll_preamble; - if (src0->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { - FARF(ERROR, "roll: unsupported type %u -> %u\n", src0->type, dst->type); - return HTP_STATUS_NO_SUPPORT; - } - - if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { - FARF(ERROR, "roll: unsupported nb0 %u -> %u\n", src0->nb[0], dst->nb[0]); - return HTP_STATUS_NO_SUPPORT; + if (kparams->kernel_type == HTP_ROLL_KERNEL_DMA_CONTIGUOUS) { + return roll_dma_f32_contiguous(octx); } - - if (src0->ne[0] != ne0 || src0->ne[1] != ne1 || - src0->ne[2] != ne2 || src0->ne[3] != ne3) { - FARF(ERROR, "roll: shape mismatch\n"); - return HTP_STATUS_INVAL_PARAMS; + if (kparams->kernel_type == HTP_ROLL_KERNEL_DMA_STRIDED) { + return roll_dma_f32_strided(octx); } - const uint32_t total_rows = ne1 * ne2 * ne3; + const uint32_t total_rows = kparams->total_rows; const size_t dst_row_size = ne0 * sizeof(float); uint32_t row_start = 0; @@ -279,28 +271,24 @@ int execute_op_roll_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (octx->ctx->mdev.count <= 1) { - if (htp_tensor_is_contiguous(src0, sizeof(float)) && htp_tensor_is_contiguous(dst, sizeof(float))) { - return roll_dma_f32_contiguous(octx); - } - return roll_dma_f32_strided(octx); - } - if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t n_threads = octx->n_threads; + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_hvx = octx->n_hvx; struct htp_roll_context rctx = { .octx = octx, + .kparams = kparams, .row_start = row_start, .nrows = nrows, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), - .div_ne1 = init_fastdiv_values(dst->ne[1]), - .div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), }; - work_queue_run(octx->ctx->work_queue, roll_thread_f32, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, roll_thread_f32, &rctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.h b/ggml/src/ggml-hexagon/htp/roll-ops.h new file mode 100644 index 000000000..f98fb8ea9 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/roll-ops.h @@ -0,0 +1,30 @@ +#ifndef HTP_ROLL_OPS_H +#define HTP_ROLL_OPS_H + +#include "hex-fastdiv.h" +#include <stdint.h> + +enum htp_roll_kernel_type { + HTP_ROLL_KERNEL_DMA_CONTIGUOUS = 1, + HTP_ROLL_KERNEL_DMA_STRIDED = 2, + HTP_ROLL_KERNEL_HVX = 3, +}; + +struct htp_roll_kernel_params { + uint8_t kernel_type; + uint32_t n_hvx; + uint32_t total_rows; + uint32_t i0_src0; + uint32_t n0; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne2_ne1; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_roll_kernel_params) <= 128, "htp_roll_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_roll_kernel_params) <= 128, "htp_roll_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_ROLL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index f6b4d383c..1c73de10b 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -66,12 +66,12 @@ struct htp_rope_context { float theta_powers[32]; float corr_dims[2]; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; struct htp_ops_context * octx; uint8_t * vtcm_base; - size_t spad_per_thread; + size_t spad_per_hvx; size_t theta_cache_offset; size_t src0_row_size; @@ -532,7 +532,7 @@ static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t } } -static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { +static void rope_job_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_rope_context * rctx = (struct htp_rope_context *) data; struct htp_ops_context * octx = rctx->octx; @@ -544,10 +544,10 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_rope_preamble; const uint32_t src0_nrows = rctx->nrows; - const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = rctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows); + const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, rctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { @@ -560,12 +560,12 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); // VTCM setup - uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread); + uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_hvx); float * theta_cache = (float *) (src0_spad_base); src0_spad_base = src0_spad_base + rctx->theta_cache_offset; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const int32_t * pos = (const int32_t *) (uintptr_t) src1->data; const float * freq_factors = rctx->freq_factors; @@ -700,17 +700,8 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const struct htp_tensor * src2 = octx->src[2]; const struct htp_tensor * dst = octx->dst; - switch (octx->op) { - case HTP_OP_ROPE: - break; - - default: - FARF(ERROR, "Unsupported Op %u\n", octx->op); - return HTP_STATUS_NO_SUPPORT; - } - const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } assert(octx->ctx->vtcm_size >= kparams->vtcm_size); @@ -738,7 +729,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; const uint32_t ne0 = dst->ne[0]; const size_t src0_row_size = src0->ne[0] * sizeof(float); @@ -746,20 +737,23 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const size_t dst_row_size = dst->ne[0] * sizeof(float); const size_t dst_row_stride = dst->nb[1]; + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_hvx, src2 ? src2->ne[0] : 0); + struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); rctx.octx = octx; - rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; - rctx.spad_per_thread = kparams->spad_per_thread; - rctx.theta_cache_offset = kparams->theta_cache_offset; + rctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_spad); + rctx.spad_per_hvx = layout.bytes_per_hvx; + rctx.theta_cache_offset = layout.theta_cache_size_aligned; if (src2) { dma_queue * dma_q = octx->ctx->dma[0]; const size_t ff_size = src2->ne[0] * sizeof(float); - float * vtcm_freq_factors = (float *) (rctx.vtcm_base + kparams->freq_factors_offset); + float * vtcm_freq_factors = VTCM_LAYOUT_PTR(float, octx->ctx->vtcm_base, layout.off_freq_factors); dma_queue_push(dma_q, dma_make_data(vtcm_freq_factors, src2->data), - kparams->freq_factors_size, 0, ff_size, 1); + layout.freq_factors_size_aligned, 0, ff_size, 1); dma_queue_pop(dma_q); rctx.freq_factors = vtcm_freq_factors; } @@ -791,19 +785,19 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rctx.src0_row_stride = src0_row_stride; rctx.dst_row_size = dst_row_size; rctx.dst_row_stride = dst_row_stride; - rctx.src0_row_size_aligned = kparams->src0_row_size_aligned; + rctx.src0_row_size_aligned = layout.src0_row_size_aligned; rctx.src0_nrows = nrows; rctx.nrows = nrows; rctx.row_start = row_start; - rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + rctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); rctx.div_ne2_ne1 = kparams->div_ne2_ne1; rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, rope_job_f32, &rctx, n_hvx); return err; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h index ee055ccbc..a21b85c06 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.h +++ b/ggml/src/ggml-hexagon/htp/rope-ops.h @@ -3,21 +3,15 @@ #include "hex-common.h" #include "hex-fastdiv.h" +#include "htp-vtcm.h" #define HTP_ROPE_SPAD_BLOCK 8 #define HTP_ROPE_SPAD_NSLOTS 4 #define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) struct htp_rope_kernel_params { - uint32_t n_threads; - uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; + uint32_t n_hvx; uint32_t vtcm_size; - uint32_t spad_per_thread; - uint32_t theta_cache_offset; - uint32_t src0_row_size_aligned; - uint32_t freq_factors_offset; - uint32_t freq_factors_size; struct fastdiv_values div_ne2_ne1; struct fastdiv_values div_ne1; @@ -31,16 +25,18 @@ _Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_pa struct htp_rope_vtcm_layout { size_t total_bytes; - size_t bytes_per_thread; + size_t off_spad; + size_t off_freq_factors; + size_t bytes_per_hvx; size_t theta_cache_size_aligned; size_t src0_row_size_aligned; size_t freq_factors_size_aligned; }; static inline void htp_rope_vtcm_layout_build( - struct htp_rope_vtcm_layout * layout, + struct htp_rope_vtcm_layout * L, uint32_t ne00, - uint32_t n_threads, + uint32_t n_hvx, uint32_t n_freq_factors ) { const size_t src0_row_size = ne00 * sizeof(float); @@ -48,11 +44,15 @@ static inline void htp_rope_vtcm_layout_build( const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256); const size_t freq_factors_size_aligned = hex_round_up(n_freq_factors * sizeof(float), 256); - layout->src0_row_size_aligned = src0_row_size_aligned; - layout->theta_cache_size_aligned = theta_cache_size_aligned; - layout->freq_factors_size_aligned = freq_factors_size_aligned; - layout->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; - layout->total_bytes = layout->bytes_per_thread * n_threads + freq_factors_size_aligned; + L->src0_row_size_aligned = src0_row_size_aligned; + L->theta_cache_size_aligned = theta_cache_size_aligned; + L->freq_factors_size_aligned = freq_factors_size_aligned; + L->bytes_per_hvx = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_freq_factors, freq_factors_size_aligned, n_freq_factors > 0); + L->total_bytes = off; } static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) { diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c index 1d72538f1..add91408f 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c @@ -61,30 +61,33 @@ struct set_rows_context { uint8_t * vtcm_base; uint32_t task_start; uint32_t tasks; - uint32_t tasks_per_thread; + uint32_t tasks_per_hvx; }; #define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \ -static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ +static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(const htp_thread_t * th, unsigned int nth, \ + unsigned int ith, void *data) { \ struct set_rows_context * srctx = (struct set_rows_context *)data; \ struct htp_ops_context * octx = srctx->octx; \ const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ set_rows_preamble; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = srctx->tasks_per_thread; \ + struct htp_thread_trace * tr = th->trace; \ + const uint32_t dr = srctx->tasks_per_hvx; \ const uint32_t ir0 = srctx->task_start + dr * ith; \ if (ir0 >= srctx->task_start + srctx->tasks) { \ return; \ } \ const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + dma_queue * dma_q = th->dma_q; \ const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ - uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ - uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ + uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_src0) + \ + ith * vtcm_layout->src0_bytes_per_hvx; \ + uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_dst) + \ + ith * vtcm_layout->dst_bytes_per_hvx; \ const uint32_t src0_row_size = ne00 * sizeof(float); \ const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \ - const uint32_t nrows_per_thread = ir1 - ir0; \ - const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \ + const uint32_t nrows_per_hvx = ir1 - ir0; \ + const uint32_t total_steps = ne03 * ne02 * nrows_per_hvx; \ uint32_t pi_step = 0; \ uint32_t pi02 = 0; \ uint32_t pi03 = 0; \ @@ -100,7 +103,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig src0_data), \ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ pi_step++; \ - if (pi_step == nrows_per_thread) { \ + if (pi_step == nrows_per_hvx) { \ pi_step = 0; \ pi02++; \ if (pi02 == ne02) { \ @@ -145,7 +148,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig dma_make_data(src_spad, psrc0_data), \ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ pi_step++; \ - if (pi_step == nrows_per_thread) { \ + if (pi_step == nrows_per_hvx) { \ pi_step = 0; \ pi02++; \ if (pi02 == ne02) { \ @@ -155,7 +158,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig } \ } \ ci_step++; \ - if (ci_step == nrows_per_thread) { \ + if (ci_step == nrows_per_hvx) { \ ci_step = 0; \ ci02++; \ ci11_base++; \ @@ -188,14 +191,6 @@ int op_set_rows(struct htp_ops_context * octx) { const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params *)octx->kernel_params; set_rows_preamble; - if (octx->src[0]->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; - } - - if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_Q8_0) { - return HTP_STATUS_NO_SUPPORT; - } - if (htp_tensor_is_extended(octx->src[1])) { return HTP_STATUS_NO_SUPPORT; } @@ -217,11 +212,11 @@ int op_set_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, (uint32_t) kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // l2fetch the src1 (indices) tensor in the main thread hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]); @@ -231,9 +226,9 @@ int op_set_rows(struct htp_ops_context * octx) { srctx.kparams = kparams; srctx.task_start = task_start; srctx.tasks = tasks; - srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); + srctx.tasks_per_hvx = fastdiv(tasks + n_hvx - 1, &octx->n_hvx_div); - htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads); + htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_hvx); srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; work_queue_func_t q_func = NULL; @@ -246,15 +241,15 @@ int op_set_rows(struct htp_ops_context * octx) { default: return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n", + FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-hvx %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - srctx.vtcm_layout.src0_bytes_per_thread * n_threads, - srctx.vtcm_layout.dst_bytes_per_thread * n_threads, - n_threads); + srctx.vtcm_layout.src0_bytes_per_hvx * n_hvx, + srctx.vtcm_layout.dst_bytes_per_hvx * n_hvx, + n_hvx); - work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, q_func, &srctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.h index 5e98d2cb5..85086b08e 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h @@ -2,18 +2,13 @@ #define HTP_SET_ROWS_OPS_H #include "hex-fastdiv.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" struct htp_set_rows_kernel_params { - int32_t n_threads; - int32_t total_tasks; - int32_t tasks_per_thread; - int32_t vtcm_size; - - // Fastdiv helpers - struct fastdiv_values div_ne11; - struct fastdiv_values div_ne12; - struct fastdiv_values div_tasks_per_thread; - struct fastdiv_values div_ne02; + uint32_t n_hvx; + uint32_t total_tasks; + uint32_t vtcm_size; }; struct htp_set_rows_vtcm_layout { @@ -21,48 +16,35 @@ struct htp_set_rows_vtcm_layout { size_t off_src0; size_t off_dst; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; }; static inline void htp_set_rows_vtcm_layout_build( - struct htp_set_rows_vtcm_layout * vtcm_layout, + struct htp_set_rows_vtcm_layout * L, int dst_type, uint32_t ne00, - uint32_t n_threads) { + uint32_t n_hvx) { - size_t src0_row_size = ne00 * 4; - size_t dst_row_size = 0; - switch (dst_type) { - case 0: // HTP_TYPE_F32 - dst_row_size = ne00 * 4; - break; - case 1: // HTP_TYPE_F16 - dst_row_size = ne00 * 2; - break; - case 8: // HTP_TYPE_Q8_0 - dst_row_size = (ne00 / 32) * 34; - break; - default: - dst_row_size = 0; - break; - } + size_t src0_row_size = ne00 * sizeof(float); + size_t dst_row_size = htp_tensor_get_row_size(dst_type, ne00); size_t src0_row_size_aligned = (src0_row_size + 255) & ~255; size_t dst_row_size_aligned = (dst_row_size + 255) & ~255; - vtcm_layout->src0_spad_half_size = src0_row_size_aligned; - vtcm_layout->dst_spad_half_size = dst_row_size_aligned; + L->src0_spad_half_size = src0_row_size_aligned; + L->dst_spad_half_size = dst_row_size_aligned; - vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2; - vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2; + L->src0_bytes_per_hvx = src0_row_size_aligned * 2; + L->dst_bytes_per_hvx = dst_row_size_aligned * 2; - vtcm_layout->off_src0 = 0; - vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads; - vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads; + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + L->total_bytes = off; } #if defined(__cplusplus) diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c index 48be5d725..22aebcd5e 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.c +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c @@ -13,7 +13,7 @@ #include "hvx-utils.h" #include "hex-fastdiv.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -38,9 +38,9 @@ struct htp_softmax_context { uint8_t * vtcm_src1; uint8_t * vtcm_dst; - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_src1_size_per_thread; - uint32_t vtcm_dst_size_per_thread; + uint32_t vtcm_src0_size_per_hvx; + uint32_t vtcm_src1_size_per_hvx; + uint32_t vtcm_dst_size_per_hvx; uint32_t src0_spad_half_size; uint32_t src1_spad_half_size; @@ -65,7 +65,7 @@ struct htp_softmax_context { struct fastdiv_values div_ne12; struct fastdiv_values div_ne13; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; uint32_t nrows; @@ -287,19 +287,19 @@ static const softmax_compute_fn_t softmax_kernels[HTP_SOFTMAX_KERNEL_COUNT] = { [HTP_SOFTMAX_KERNEL_MASK_F16] = compute_fast_softmax_f32_mask_f16, }; -static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) { +static void softmax_thread_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; struct htp_ops_context * octx = smctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t src0_nrows = smctx->nrows; - const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = smctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, smctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -311,13 +311,13 @@ static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) const size_t src0_row_size = src0->ne[0] * sizeof(float); const size_t dst_row_size = src0->ne[0] * sizeof(float); - uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half = smctx->src0_spad_half_size; const size_t dst_vtcm_half = smctx->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { dma_addr_t cur_dst = data_dst + r * dst_row_size; @@ -357,20 +357,20 @@ static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) dma_queue_flush(dma_q); } -static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * data) { +static void softmax_thread_mask_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; struct htp_ops_context * octx = smctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t src0_nrows = smctx->nrows; - const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = smctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, smctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -384,9 +384,9 @@ static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * d const size_t dst_row_size = src0->ne[0] * sizeof(float); const size_t mask_row_size = smctx->use_f16 ? (src1->ne[0] * sizeof(__fp16)) : (src1->ne[0] * sizeof(float)); - uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); - uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_thread); - uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_hvx); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half = smctx->src0_spad_half_size; const size_t src1_vtcm_half = smctx->src1_spad_half_size; @@ -408,7 +408,7 @@ static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * d const struct fastdiv_values * div_ne12 = &smctx->div_ne12; const struct fastdiv_values * div_ne13 = &smctx->div_ne13; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { dma_addr_t cur_dst = data_dst + r * dst_row_size; @@ -492,11 +492,11 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { const struct htp_softmax_kernel_params * kparams = (const struct htp_softmax_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - if (kparams->kernel_id >= HTP_SOFTMAX_KERNEL_COUNT) { + if (kparams->kernel_type >= HTP_SOFTMAX_KERNEL_COUNT) { return HTP_STATUS_INVAL_PARAMS; } @@ -520,8 +520,8 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); row_start = range.start; nrows = range.count; - if (nrows < octx->n_threads) { - htp_ops_context_set_n_threads(octx, nrows ? nrows : 1); + if (nrows < octx->n_hvx) { + htp_ops_context_set_n_hvx(octx, nrows ? nrows : 1); } } @@ -529,37 +529,41 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base; - const uint32_t off_src0 = 0; - const uint32_t off_dst = off_src0 + kparams->vtcm_src0_size_per_thread * kparams->n_threads; - const uint32_t off_src1 = off_dst + kparams->vtcm_dst_size_per_thread * kparams->n_threads; + struct htp_softmax_vtcm_layout layout; + htp_softmax_vtcm_layout_build(&layout, + src0->ne[0], + kparams->use_src1 ? octx->src[1]->ne[0] : 0, + kparams->use_src1 != 0, + kparams->use_f16 != 0, + kparams->n_hvx); struct htp_softmax_context smctx = { .octx = octx, .kparams = kparams, - .compute = (void *) softmax_kernels[kparams->kernel_id], + .compute = (void *) softmax_kernels[kparams->kernel_type], .data_src0 = src0->data, .data_src1 = kparams->use_src1 ? octx->src[1]->data : 0, .data_dst = dst->data, - .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, off_src0), - .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, off_dst), - .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, off_src1, kparams->use_src1), + .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_src0), + .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_dst), + .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, layout.off_src1, kparams->use_src1), - .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread, - .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread, - .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread, + .vtcm_src0_size_per_hvx = (uint32_t) layout.src0_bytes_per_hvx, + .vtcm_src1_size_per_hvx = (uint32_t) layout.src1_bytes_per_hvx, + .vtcm_dst_size_per_hvx = (uint32_t) layout.dst_bytes_per_hvx, - .src0_spad_half_size = kparams->src0_spad_half_size, - .src1_spad_half_size = kparams->src1_spad_half_size, - .dst_spad_half_size = kparams->dst_spad_half_size, + .src0_spad_half_size = (uint32_t) layout.src0_spad_half_size, + .src1_spad_half_size = (uint32_t) layout.src1_spad_half_size, + .dst_spad_half_size = (uint32_t) layout.dst_spad_half_size, - .src0_row_size_aligned = kparams->src0_row_size_aligned, - .src1_row_size_aligned = kparams->src1_row_size_aligned, - .dst_row_size_aligned = kparams->dst_row_size_aligned, + .src0_row_size_aligned = (uint32_t) layout.src0_spad_half_size, + .src1_row_size_aligned = (uint32_t) layout.src1_spad_half_size, + .dst_row_size_aligned = (uint32_t) layout.dst_spad_half_size, .use_f16 = kparams->use_f16 != 0, @@ -576,7 +580,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { .div_ne12 = kparams->div_ne12, .div_ne13 = kparams->div_ne13, - .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .row_start = row_start, .nrows = nrows, }; @@ -592,7 +596,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { } work_queue_func_t task_func = kparams->use_src1 ? softmax_thread_mask_dma : softmax_thread_dma; - work_queue_run(octx->ctx->work_queue, task_func, &smctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, task_func, &smctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.h b/ggml/src/ggml-hexagon/htp/softmax-ops.h index 8d976adb8..9e5f9c759 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.h +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.h @@ -7,8 +7,9 @@ #include <math.h> #include "hex-fastdiv.h" #include "hex-common.h" +#include "htp-vtcm.h" -enum htp_softmax_kernel_id { +enum htp_softmax_kernel_type { HTP_SOFTMAX_KERNEL_NOMASK = 0, HTP_SOFTMAX_KERNEL_MASK_F32, HTP_SOFTMAX_KERNEL_MASK_F16, @@ -16,28 +17,14 @@ enum htp_softmax_kernel_id { }; struct htp_softmax_kernel_params { - uint32_t n_threads; - uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; + uint8_t kernel_type; + uint32_t n_hvx; uint32_t vtcm_size; - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_src1_size_per_thread; - uint32_t vtcm_dst_size_per_thread; - - uint32_t src0_row_size_aligned; - uint32_t src1_row_size_aligned; - uint32_t dst_row_size_aligned; - - uint32_t src0_spad_half_size; - uint32_t src1_spad_half_size; - uint32_t dst_spad_half_size; - uint32_t n_head; uint32_t n_head_log2; uint32_t use_src1; uint32_t use_f16; - uint32_t kernel_id; float scale; float max_bias; @@ -62,9 +49,9 @@ struct htp_softmax_vtcm_layout { size_t off_dst; size_t off_src1; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; - size_t src1_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; + size_t src1_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; @@ -72,12 +59,12 @@ struct htp_softmax_vtcm_layout { }; static inline void htp_softmax_vtcm_layout_build( - struct htp_softmax_vtcm_layout * layout, + struct htp_softmax_vtcm_layout * L, uint32_t ne00, uint32_t ne10, bool use_src1, bool use_f16, - uint32_t n_threads + uint32_t n_hvx ) { size_t src0_row_size = ne00 * sizeof(float); size_t dst_row_size = ne00 * sizeof(float); @@ -87,20 +74,21 @@ static inline void htp_softmax_vtcm_layout_build( size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128); size_t src1_row_size_aligned = use_src1 ? hex_round_up(src1_row_size, 128) : 0; - layout->src0_spad_half_size = src0_row_size_aligned; - layout->dst_spad_half_size = dst_row_size_aligned; - layout->src1_spad_half_size = src1_row_size_aligned; + L->src0_spad_half_size = src0_row_size_aligned; + L->dst_spad_half_size = dst_row_size_aligned; + L->src1_spad_half_size = src1_row_size_aligned; // Double buffering: 2 half-buffers per thread - layout->src0_bytes_per_thread = src0_row_size_aligned * 2; - layout->dst_bytes_per_thread = dst_row_size_aligned * 2; - layout->src1_bytes_per_thread = src1_row_size_aligned * 2; + L->src0_bytes_per_hvx = src0_row_size_aligned * 2; + L->dst_bytes_per_hvx = dst_row_size_aligned * 2; + L->src1_bytes_per_hvx = src1_row_size_aligned * 2; - layout->off_src0 = 0; - layout->off_dst = layout->off_src0 + layout->src0_bytes_per_thread * n_threads; - layout->off_src1 = layout->off_dst + layout->dst_bytes_per_thread * n_threads; + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_hvx * n_hvx, use_src1); - layout->total_bytes = layout->off_src1 + layout->src1_bytes_per_thread * n_threads; + L->total_bytes = off; } #endif // HTP_SOFTMAX_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c index 182982fcd..73eff927e 100644 --- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c @@ -4,7 +4,7 @@ #include <string.h> #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -13,14 +13,16 @@ #include "htp-tensor.h" #include "hvx-types.h" #include "hvx-utils.h" +#include "solve-tri-ops.h" struct htp_solve_tri_context { - struct htp_ops_context * octx; - uint32_t jobs_per_thread; - uint32_t total_jobs; - uint32_t job_start; - uint32_t k_chunks; - uint32_t col_block; + struct htp_ops_context * octx; + const struct htp_solve_tri_kernel_params * kparams; + uint32_t jobs_per_hvx; + uint32_t total_jobs; + uint32_t job_start; + uint32_t k_chunks; + uint32_t col_block; }; static inline void solve_tri_row_scalar(const float * A_row, @@ -77,9 +79,10 @@ static inline void solve_tri_row_hvx(const float * A_row, } // Batch-level thread: each job is one full batch. -static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void solve_tri_batch_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data; struct htp_ops_context * octx = sctx->octx; + const struct htp_solve_tri_kernel_params * kparams = sctx->kparams; const struct htp_tensor * src0 = octx->src[0]; // A const struct htp_tensor * src1 = octx->src[1]; // B @@ -91,16 +94,16 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t ne02 = src0->ne[2]; const uint32_t col_block = VLEN_FP32; - const uint32_t k_full = (k / col_block) * col_block; + const uint32_t k_full = k & ~(col_block - 1); - const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith; - const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); + const uint32_t start_batch = sctx->job_start + sctx->jobs_per_hvx * ith; + const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_hvx, sctx->job_start + sctx->total_jobs); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch); for (uint32_t batch = start_batch; batch < end_batch; ++batch) { - const uint32_t i03 = batch / ne02; + const uint32_t i03 = fastdiv(batch, &kparams->div_ne02); const uint32_t i02 = batch - i03 * ne02; const float * A_batch = @@ -138,9 +141,10 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void } // Chunk-level thread: each job is one (batch, col_chunk) pair. -static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void solve_tri_chunk_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data; struct htp_ops_context * octx = sctx->octx; + const struct htp_solve_tri_kernel_params * kparams = sctx->kparams; const struct htp_tensor * src0 = octx->src[0]; // A const struct htp_tensor * src1 = octx->src[1]; // B @@ -151,17 +155,17 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t ne02 = src0->ne[2]; - const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith; - const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); + const uint32_t start_job = sctx->job_start + sctx->jobs_per_hvx * ith; + const uint32_t end_job = MIN(start_job + sctx->jobs_per_hvx, sctx->job_start + sctx->total_jobs); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job); for (uint32_t job = start_job; job < end_job; ++job) { - const uint32_t batch = job / sctx->k_chunks; + const uint32_t batch = fastdiv(job, &kparams->div_k_chunks); const uint32_t chunk = job - batch * sctx->k_chunks; - const uint32_t i03 = batch / ne02; + const uint32_t i03 = fastdiv(batch, &kparams->div_ne02); const uint32_t i02 = batch - i03 * ne02; const float * A_batch = @@ -198,36 +202,17 @@ int op_solve_tri(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; // A const struct htp_tensor * src1 = octx->src[1]; // B const struct htp_tensor * dst = octx->dst; // X - - if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; - } - - // left=true, lower=true, uni=false only - if (src0->ne[0] != src0->ne[1]) { - return HTP_STATUS_INVAL_PARAMS; - } - if (src0->ne[1] != src1->ne[1]) { - return HTP_STATUS_INVAL_PARAMS; - } - if (src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3]) { - return HTP_STATUS_INVAL_PARAMS; - } - if (dst->ne[0] != src1->ne[0] || dst->ne[1] != src1->ne[1] || dst->ne[2] != src1->ne[2] || - dst->ne[3] != src1->ne[3]) { - return HTP_STATUS_INVAL_PARAMS; - } + const struct htp_solve_tri_kernel_params * kparams = + (const struct htp_solve_tri_kernel_params *) octx->kernel_params; if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t k = src1->ne[0]; - - const uint32_t col_block = VLEN_FP32; - const uint32_t k_chunks = (k + col_block - 1) / col_block; - const uint32_t total_batches = src0->ne[2] * src0->ne[3]; - const bool batched = total_batches >= (uint32_t) octx->n_threads; + const uint32_t col_block = kparams->col_block; + const uint32_t k_chunks = kparams->k_chunks; + const uint32_t total_batches = kparams->total_batches; + const bool batched = (kparams->batched != 0); FARF(HIGH, "solve-tri: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : batched %d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], @@ -239,9 +224,7 @@ int op_solve_tri(struct htp_ops_context * octx) { uint32_t njobs = total_batches; if (octx->ctx->mdev.count > 1) { - const uint32_t batch_size = dst->nb[2]; - const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1; - const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); job_start = range.start; njobs = range.count; } @@ -250,22 +233,25 @@ int op_solve_tri(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - // Batch-level parallelism - const uint32_t n_threads = octx->n_threads; + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, njobs))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_hvx = octx->n_hvx; struct htp_solve_tri_context sctx = { .octx = octx, - .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .kparams = kparams, + .jobs_per_hvx = fastdiv(njobs + n_hvx - 1, &octx->n_hvx_div), .total_jobs = njobs, .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, solve_tri_batch_thread_f32, &sctx, n_hvx); } else { - // Chunk-level parallelism - const uint32_t total_jobs = total_batches * k_chunks; + const uint32_t total_jobs = kparams->total_jobs; uint32_t job_start = 0; uint32_t njobs = total_jobs; @@ -281,18 +267,23 @@ int op_solve_tri(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, njobs))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_hvx = octx->n_hvx; struct htp_solve_tri_context sctx = { .octx = octx, - .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .kparams = kparams, + .jobs_per_hvx = fastdiv(njobs + n_hvx - 1, &octx->n_hvx_div), .total_jobs = njobs, .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, solve_tri_chunk_thread_f32, &sctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h new file mode 100644 index 000000000..6a46cfdc4 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h @@ -0,0 +1,26 @@ +#ifndef HTP_SOLVE_TRI_OPS_H +#define HTP_SOLVE_TRI_OPS_H + +#include "hex-fastdiv.h" +#include <stdint.h> + +struct htp_solve_tri_kernel_params { + uint32_t n_hvx; + uint32_t total_batches; + uint32_t total_jobs; + uint32_t k_chunks; + uint32_t col_block; + uint32_t batched; + uint32_t batches_per_chunk; + + struct fastdiv_values div_ne02; + struct fastdiv_values div_k_chunks; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_solve_tri_kernel_params) <= 128, "htp_solve_tri_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_solve_tri_kernel_params) <= 128, "htp_solve_tri_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_SOLVE_TRI_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c index 931aa406e..d75304fef 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.c +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c @@ -15,7 +15,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" #include "hvx-utils.h" @@ -24,7 +24,7 @@ struct htp_ssm_conv_context { struct htp_ops_context * octx; const struct htp_ssm_conv_kernel_params * kparams; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t d_inner_tile; uint32_t row_start; uint32_t nrows; @@ -80,8 +80,8 @@ static inline void hvx_transpose_32x32_f32(HVX_Vector m[32]) { } // HVX deinterleave for d_conv == 4: channel-major raw VTCM -> tap-major T VTCM -static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride) { - for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { +static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride) { + for (uint32_t cb = 0; cb < d_inner_per_hvx; cb += VLEN_FP32) { HVX_Vector v0 = *(const HVX_Vector *)(raw + (cb + 0) * 4); HVX_Vector v1 = *(const HVX_Vector *)(raw + (cb + 8) * 4); HVX_Vector v2 = *(const HVX_Vector *)(raw + (cb + 16) * 4); @@ -101,15 +101,15 @@ static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint } // HVX transpose for general d_conv <= 32: channel-major raw VTCM -> tap-major T VTCM -static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { +static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride, uint32_t d_conv) { uint32_t __attribute__((aligned(VLEN))) mask_buf[VLEN_FP32] = { 0 }; for (uint32_t j = 0; j < d_conv; ++j) { mask_buf[j] = 0xFFFFFFFF; } const HVX_Vector mask = *(const HVX_Vector *) mask_buf; - for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { - const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_thread - cb); + for (uint32_t cb = 0; cb < d_inner_per_hvx; cb += VLEN_FP32) { + const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_hvx - cb); HVX_Vector sub[32]; for (uint32_t r = 0; r < cb_n; ++r) { const float * ch_ptr = raw + (cb + r) * d_conv; @@ -127,57 +127,66 @@ static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, ui } } -static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { - if (d_conv == 4 && (d_inner_per_thread % VLEN_FP32 == 0)) { - hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_thread, d_inner_stride); +static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride, uint32_t d_conv) { + if (d_conv == 4 && (d_inner_per_hvx % VLEN_FP32 == 0)) { + hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_hvx, d_inner_stride); } else { - hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_thread, d_inner_stride, d_conv); + hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_hvx, d_inner_stride, d_conv); } } -// HVX 32x32 src0 transpose for prefill: src0 {tile_n, ncs} (VTCM) -> src0_T {ncs, d_inner_tile} (VTCM) -static inline void transpose_src0_block(const float * src0_block, - uint32_t ncs, - uint32_t cb_n, - uint32_t d_inner_tile, - float * src0_T_block_dst, - uint32_t cb) { - const uint32_t T_TILE = VLEN_FP32; +static __attribute__((noinline)) void hvx_ssm_conv_silu_vtcm(float * restrict buf, uint32_t n_vecs) { + HVX_Vector * restrict vbuf = (HVX_Vector *) buf; + for (uint32_t i = 0; i < n_vecs; ++i) { + vbuf[i] = hvx_vec_silu_f32(vbuf[i]); + } +} - HVX_Vector __attribute__((aligned(VLEN))) sub[32]; +// Decode dot product specialization for d_conv == 4: multiply in the raw channel-major layout, +// then deinterleave the products so each vector holds one tap of 32 channels, and sum. +// Keeps both operands in DMA layout - no transpose, no scratch. +static inline void hvx_ssm_conv_decode_4(const float * x, const float * w, float * out, uint32_t n_ch) { + for (uint32_t cb = 0; cb < n_ch; cb += VLEN_FP32) { + const float * xp = x + cb * 4; + const float * wp = w + cb * 4; - for (uint32_t t0 = 0; t0 < ncs; t0 += T_TILE) { - const uint32_t t_n = MIN(T_TILE, ncs - t0); + HVX_Vector p0 = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *)(xp + 0), *(const HVX_Vector *)(wp + 0)); + HVX_Vector p1 = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *)(xp + 32), *(const HVX_Vector *)(wp + 32)); + HVX_Vector p2 = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *)(xp + 64), *(const HVX_Vector *)(wp + 64)); + HVX_Vector p3 = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *)(xp + 96), *(const HVX_Vector *)(wp + 96)); - uint32_t __attribute__((aligned(VLEN))) mask_buf[VLEN_FP32] = { 0 }; - for (uint32_t k = 0; k < t_n; ++k) { - mask_buf[k] = 0xFFFFFFFF; - } - const HVX_Vector mask = *(const HVX_Vector *) mask_buf; + HVX_VectorPair p01 = Q6_W_vdeal_VVR(p1, p0, -4); + HVX_VectorPair p23 = Q6_W_vdeal_VVR(p3, p2, -4); - for (uint32_t r = 0; r < cb_n; ++r) { - const float * src_row = src0_block + r * ncs + t0; - sub[r] = (t_n == T_TILE) ? *(const HVX_UVector *) src_row : Q6_V_vand_VV(*(const HVX_UVector *) src_row, mask); - } - for (uint32_t r = cb_n; r < T_TILE; ++r) { - sub[r] = hvx_vec_splat_f32(0.0f); - } + HVX_VectorPair q02 = Q6_W_vdeal_VVR(Q6_V_lo_W(p23), Q6_V_lo_W(p01), -4); + HVX_VectorPair q13 = Q6_W_vdeal_VVR(Q6_V_hi_W(p23), Q6_V_hi_W(p01), -4); - hvx_transpose_32x32_f32(sub); + HVX_Vector a = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(q02), Q6_V_lo_W(q13)); + HVX_Vector b = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_hi_W(q02), Q6_V_hi_W(q13)); - for (uint32_t r = 0; r < t_n; ++r) { - float * dst = src0_T_block_dst + (t0 + r) * d_inner_tile + cb; - if (cb_n == T_TILE) { - *(HVX_UVector *) dst = sub[r]; - } else { - hvx_vec_store_u(dst, cb_n * sizeof(float), sub[r]); - } - } + *(HVX_Vector *)(out + cb) = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(a, b)); } } -// Single-row decode worker (n_t == 1) -static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void * data) { +// Transpose src0 for prefill: one 32-channel block {32, ncs} (VTCM) -> T {ncs, 32} (VTCM). +// One VTCM gather per output row: lane r picks channel cb+r, the region bound drops +// the lanes past the channel tail. +static inline void hvx_ssm_conv_transpose_block(const float * raw_block, + float * T, + uint32_t raw_stride_bytes, + uint32_t ncs, + uint32_t cb_n, + HVX_Vector vv) { + const size_t base = (size_t) raw_block; + const uint32_t mu = cb_n * raw_stride_bytes - 1; + + for (uint32_t t = 0; t < ncs; ++t) { + Q6_vgather_ARMVw((HVX_Vector *) (T + (size_t) t * VLEN_FP32), base + t * sizeof(float), mu, vv); + } +} + +// Single-token decode worker (n_t == 1) +static void ssm_conv_thread_f32_decode(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; struct htp_ops_context * octx = scctx->octx; const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; @@ -186,13 +195,14 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void const struct htp_tensor * restrict src1 = octx->src[1]; const struct htp_tensor * restrict dst = octx->dst; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; - const uint32_t d_conv = kparams->d_conv; - const uint32_t d_inner = kparams->d_inner; - const uint32_t n_s = kparams->n_s; + const uint32_t d_conv = kparams->d_conv; + const uint32_t d_inner = kparams->d_inner; + const uint32_t n_s = kparams->n_s; + const bool apply_silu = kparams->silu != 0; - const uint32_t dr = scctx->nrows_per_thread; + const uint32_t dr = scctx->nrows_per_hvx; const uint32_t ir0 = scctx->row_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); @@ -200,73 +210,136 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void return; } - const uint32_t d_inner_per_thread = ir1 - ir0; - const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + const uint32_t d_inner_per_hvx = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_hvx, VLEN_FP32); + const uint32_t d_inner_tile = scctx->d_inner_tile; - const size_t src0_stride_seq_bytes = src0->nb[2]; - const size_t dst_stride_seq_bytes = dst->nb[2]; + const size_t src0_stride_inner_bytes = src0->nb[1]; + const size_t src0_stride_seq_bytes = src0->nb[2]; + const size_t dst_stride_seq_bytes = dst->nb[2]; - uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; - uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_hvx; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; - const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_bytes = (size_t) d_inner_per_hvx * d_conv * sizeof(float); const size_t weight_raw_size = hex_round_up(weight_bytes, 128); float * src1_raw = (float *) src1_spad_base; float * src1_T = (float *) (src1_spad_base + weight_raw_size); - float * src0_raw = (float *) src0_spad_base; - float * src0_T = (float *) (src0_spad_base + weight_raw_size); + const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * d_conv * sizeof(float), 128); + const size_t dst_tile_bytes = hex_round_up(d_inner_tile * sizeof(float), 128); + + float * src0_tile_raw[2] = { (float *) src0_spad_base, (float *) (src0_spad_base + src0_tile_raw_bytes) }; + float * src0_T = (float *) (src0_spad_base + 2 * src0_tile_raw_bytes); + + float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) }; - float * dst_spad = (float *) dst_spad_base; + struct htp_thread_trace * tr = th->trace; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + // raw_dot keeps both operands in the DMA layout, so src1 needs no prep pass + const bool raw_dot = (d_conv == 4) && (d_inner_per_hvx % VLEN_FP32 == 0); - // 1. Fetch weights src1 from DDR into VTCM via DMA (DMA64-safe) + const uint32_t n_tiles = fastdiv(d_inner_per_hvx + d_inner_tile - 1, &kparams->div_d_inner_tile); + const uint32_t n_chunks = n_s * n_tiles; + const size_t row_bytes = d_conv * sizeof(float); + + uint32_t s_fetch = 0; + uint32_t tile_off_fetch = 0; + + #define SSM_CONV_DECODE_PUSH_FETCH(c) \ + do { \ + const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off_fetch); \ + const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \ + (ir0 + tile_off_fetch) * src0_stride_inner_bytes; \ + dma_queue_push(dma_q, \ + dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \ + row_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \ + tile_off_fetch += d_inner_tile; \ + if (tile_off_fetch >= d_inner_per_hvx) { \ + tile_off_fetch = 0; \ + s_fetch++; \ + } \ + } while (0) + + // Queue weights and initial input tiles together so DDR reads overlap const dma_addr_t src1_ddr = src1->data + ir0 * d_conv * sizeof(float); dma_queue_push(dma_q, dma_make_data((uint8_t *) src1_raw, src1_ddr), weight_bytes, weight_bytes, weight_bytes, 1); - dma_queue_pop(dma_q); - - // 2. Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); - hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); - - const size_t input_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); - const size_t output_bytes = (size_t) d_inner_per_thread * sizeof(float); - - // 3. Process each sequence - for (uint32_t s = 0; s < n_s; ++s) { - const dma_addr_t src0_ddr = src0->data + s * src0_stride_seq_bytes + ir0 * d_conv * sizeof(float); - dma_queue_push(dma_q, dma_make_data((uint8_t *) src0_raw, src0_ddr), input_bytes, input_bytes, input_bytes, 1); - dma_queue_pop(dma_q); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) s); - hvx_ssm_conv_unpack_to_T(src0_raw, src0_T, d_inner_per_thread, d_inner_stride, d_conv); - - for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { - const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_thread - cb); - HVX_Vector acc = hvx_vec_splat_f32(0.0f); - for (uint32_t j = 0; j < d_conv; ++j) { - HVX_Vector x = *(const HVX_Vector *)(src0_T + j * d_inner_stride + cb); - HVX_Vector w = *(const HVX_Vector *)(src1_T + j * d_inner_stride + cb); - acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); - } - HVX_Vector y = Q6_Vsf_equals_Vqf32(acc); - if (cb_n == VLEN_FP32) { - *(HVX_Vector *)(dst_spad + cb) = y; - } else { - hvx_vec_store_u(dst_spad + cb, cb_n * sizeof(float), y); + + SSM_CONV_DECODE_PUSH_FETCH(0); + if (n_chunks > 1) { + SSM_CONV_DECODE_PUSH_FETCH(1); + } + + dma_queue_pop(dma_q); // weights + + if (!raw_dot) { + // Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_hvx, d_inner_stride, d_conv); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); + } + + uint32_t i3 = 0; + uint32_t tile_off = 0; + + for (uint32_t c = 0; c < n_chunks; ++c) { + const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off); + + if (c >= 2) { + dma_queue_pop(dma_q); // writeback of chunk c-2, frees dst_tile[c & 1] + } + dma_queue_pop(dma_q); // fetch chunk c + + float * restrict out = dst_tile[c & 1]; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i3); + if (raw_dot) { + const float * xp = src0_tile_raw[c & 1]; + const float * wp = src1_raw + tile_off * 4; + hvx_ssm_conv_decode_4(xp, wp, out, tile_n); + } else { + const uint32_t tile_stride = hex_round_up(tile_n, VLEN_FP32); + hvx_ssm_conv_unpack_to_T(src0_tile_raw[c & 1], src0_T, tile_n, tile_stride, d_conv); + + for (uint32_t cb = 0; cb < tile_n; cb += VLEN_FP32) { + HVX_Vector acc = hvx_vec_splat_f32(0.0f); + for (uint32_t j = 0; j < d_conv; ++j) { + HVX_Vector x = *(const HVX_Vector *)(src0_T + j * tile_stride + cb); + HVX_Vector w = *(const HVX_Vector *)(src1_T + j * d_inner_stride + tile_off + cb); + acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); + } + *(HVX_Vector *)(out + cb) = Q6_Vsf_equals_Vqf32(acc); } } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) s); + if (apply_silu) { + hvx_ssm_conv_silu_vtcm(out, hex_round_up(tile_n, VLEN_FP32) / VLEN_FP32); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i3); + + const dma_addr_t dst_ddr = dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float); + const size_t tile_out_bytes = (size_t) tile_n * sizeof(float); + dma_queue_push(dma_q, dma_make_data(dst_ddr, (uint8_t *) out), + tile_out_bytes, tile_out_bytes, tile_out_bytes, 1); + + if (c + 2 < n_chunks) { + SSM_CONV_DECODE_PUSH_FETCH(c + 2); + } + + tile_off += d_inner_tile; + if (tile_off >= d_inner_per_hvx) { + tile_off = 0; + i3++; + } + } - const dma_addr_t dst_ddr = dst->data + s * dst_stride_seq_bytes + ir0 * sizeof(float); - dma_queue_push(dma_q, dma_make_data(dst_ddr, (uint8_t *) dst_spad), output_bytes, output_bytes, output_bytes, 1); - dma_queue_pop(dma_q); + for (uint32_t k = MIN(n_chunks, 2); k > 0; --k) { + dma_queue_pop(dma_q); // drain the last writebacks } + #undef SSM_CONV_DECODE_PUSH_FETCH + FARF(HIGH, "ssm-conv-f32-decode %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], @@ -274,7 +347,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void } // Multi-token prefill worker (n_t > 1) -static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void * data) { +static void ssm_conv_thread_f32_prefill(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; struct htp_ops_context * octx = scctx->octx; const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; @@ -283,15 +356,16 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void const struct htp_tensor * restrict src1 = octx->src[1]; const struct htp_tensor * restrict dst = octx->dst; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; - const uint32_t d_conv = kparams->d_conv; - const uint32_t d_inner = kparams->d_inner; - const uint32_t n_t = kparams->n_t; - const uint32_t n_s = kparams->n_s; - const uint32_t ncs = src0->ne[0]; + const uint32_t d_conv = kparams->d_conv; + const uint32_t d_inner = kparams->d_inner; + const uint32_t n_t = kparams->n_t; + const uint32_t n_s = kparams->n_s; + const uint32_t ncs = src0->ne[0]; + const bool apply_silu = kparams->silu != 0; - const uint32_t dr = scctx->nrows_per_thread; + const uint32_t dr = scctx->nrows_per_hvx; const uint32_t ir0 = scctx->row_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); @@ -299,8 +373,8 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void return; } - const uint32_t d_inner_per_thread = ir1 - ir0; - const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + const uint32_t d_inner_per_hvx = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_hvx, VLEN_FP32); const uint32_t d_inner_tile = scctx->d_inner_tile; const size_t src0_stride_inner_bytes = src0->nb[1]; @@ -308,92 +382,169 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void const size_t dst_stride_token_bytes = dst->nb[1]; const size_t dst_stride_seq_bytes = dst->nb[2]; - uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; - uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_hvx; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; - const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_bytes = (size_t) d_inner_per_hvx * d_conv * sizeof(float); const size_t weight_raw_size = hex_round_up(weight_bytes, 128); float * src1_raw = (float *) src1_spad_base; float * src1_T = (float *) (src1_spad_base + weight_raw_size); - const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * ncs * sizeof(float), 128); - float * src0_tile_raw = (float *) src0_spad_base; - float * src0_T = (float *) (src0_spad_base + src0_tile_raw_bytes); - - float * dst_tile = (float *) dst_spad_base; - - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - - // 1. Fetch weights src1 from DDR into VTCM via DMA (DMA64-safe) + // src0 spad holds two raw tiles (fetch of tile n+1 overlaps compute of tile n) plus + // the transposed block. dst spad holds two tiles so a writeback can stay in flight. + const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float); + const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * raw_stride_bytes, 128); + const size_t dst_tile_bytes = hex_round_up(d_inner_tile * n_t * sizeof(float), 128); + + float * src0_tile_raw[2] = { (float *) src0_spad_base, (float *) (src0_spad_base + src0_tile_raw_bytes) }; + float * src0_T = (float *) (src0_spad_base + 2 * src0_tile_raw_bytes); + + float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) }; + + struct htp_thread_trace * tr = th->trace; + + const uint32_t n_tiles = fastdiv(d_inner_per_hvx + d_inner_tile - 1, &kparams->div_d_inner_tile); + const uint32_t n_chunks = n_s * n_tiles; + const size_t row_bytes = ncs * sizeof(float); + + uint32_t s_fetch = 0; + uint32_t tile_off_fetch = 0; + + // Chunk c fetches into src0_tile_raw[c & 1] and writes back from dst_tile[c & 1]. + // Two fetches run ahead, so the queue order is F0 F1 W0 F2 W1 ... and pops follow it. + #define SSM_CONV_PUSH_FETCH(c) \ + do { \ + const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off_fetch); \ + const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \ + (ir0 + tile_off_fetch) * src0_stride_inner_bytes; \ + dma_queue_push(dma_q, \ + dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \ + raw_stride_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \ + tile_off_fetch += d_inner_tile; \ + if (tile_off_fetch >= d_inner_per_hvx) { \ + tile_off_fetch = 0; \ + s_fetch++; \ + } \ + } while (0) + + // Queue weights and initial input tiles together so DDR reads overlap const dma_addr_t src1_ddr = src1->data + ir0 * d_conv * sizeof(float); dma_queue_push(dma_q, dma_make_data((uint8_t *) src1_raw, src1_ddr), weight_bytes, weight_bytes, weight_bytes, 1); - dma_queue_pop(dma_q); - // 2. Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); - hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + SSM_CONV_PUSH_FETCH(0); + if (n_chunks > 1) { + SSM_CONV_PUSH_FETCH(1); + } + + dma_queue_pop(dma_q); // weights + + // Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_hvx, d_inner_stride, d_conv); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); const uint32_t C_TILE = VLEN_FP32; - for (uint32_t i3 = 0; i3 < n_s; ++i3) { - for (uint32_t tile_off = 0; tile_off < d_inner_per_thread; tile_off += d_inner_tile) { - const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off); + // gather offsets: lane r reads channel r of the raw tile block + const HVX_Vector vv = hvx_vec_gather_offsets_w(raw_stride_bytes); + + uint32_t i3 = 0; + uint32_t tile_off = 0; - // Fetch src0 chunk from DDR to VTCM via 2D DMA - const dma_addr_t src0_tile_ddr = src0->data + - i3 * src0_stride_seq_bytes + - (ir0 + tile_off) * src0_stride_inner_bytes; - const size_t row_bytes = ncs * sizeof(float); + for (uint32_t c = 0; c < n_chunks; ++c) { + const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off); - dma_queue_push(dma_q, dma_make_data((uint8_t *) src0_tile_raw, src0_tile_ddr), - row_bytes, src0_stride_inner_bytes, row_bytes, tile_n); - dma_queue_pop(dma_q); + const float * restrict raw = src0_tile_raw[c & 1]; + float * restrict out = dst_tile[c & 1]; + + if (c >= 2) { + dma_queue_pop(dma_q); // writeback of chunk c-2, frees dst_tile[c & 1] + } + dma_queue_pop(dma_q); // fetch chunk c + + // Channel block outer, token inner: the taps and the sliding window stay in + // registers, so a new output row costs one src0_T load. + const uint32_t dst_tile_stride = hex_round_up(tile_n, C_TILE); + + for (uint32_t cb = 0; cb < tile_n; cb += C_TILE) { + const uint32_t cb_n = MIN(C_TILE, tile_n - cb); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) tile_off); + const float * raw_ch = (const float *) ((const uint8_t *) raw + (size_t) cb * raw_stride_bytes); + hvx_ssm_conv_transpose_block(raw_ch, src0_T, raw_stride_bytes, ncs, cb_n, vv); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) tile_off); + + const float * restrict wp = src1_T + tile_off + cb; + float * restrict op = out + cb; - // Transpose src0 chunk in VTCM into {d_inner_tile, ncs} layout htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); - for (uint32_t cb = 0; cb < tile_n; cb += C_TILE) { - const uint32_t cb_n = MIN(C_TILE, tile_n - cb); - transpose_src0_block(src0_tile_raw + cb * ncs, ncs, cb_n, d_inner_tile, src0_T, cb); - } + if (d_conv == 4) { + const HVX_Vector w0 = *(const HVX_Vector *) (wp); + const HVX_Vector w1 = *(const HVX_Vector *) (wp + d_inner_stride); + const HVX_Vector w2 = *(const HVX_Vector *) (wp + 2 * d_inner_stride); + const HVX_Vector w3 = *(const HVX_Vector *) (wp + 3 * d_inner_stride); + + HVX_Vector x0 = *(const HVX_Vector *) (src0_T); + HVX_Vector x1 = *(const HVX_Vector *) (src0_T + C_TILE); + HVX_Vector x2 = *(const HVX_Vector *) (src0_T + 2 * C_TILE); + + for (uint32_t t = 0; t < n_t; ++t) { + const HVX_Vector x3 = *(const HVX_Vector *) (src0_T + (t + 3) * C_TILE); + + HVX_Vector a = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(x0, w0), + Q6_Vqf32_vmpy_VsfVsf(x1, w1)); + HVX_Vector b = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(x2, w2), + Q6_Vqf32_vmpy_VsfVsf(x3, w3)); - // Compute convolution - for (uint32_t t = 0; t < n_t; ++t) { - for (uint32_t cb = 0; cb < tile_n; cb += C_TILE) { - const uint32_t cb_n = MIN(C_TILE, tile_n - cb); + *(HVX_Vector *) (op + t * dst_tile_stride) = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(a, b)); + x0 = x1; + x1 = x2; + x2 = x3; + } + } else { + for (uint32_t t = 0; t < n_t; ++t) { HVX_Vector acc = hvx_vec_splat_f32(0.0f); for (uint32_t j = 0; j < d_conv; ++j) { - HVX_Vector x = *(const HVX_Vector *) (src0_T + (t + j) * d_inner_tile + cb); - HVX_Vector w = *(const HVX_Vector *) (src1_T + j * d_inner_stride + tile_off + cb); + HVX_Vector x = *(const HVX_Vector *) (src0_T + (t + j) * C_TILE); + HVX_Vector w = *(const HVX_Vector *) (wp + j * d_inner_stride); acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); } - - HVX_Vector y = Q6_Vsf_equals_Vqf32(acc); - float * dst_tile_ptr = dst_tile + t * tile_n + cb; - if (cb_n == C_TILE) { - *(HVX_Vector *) dst_tile_ptr = y; - } else { - hvx_vec_store_u(dst_tile_ptr, cb_n * sizeof(float), y); - } + *(HVX_Vector *) (op + t * dst_tile_stride) = Q6_Vsf_equals_Vqf32(acc); } } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); + } + + if (apply_silu) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); + hvx_ssm_conv_silu_vtcm(out, n_t * (dst_tile_stride / C_TILE)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); + } + + // Writeback dst_tile VTCM -> DDR via 2D DMA + dma_queue_push(dma_q, dma_make_data(dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float), (uint8_t *) out), + dst_stride_token_bytes, dst_tile_stride * sizeof(float), tile_n * sizeof(float), n_t); - // Writeback dst_tile from VTCM to DDR via 2D DMA - const dma_addr_t dst_tile_ddr = dst->data + - i3 * dst_stride_seq_bytes + - (ir0 + tile_off) * sizeof(float); - const size_t dst_row_bytes = tile_n * sizeof(float); + if (c + 2 < n_chunks) { + SSM_CONV_PUSH_FETCH(c + 2); + } - dma_queue_push(dma_q, dma_make_data(dst_tile_ddr, (uint8_t *) dst_tile), - dst_stride_token_bytes, dst_row_bytes, dst_row_bytes, n_t); - dma_queue_pop(dma_q); + tile_off += d_inner_tile; + if (tile_off >= d_inner_per_hvx) { + tile_off = 0; + i3++; } } + for (uint32_t k = MIN(n_chunks, 2); k > 0; --k) { + dma_queue_pop(dma_q); // drain the last writebacks + } + + #undef SSM_CONV_PUSH_FETCH + FARF(HIGH, "ssm-conv-f32-prefill %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], @@ -405,14 +556,9 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; - } - const struct htp_ssm_conv_kernel_params * kparams = (const struct htp_ssm_conv_kernel_params *) octx->kernel_params; - - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -440,30 +586,34 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; + const uint32_t raw_rpt = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); + const uint32_t d_inner_per_hvx = hex_round_up(raw_rpt, VLEN_FP32); + + struct htp_ssm_conv_vtcm_layout L; + htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_hvx, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_hvx); - octx->src0_spad.size_per_thread = kparams->vtcm_src0_size_per_thread; - octx->src1_spad.size_per_thread = kparams->vtcm_src1_size_per_thread; - octx->dst_spad.size_per_thread = kparams->vtcm_dst_size_per_thread; + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - octx->src0_spad.size = kparams->vtcm_src0_size; - octx->src1_spad.size = kparams->vtcm_src1_size; - octx->dst_spad.size = kparams->vtcm_dst_size; + octx->src0_spad.size_per_hvx = L.src0_bytes_per_hvx; + octx->src1_spad.size_per_hvx = L.src1_bytes_per_hvx; + octx->dst_spad.size_per_hvx = L.dst_bytes_per_hvx; - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; - octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; + octx->src0_spad.size = L.src0_bytes; + octx->src1_spad.size = L.src1_bytes; + octx->dst_spad.size = L.dst_bytes; + + octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + octx->src1_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); octx->src0_spad.src = NULL; octx->src1_spad.src = NULL; octx->dst_spad.src = NULL; - const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); - const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32); - struct htp_ssm_conv_context scctx = { .octx = octx, .kparams = kparams, - .nrows_per_thread = d_inner_per_thread, + .nrows_per_hvx = d_inner_per_hvx, .d_inner_tile = kparams->d_inner_tile, .row_start = row_start, .nrows = nrows, @@ -476,9 +626,9 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { kparams->n_t == 1 ? "decode" : "prefill"); if (kparams->n_t == 1) { - work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_decode, &scctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, ssm_conv_thread_f32_decode, &scctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_prefill, &scctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, ssm_conv_thread_f32_prefill, &scctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.h index be62d7bf5..7c3f3041f 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.h +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h @@ -3,34 +3,76 @@ #include <stdint.h> +#include "hex-common.h" #include "hex-fastdiv.h" #include "htp-ops.h" +#include "htp-vtcm.h" struct htp_ssm_conv_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t d_conv; uint32_t d_inner; uint32_t n_t; uint32_t n_s; - uint32_t d_inner_per_thread; uint32_t d_inner_tile; - uint32_t src0_row_size_aligned; - uint32_t src1_row_size_aligned; - uint32_t dst_row_size_aligned; - - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_src1_size_per_thread; - uint32_t vtcm_dst_size_per_thread; - - uint32_t vtcm_src0_size; - uint32_t vtcm_src1_size; - uint32_t vtcm_dst_size; uint32_t vtcm_size; + uint32_t silu; - struct fastdiv_values div_n_threads; + struct fastdiv_values div_d_inner_tile; }; +struct htp_ssm_conv_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_src1; + size_t off_dst; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; + size_t src0_bytes; + size_t src1_bytes; + size_t dst_bytes; +}; + +static inline void htp_ssm_conv_vtcm_layout_build( + struct htp_ssm_conv_vtcm_layout * L, + uint32_t d_inner_per_hvx, + uint32_t d_conv, + uint32_t n_t, + uint32_t d_inner_tile, + uint32_t n_hvx +) { + const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_hvx * d_conv * sizeof(float), 128) + 128; + const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_hvx * sizeof(float), 128); + + L->src1_bytes_per_hvx = src1_raw_bytes + src1_T_bytes; + + if (n_t == 1) { + const size_t src0_tile_raw_bytes = hex_round_up((size_t) d_inner_tile * d_conv * sizeof(float), 128); + const size_t src0_T_bytes = hex_round_up((size_t) d_conv * d_inner_tile * sizeof(float), 128); + L->src0_bytes_per_hvx = 2 * src0_tile_raw_bytes + src0_T_bytes; + L->dst_bytes_per_hvx = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128); + } else { + const uint32_t ncs = d_conv - 1 + n_t; + const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float); + const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128); + const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * raw_stride_bytes, 128); + L->src0_bytes_per_hvx = 2 * src0_tile_raw + src0_block_T; + L->dst_bytes_per_hvx = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128); + } + + L->src0_bytes = L->src0_bytes_per_hvx * n_hvx; + L->src1_bytes = L->src1_bytes_per_hvx * n_hvx; + L->dst_bytes = L->dst_bytes_per_hvx * n_hvx; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes); + VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes); + L->total_bytes = off; +} + #if defined(__cplusplus) static_assert(sizeof(struct htp_ssm_conv_kernel_params) <= 128, "htp_ssm_conv_kernel_params is too large for kernel_params blob"); #else diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c index 177c74b33..b3f8a7090 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c @@ -14,10 +14,11 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" +#include "sum-rows-ops.h" #define sum_rows_preamble \ const struct htp_tensor *src0 = octx->src[0]; \ @@ -50,19 +51,18 @@ struct sum_rows_context { uint32_t ne00; size_t src_stride; size_t dst_stride; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint32_t total_rows; - bool opt_path; }; -static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) { +static void sum_rows_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void *data) { const struct sum_rows_context * smctx = (const struct sum_rows_context *) data; - const uint32_t rows_per_thread = smctx->rows_per_thread; + const uint32_t rows_per_hvx = smctx->rows_per_hvx; const uint32_t total_rows = smctx->total_rows; - const uint32_t start_row = rows_per_thread * ith; - const uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); + const uint32_t start_row = rows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + rows_per_hvx, total_rows); if (start_row >= end_row) { return; @@ -71,7 +71,6 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) const size_t src_stride = smctx->src_stride; const size_t dst_stride = smctx->dst_stride; const uint32_t ne00 = smctx->ne00; - const bool opt_path = smctx->opt_path; const float * restrict src_th = (const float *) (smctx->src_data + (start_row * src_stride)); float * restrict dst_th = (float *) (smctx->dst_data + (start_row * dst_stride)); @@ -79,7 +78,7 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) // Calculate actual number of rows for this thread const uint32_t n_rows = end_row - start_row; - struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); for (uint32_t ir = 0; ir < n_rows; ir++) { @@ -90,11 +89,7 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) hex_l2fetch((const uint8_t *) src_local + src_stride, src_stride, src_stride, 1); } - if (opt_path) { - *dst_local = hvx_reduce_sum_f32_a((const uint8_t *) src_local, ne00); - } else { - *dst_local = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00); - } + *dst_local = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); @@ -111,6 +106,13 @@ int op_sum_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } + const struct htp_sum_rows_kernel_params * kparams = + (const struct htp_sum_rows_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t src0_nrows = ne01 * ne02 * ne03; const size_t dst_data_row_size = dst->ne[0] * sizeof(float); @@ -129,13 +131,8 @@ int op_sum_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); - - bool opt_path = false; - if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) { - opt_path = true; - } + const uint32_t n_hvx = octx->n_hvx; + const uint32_t rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); struct sum_rows_context smctx = { .octx = octx, @@ -144,12 +141,11 @@ int op_sum_rows(struct htp_ops_context * octx) { .ne00 = ne00, .src_stride = nb01, .dst_stride = nb1, - .rows_per_thread = rows_per_thread, + .rows_per_hvx = rows_per_hvx, .total_rows = nrows, - .opt_path = opt_path, }; - work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, sum_rows_thread_f32, &smctx, n_hvx); return HTP_STATUS_OK; } @@ -157,15 +153,15 @@ int op_sum_rows(struct htp_ops_context * octx) { struct sum_context { struct htp_ops_context * octx; const float * src_data; - float partial_sums[HTP_MAX_NTHREADS]; + float partial_sums[HTP_MAX_NHVX]; uint32_t total_elems; - uint32_t elems_per_thread; + uint32_t elems_per_hvx; }; -static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void sum_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct sum_context * sctx = (struct sum_context *) data; - const uint32_t start = sctx->elems_per_thread * ith; - const uint32_t end = MIN(start + sctx->elems_per_thread, sctx->total_elems); + const uint32_t start = sctx->elems_per_hvx * ith; + const uint32_t end = MIN(start + sctx->elems_per_hvx, sctx->total_elems); if (start >= end) { sctx->partial_sums[ith] = 0.0f; @@ -175,7 +171,7 @@ static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) { const uint32_t n = end - start; const float * src = sctx->src_data + start; - struct htp_thread_trace * tr = &sctx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); hex_l2fetch_block((const void *) src, n * sizeof(float)); @@ -197,6 +193,13 @@ int op_sum(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } + const struct htp_sum_rows_kernel_params * kparams = + (const struct htp_sum_rows_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { return HTP_STATUS_OK; } @@ -207,21 +210,20 @@ int op_sum(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = (total_elems >= 1024) ? MIN(octx->n_threads, HTP_MAX_NTHREADS) : 1; - const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads; - const uint32_t elems_per_thread = hex_round_up(raw_chunk, 32); + const uint32_t n_hvx = octx->n_hvx; + const uint32_t elems_per_hvx = kparams->elems_per_hvx; struct sum_context sctx = { .octx = octx, .src_data = (const float *) src0->data, .total_elems = total_elems, - .elems_per_thread = elems_per_thread, + .elems_per_hvx = elems_per_hvx, }; - work_queue_run(octx->ctx->work_queue, sum_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, sum_thread_f32, &sctx, n_hvx); float sum = 0.0f; - for (uint32_t i = 0; i < n_threads; i++) { + for (uint32_t i = 0; i < n_hvx; i++) { sum += sctx.partial_sums[i]; } ((float *) dst->data)[0] = sum; @@ -248,17 +250,17 @@ struct argmax_context { uint32_t dst_stride; uint32_t row_start; uint32_t nrows; - uint32_t rows_per_thread; - uint32_t elems_per_thread; + uint32_t rows_per_hvx; + uint32_t elems_per_hvx; - float partial_max[HTP_MAX_NTHREADS]; - int32_t partial_idx[HTP_MAX_NTHREADS]; + float partial_max[HTP_MAX_NHVX]; + int32_t partial_idx[HTP_MAX_NHVX]; }; -static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * data) { +static void argmax_thread_single_row(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct argmax_context * actx = (struct argmax_context *) data; - const uint32_t start = actx->elems_per_thread * ith; - const uint32_t end = MIN(start + actx->elems_per_thread, actx->ne00); + const uint32_t start = actx->elems_per_hvx * ith; + const uint32_t end = MIN(start + actx->elems_per_hvx, actx->ne00); if (start >= end) { actx->partial_max[ith] = -INFINITY; @@ -269,7 +271,7 @@ static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * const uint32_t n = end - start; const float * src = actx->src_data + start; - struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); hex_l2fetch_block((const void *) src, n * sizeof(float)); @@ -284,16 +286,16 @@ static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); } -static void argmax_thread_multi_row(unsigned int nth, unsigned int ith, void * data) { +static void argmax_thread_multi_row(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct argmax_context * actx = (struct argmax_context *) data; - const uint32_t r0 = actx->row_start + actx->rows_per_thread * ith; - const uint32_t r1 = MIN(r0 + actx->rows_per_thread, actx->row_start + actx->nrows); + const uint32_t r0 = actx->row_start + actx->rows_per_hvx * ith; + const uint32_t r1 = MIN(r0 + actx->rows_per_hvx, actx->row_start + actx->nrows); if (r0 >= r1) { return; } - struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r0); for (uint32_t r = r0; r < r1; r++) { @@ -324,6 +326,13 @@ int op_argmax(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } + const struct htp_sum_rows_kernel_params * kparams = + (const struct htp_sum_rows_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t ne00 = src0->ne[0]; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; @@ -341,9 +350,8 @@ int op_argmax(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = (ne00 >= 1024) ? MIN(octx->n_threads, HTP_MAX_NTHREADS) : 1; - const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads; - const uint32_t elems_per_thread = hex_round_up(raw_chunk, 32); + const uint32_t n_hvx = octx->n_hvx; + const uint32_t elems_per_hvx = kparams->elems_per_hvx; struct argmax_context actx = { .octx = octx, @@ -354,15 +362,15 @@ int op_argmax(struct htp_ops_context * octx) { .dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t), .row_start = 0, .nrows = 1, - .rows_per_thread = 1, - .elems_per_thread = elems_per_thread, + .rows_per_hvx = 1, + .elems_per_hvx = elems_per_hvx, }; - work_queue_run(octx->ctx->work_queue, argmax_thread_single_row, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, argmax_thread_single_row, &actx, n_hvx); float best_val = actx.partial_max[0]; int32_t best_idx = actx.partial_idx[0]; - for (uint32_t i = 1; i < n_threads; i++) { + for (uint32_t i = 1; i < n_hvx; i++) { if (actx.partial_max[i] > best_val) { best_val = actx.partial_max[i]; best_idx = actx.partial_idx[i]; @@ -389,8 +397,8 @@ int op_argmax(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = MIN(octx->n_threads, nrows); - const uint32_t rows_per_thread = (nrows + n_threads - 1) / n_threads; + const uint32_t n_hvx = octx->n_hvx; + const uint32_t rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); struct argmax_context actx = { .octx = octx, @@ -401,10 +409,10 @@ int op_argmax(struct htp_ops_context * octx) { .dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t), .row_start = row_start, .nrows = nrows, - .rows_per_thread = rows_per_thread, - .elems_per_thread = 0, + .rows_per_hvx = rows_per_hvx, + .elems_per_hvx = 0, }; - work_queue_run(octx->ctx->work_queue, argmax_thread_multi_row, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, argmax_thread_multi_row, &actx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h new file mode 100644 index 000000000..0b8acf030 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h @@ -0,0 +1,17 @@ +#ifndef HTP_SUM_ROWS_OPS_H +#define HTP_SUM_ROWS_OPS_H + +#include <stdint.h> + +struct htp_sum_rows_kernel_params { + uint32_t n_hvx; + uint32_t elems_per_hvx; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_sum_rows_kernel_params) <= 128, "htp_sum_rows_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_sum_rows_kernel_params) <= 128, "htp_sum_rows_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_SUM_ROWS_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c new file mode 100644 index 000000000..9c4dc583c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c @@ -0,0 +1,665 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include <HAP_farf.h> +#include <HAP_perf.h> +#include <stdatomic.h> + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "hex-common.h" +#include "htp-trace.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hvx-types.h" +#include "hvx-utils.h" +#include "dma-queue.h" +#include "top-k-mask-ops.h" + +#define TOP_K_MASK_NINF_F32 0xff800000u // f32 score compare + +struct htp_top_k_mask_context { + struct htp_ops_context * octx; + const struct htp_top_k_mask_kernel_params * kparams; + + uint32_t rows_per_hvx; + uint32_t row0, row1; // rows this device handles + uint32_t off_extra, off_score; // parts inside one row input slot + uint32_t col0, ncols_out; // PICKS: columns this device writes back + atomic_uint barrier; // PICKS: arrivals, barrier k waits for k * n_hvx + + uint8_t * blocks; + uint8_t * mask; + uint8_t * rowin; + uint8_t * tiles; + uint8_t * mtiles; + uint8_t * scratch; +}; + +// DMA queue pop with a trace event around the wait +static inline void top_k_mask_pop(dma_queue * dma_q, struct htp_thread_trace * tr, uint16_t info) { + htp_trace_event_start(tr, HTP_TRACE_EVT_DMA, info); + dma_queue_pop(dma_q); + htp_trace_event_stop(tr, HTP_TRACE_EVT_DMA, info); +} + +// Lanes 0..31 as words, in registers: inclusive scan of ones, minus one +static inline HVX_Vector top_k_mask_lane_iota(void) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector one = Q6_V_vsplat_R(1); + HVX_Vector v = one; + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 4)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 8)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 16)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 32)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 64)); + return Q6_Vw_vsub_VwVw(v, one); +} + +static __attribute__((noinline)) void top_k_mask_fill(uint8_t * restrict dst, uint32_t nbytes, uint32_t value) { + const HVX_Vector v = Q6_V_vsplat_R(value); + HVX_Vector * vdst = (HVX_Vector *) dst; + const uint32_t nvec = nbytes / VLEN; + + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + vdst[i] = v; + } +} + +#define TOP_K_MASK_DEAD 0x7fffffffu // cell value of a dead lane: c - c0 >=u ncols for every tile, signed max + +// Byte offsets of the cells in [c0, c0 + ncols) relative to c0, 0 for the others +static inline HVX_Vector top_k_mask_keep_off(HVX_Vector c, HVX_Vector v_c0, HVX_Vector v_ncols, const bool is_f16) { + const HVX_Vector cr = Q6_Vw_vsub_VwVw(c, v_c0); + return Q6_Vw_vasl_VwR(Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VuwVuw(v_ncols, cr), cr, Q6_V_vzero()), is_f16 ? 1 : 2); +} + +// Mask value of each cell lane in [c0, c0 + ncols) into tmp (predicated gather), returns the scatter offsets +static inline HVX_Vector top_k_mask_keep_gather(HVX_Vector c, const uint8_t * restrict mrow, HVX_Vector v_c0, HVX_Vector v_ncols, + uint32_t mu, HVX_Vector * restrict tmp, const bool is_f16) { + const HVX_VectorPred q = Q6_Q_vcmp_gt_VuwVuw(v_ncols, Q6_Vw_vsub_VwVw(c, v_c0)); + const HVX_Vector off = top_k_mask_keep_off(c, v_c0, v_ncols, is_f16); + + // Halfword data with word offsets: both halves of the pair hold the same offsets, the word predicate covers both halfwords of a lane + if (is_f16) { + Q6_vgather_AQRMWw(tmp, q, (size_t) mrow, mu, Q6_W_vcombine_VV(off, off)); + } else { + Q6_vgather_AQRMVw(tmp, q, (size_t) mrow, mu, off); + } + return off; +} + +static inline void top_k_mask_keep_scatter(HVX_Vector c, HVX_Vector off, uint8_t * restrict tile, HVX_Vector v_c0, HVX_Vector v_ncols, + uint32_t mu, HVX_Vector val, const bool is_f16) { + const HVX_VectorPred q = Q6_Q_vcmp_gt_VuwVuw(v_ncols, Q6_Vw_vsub_VwVw(c, v_c0)); + if (is_f16) { + Q6_vscatter_QRMWwV(q, (size_t) tile, mu, Q6_W_vcombine_VV(off, off), val); + } else { + Q6_vscatter_QRMVwV(q, (size_t) tile, mu, off, val); + } +} + +// Keep 4 cell vectors (dead lanes = TOP_K_MASK_DEAD): tile[c - c0] = mrow[c - c0] for c in [c0, c0 + ncols). One gather sync. +// flag: no mask, tile[c - c0] = all ones (keep flag) +static inline void top_k_mask_keep4(HVX_Vector c0v, HVX_Vector c1v, HVX_Vector c2v, HVX_Vector c3v, uint8_t * restrict tile, + const uint8_t * restrict mrow, HVX_Vector v_c0, HVX_Vector v_ncols, uint32_t mu, + HVX_Vector * restrict tmp, const bool is_f16, const bool flag) { + if (flag) { + const HVX_Vector ones = Q6_V_vsplat_R(0xffffffffu); + top_k_mask_keep_scatter(c0v, top_k_mask_keep_off(c0v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c1v, top_k_mask_keep_off(c1v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c2v, top_k_mask_keep_off(c2v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c3v, top_k_mask_keep_off(c3v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + return; + } + const HVX_Vector o0 = top_k_mask_keep_gather(c0v, mrow, v_c0, v_ncols, mu, tmp + 0, is_f16); + const HVX_Vector o1 = top_k_mask_keep_gather(c1v, mrow, v_c0, v_ncols, mu, tmp + 1, is_f16); + const HVX_Vector o2 = top_k_mask_keep_gather(c2v, mrow, v_c0, v_ncols, mu, tmp + 2, is_f16); + const HVX_Vector o3 = top_k_mask_keep_gather(c3v, mrow, v_c0, v_ncols, mu, tmp + 3, is_f16); + hvx_gather_sync(tmp + 3); + top_k_mask_keep_scatter(c0v, o0, tile, v_c0, v_ncols, mu, tmp[0], is_f16); + top_k_mask_keep_scatter(c1v, o1, tile, v_c0, v_ncols, mu, tmp[1], is_f16); + top_k_mask_keep_scatter(c2v, o2, tile, v_c0, v_ncols, mu, tmp[2], is_f16); + top_k_mask_keep_scatter(c3v, o3, tile, v_c0, v_ncols, mu, tmp[3], is_f16); +} + +// Pick vector j: picks below j1 and n_pick are valid, returns them with dead lanes marked. v_rem = j1 - lane index. +static inline HVX_Vector top_k_mask_load_picks(const int32_t * restrict tk, uint32_t j, HVX_Vector v_rem, HVX_Vector v_npick) { + const HVX_Vector p_raw = hvx_vmem(tk + j); + // lane + j <u j1 is the tail, p <u n_pick drops p < 0 and p >= n_pick + const HVX_VectorPred valid = Q6_Q_vcmp_gtand_QVuwVuw(Q6_Q_vcmp_gt_VwVw(v_rem, Q6_V_vsplat_R(j)), v_npick, p_raw); + return Q6_V_vmux_QVV(valid, p_raw, Q6_V_vsplat_R(TOP_K_MASK_DEAD)); +} + +// Scatter the kept cells of picks [j0, j1) (and the extras) into the tile of columns [c0, c0 + ncols). slot holds the row inputs, mrow the mask columns. +// Gathers are issued in batches with one sync per batch: tmp[0..3] scores/blocks, tmp[4] score with blocks, tmp[5..8] mask values. +static inline __attribute__((always_inline)) void top_k_mask_scatter_tile( + const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, const uint8_t * restrict mrow, uint8_t * restrict tile, + HVX_Vector * restrict tmp, uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra, + const bool is_f16, const bool flag) { + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + + const int32_t * restrict tk = (const int32_t *) slot; + const int32_t * restrict ex = (const int32_t *) (slot + tctx->off_extra); + const uint8_t * restrict sc = slot + tctx->off_score; + const uint8_t * restrict blk = tctx->blocks; + + const bool has_score = kp->flags & HTP_TOP_K_MASK_HAS_SCORE; + const bool has_blocks = kp->flags & HTP_TOP_K_MASK_HAS_BLOCKS; + const uint32_t n_blk = kp->n_blk; + const uint32_t n_extra = do_extra ? kp->n_extra : 0; + const uint32_t mu = ncols * kp->elem_size - 1; + const uint32_t sc_mu = kp->score_bytes - 1; + const uint32_t blk_mu = kp->blocks_bytes - 1; + const uint32_t nb1_h = ((uint32_t) kp->blocks_nb1 << 16) | kp->blocks_nb1; + + const HVX_Vector v_lane = top_k_mask_lane_iota(); // in the HVX_COMP scope of the caller + const HVX_Vector v_c0 = Q6_V_vsplat_R(c0); + const HVX_Vector v_ncols = Q6_V_vsplat_R(ncols); + const HVX_Vector v_npick = Q6_V_vsplat_R(kp->n_pick); + const HVX_Vector v_pmax = Q6_V_vsplat_R(kp->n_pick - 1); // dead picks gather the last entry, every gather stays in its region + const HVX_Vector v_rem = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(j1), v_lane); + const HVX_Vector v_ninf = Q6_V_vsplat_R(TOP_K_MASK_NINF_F32); + const HVX_Vector v_dead = Q6_V_vsplat_R(TOP_K_MASK_DEAD); + + HVX_Vector * restrict mtmp = tmp + 5; + + if (has_blocks) { + for (uint32_t j = j0; j < j1; j += VLEN_FP32) { + HVX_Vector cell = top_k_mask_load_picks(tk, j, v_rem, v_npick); + const HVX_Vector p = Q6_Vw_vmin_VwVw(cell, v_pmax); + if (has_score) { + Q6_vgather_ARMVw(tmp + 4, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(p, 2)); + } + const HVX_Vector boff = Q6_Vw_vmpyi_VwRh(p, nb1_h); + for (uint32_t k = 0; k < n_blk; k += 4) { + // blocks k..k+3 of every pick; the missing ones (k + i >= n_blk) are dead + const uint32_t nk = MIN(n_blk - k, 4); + const HVX_Vector bo = Q6_Vw_vadd_VwVw(boff, Q6_V_vsplat_R(k * 4)); + for (uint32_t i = 0; i < nk; i++) { + Q6_vgather_ARMVw(tmp + i, (size_t) blk, blk_mu, Q6_Vw_vadd_VwVw(bo, Q6_V_vsplat_R(i * 4))); + } + hvx_gather_sync(tmp + nk - 1); + if (has_score && k == 0) { + // IEEE compare: NaN and -inf scores are dead + cell = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[4], v_ninf), cell, v_dead); + } + const HVX_VectorPred dead = Q6_Q_vcmp_eq_VwVw(cell, v_dead); + top_k_mask_keep4(Q6_V_vmux_QVV(dead, v_dead, tmp[0]), + nk > 1 ? Q6_V_vmux_QVV(dead, v_dead, tmp[1]) : v_dead, + nk > 2 ? Q6_V_vmux_QVV(dead, v_dead, tmp[2]) : v_dead, + nk > 3 ? Q6_V_vmux_QVV(dead, v_dead, tmp[3]) : v_dead, + tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + } + } else { + // 4 pick vectors per batch, the vectors at or past j1 are dead and not loaded + for (uint32_t j = j0; j < j1; j += 4 * VLEN_FP32) { + HVX_Vector c0v = top_k_mask_load_picks(tk, j, v_rem, v_npick); + HVX_Vector c1v = j + 1 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 1 * VLEN_FP32, v_rem, v_npick) : v_dead; + HVX_Vector c2v = j + 2 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 2 * VLEN_FP32, v_rem, v_npick) : v_dead; + HVX_Vector c3v = j + 3 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 3 * VLEN_FP32, v_rem, v_npick) : v_dead; + if (has_score) { + Q6_vgather_ARMVw(tmp + 0, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c0v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 1, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c1v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 2, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c2v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 3, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c3v, v_pmax), 2)); + hvx_gather_sync(tmp + 3); + c0v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[0], v_ninf), c0v, v_dead); + c1v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[1], v_ninf), c1v, v_dead); + c2v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[2], v_ninf), c2v, v_dead); + c3v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[3], v_ninf), c3v, v_dead); + } + top_k_mask_keep4(c0v, c1v, c2v, c3v, tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + } + + const HVX_Vector v_rem_ex = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(n_extra), v_lane); + for (uint32_t j = 0; j < n_extra; j += VLEN_FP32) { + const HVX_VectorPred tail = Q6_Q_vcmp_gt_VwVw(v_rem_ex, Q6_V_vsplat_R(j)); + top_k_mask_keep4(Q6_V_vmux_QVV(tail, hvx_vmem(ex + j), v_dead), v_dead, v_dead, v_dead, tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + + // release the scatters before DMA reads the tile + hvx_gather_sync(tile); +} + +static __attribute__((noinline)) void top_k_mask_scatter_tile_f32(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + const uint8_t * restrict mrow, uint8_t * restrict tile, HVX_Vector * restrict tmp, + uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, mrow, tile, tmp, c0, ncols, j0, j1, do_extra, false, false); +} + +static __attribute__((noinline)) void top_k_mask_scatter_tile_f16(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + const uint8_t * restrict mrow, uint8_t * restrict tile, HVX_Vector * restrict tmp, + uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, mrow, tile, tmp, c0, ncols, j0, j1, do_extra, true, false); +} + +// PICKS: keep flags (all ones) of the picks [j0, j1) into the flag tile of the whole row, the element size picks the flag width +static __attribute__((noinline)) void top_k_mask_flag_tile_f32(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + uint8_t * restrict flags, HVX_Vector * restrict tmp, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, NULL, flags, tmp, 0, tctx->kparams->n_kv, j0, j1, do_extra, false, true); +} + +static __attribute__((noinline)) void top_k_mask_flag_tile_f16(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + uint8_t * restrict flags, HVX_Vector * restrict tmp, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, NULL, flags, tmp, 0, tctx->kparams->n_kv, j0, j1, do_extra, true, true); +} + +// out = flag ? out : -inf over nbytes (a multiple of 128) of the row tile +static __attribute__((noinline)) void top_k_mask_select(uint8_t * restrict out, const uint8_t * restrict flags, uint32_t nbytes, uint32_t ninf) { + const HVX_Vector v_ninf = Q6_V_vsplat_R(ninf); + HVX_Vector * vout = (HVX_Vector *) out; + const HVX_Vector * vflag = (const HVX_Vector *) flags; + const uint32_t nvec = nbytes / VLEN; + + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + vout[i] = Q6_V_vxor_VV(v_ninf, Q6_V_vand_VV(Q6_V_vxor_VV(vout[i], v_ninf), vflag[i])); + } +} + +// Push the row inputs of flat row r into slot: top_k, extra, score. Returns the push count. +static inline uint32_t top_k_mask_push_rowin(const struct htp_top_k_mask_context * tctx, dma_queue * dma_q, uint32_t r, uint8_t * slot) { + const struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * top_k = octx->src[1]; + const struct htp_tensor * score = octx->src[2]; + const struct htp_tensor * extra = octx->src[4]; + + const uint32_t tk_bytes = kp->n_top * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(slot, top_k->data + r * kp->u.rows.rs_top), tk_bytes, tk_bytes, tk_bytes, 1); + uint32_t n = 1; + + if (kp->flags & HTP_TOP_K_MASK_HAS_EXTRA) { + const uint32_t ex_bytes = kp->n_extra * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(slot + tctx->off_extra, extra->data + r * kp->u.rows.rs_extra), ex_bytes, ex_bytes, ex_bytes, 1); + n++; + } + + if (kp->flags & HTP_TOP_K_MASK_HAS_SCORE) { + dma_queue_push(dma_q, dma_make_data(slot + tctx->off_score, score->data + r * kp->u.rows.rs_score), kp->score_bytes, kp->score_bytes, kp->score_bytes, 1); + n++; + } + + return n; +} + +static inline struct fastdiv_values top_k_mask_fd(const struct htp_top_k_mask_kernel_params * kp, uint32_t k) { + const struct fastdiv_values fd = { kp->u.rows.m.dec.mp[k], kp->u.rows.m.dec.l[k] }; + return fd; +} + +// Mask row of flat row r: (i1, i2, i3) of dst, each taken modulo the mask dims, rows dense above the mask ne1 +static inline uint32_t top_k_mask_mask_row(const struct htp_top_k_mask_kernel_params * kp, uint32_t r) { + const uint16_t * ne = kp->u.rows.m.dec.ne; + const struct fastdiv_values fd1 = top_k_mask_fd(kp, 0); + const struct fastdiv_values fd2 = top_k_mask_fd(kp, 1); + const struct fastdiv_values fm1 = top_k_mask_fd(kp, 2); + const struct fastdiv_values fm2 = top_k_mask_fd(kp, 3); + const struct fastdiv_values fm3 = top_k_mask_fd(kp, 4); + + const uint32_t q1 = fastdiv(r, &fd1); + const uint32_t i1 = r - q1 * ne[0]; + const uint32_t i3 = fastdiv(q1, &fd2); + const uint32_t i2 = q1 - i3 * ne[1]; + + const uint32_t m1 = fastmodulo(i1, ne[2], &fm1); + const uint32_t m2 = fastmodulo(i2, ne[3], &fm2); + const uint32_t m3 = fastmodulo(i3, ne[4], &fm3); + return (m3 * ne[3] + m2) * ne[2] + m1; +} + +// Push the inputs of item (row r, col tile ct): row inputs on the first tile of a row, then the mask tile. Returns the push count. +static inline uint32_t top_k_mask_push_item(const struct htp_top_k_mask_context * tctx, dma_queue * dma_q, uint32_t r, uint32_t ct, + uint8_t * slot, uint8_t * mslot) { + const struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * mask = octx->src[0]; + + const uint32_t c0 = ct * kp->tile_cols; + + uint32_t n = 0; + if (ct == 0) { + n += top_k_mask_push_rowin(tctx, dma_q, r, slot); + } + + if (mslot) { + const uint32_t m_off = top_k_mask_mask_row(kp, r) * kp->u.rows.mask_nb1 + c0 * kp->elem_size; + const uint32_t nb = MIN(kp->tile_cols, kp->n_kv - c0) * kp->elem_size; + dma_queue_push(dma_q, dma_make_data(mslot, mask->data + m_off), nb, nb, nb, 1); + n++; + } + + return n; +} + +static inline void top_k_mask_next(uint32_t * r, uint32_t * ct, uint32_t n_col_tiles) { + if (++(*ct) == n_col_tiles) { + *ct = 0; + ++(*r); + } +} + +// ROWS: the thread count follows the device rows (mdev), so the slice of an n byte copy is sized here +static inline uint32_t top_k_mask_slice(const struct htp_ops_context * octx, unsigned int nth, uint32_t n) { + return hex_round_up(fastdiv(n + nth - 1, &octx->n_hvx_div), HTP_TOP_K_MASK_UNIT); +} + +// Push this thread's slice (128B aligned) of a contiguous n byte copy. Returns the push count. +static inline uint32_t top_k_mask_push_slice(dma_queue * dma_q, unsigned int ith, dma_addr_t dst, dma_addr_t src, uint32_t n, uint32_t slice) { + const uint32_t b0 = ith * slice; + if (b0 >= n) { + return 0; + } + const uint32_t nb = MIN(slice, n - b0); + dma_queue_push(dma_q, dma_make_data(dst + b0, src + b0), nb, nb, nb, 1); + return 1; +} + +// Pass 1 (ROWS): load the blocks table and the resident mask row, split across threads. Fill the tiles of this thread with -inf meanwhile. +static void top_k_mask_preload(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + struct htp_ops_context * octx = tctx->octx; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + + uint32_t n_pushed = 0; + + if (kp->blocks_bytes) { + n_pushed += top_k_mask_push_slice(dma_q, ith, (dma_addr_t) tctx->blocks, octx->src[3]->data, kp->blocks_bytes, top_k_mask_slice(octx, nth, kp->blocks_bytes)); + } + if (kp->flags & HTP_TOP_K_MASK_MASK_RESIDENT) { + const uint32_t mask_bytes = kp->u.rows.m.mask_bytes; + n_pushed += top_k_mask_push_slice(dma_q, ith, (dma_addr_t) tctx->mask, octx->src[0]->data, mask_bytes, top_k_mask_slice(octx, nth, mask_bytes)); + } + + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + top_k_mask_fill(tctx->tiles + ith * 2 * tile_bytes, 2 * tile_bytes, kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + + for (uint32_t i = 0; i < n_pushed; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } +} + +// Pass 2 (ROWS): depth-2 DMA pipeline over the items of rows [row_first, row_last), item = (row, col tile). +// The row inputs alternate between 2 slots by row. +static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, const htp_thread_t * th, unsigned int ith, uint32_t row_first, uint32_t row_last) { + struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * dst = octx->dst; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + + const bool is_f16 = kp->flags & HTP_TOP_K_MASK_F16; + const uint32_t elem_size = kp->elem_size; + + const uint32_t rowin_bytes = kp->rowin_units * HTP_TOP_K_MASK_UNIT; + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + const uint32_t tile_cols = kp->tile_cols; + const uint32_t n_col_tiles = kp->n_col_tiles; + const uint32_t n_items = (row_last - row_first) * n_col_tiles; + + // 2 row input slots per thread, alternating by row + uint8_t * rowin = tctx->rowin + ith * 2 * rowin_bytes; + uint8_t * tiles = tctx->tiles + ith * 2 * tile_bytes; + uint8_t * mtiles = (kp->flags & HTP_TOP_K_MASK_MASK_RESIDENT) ? NULL : tctx->mtiles + ith * 2 * tile_bytes; + + HVX_Vector * tmp = (HVX_Vector *) (tctx->scratch + ith * HTP_TOP_K_MASK_SCRATCH_BYTES); + + const dma_addr_t dst_data = dst->data; + + uint32_t n_in[2] = { 0, 0 }; + + // prefetch cursor + uint32_t pr = row_first, pct = 0; + + for (uint32_t it = 0; it < MIN(n_items, 2); it++) { + // Dummy dst writeback to establish queue ordering + dma_queue_push(dma_q, dma_make_data(dst_data, tiles + it * tile_bytes), 0, 0, 0, 0); + n_in[it] = top_k_mask_push_item(tctx, dma_q, pr, pct, rowin + ((pr - row_first) & 1) * rowin_bytes, mtiles ? mtiles + it * tile_bytes : NULL); + top_k_mask_next(&pr, &pct, n_col_tiles); + } + + uint32_t r = row_first, ct = 0; + + for (uint32_t it = 0; it < n_items; it++) { + const uint32_t s = it & 1; + uint8_t * tile = tiles + s * tile_bytes; + + // writeback of item it - 2 (or the dummy), then the tile is free again + top_k_mask_pop(dma_q, tr, (uint16_t) it); + if (it >= 2) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + top_k_mask_fill(tile, tile_bytes, kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + } + + for (uint32_t i = 0; i < n_in[s]; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) it); + } + + const uint32_t c0 = ct * tile_cols; + const uint32_t ncols = MIN(tile_cols, kp->n_kv - c0); + + const uint8_t * slot = rowin + ((r - row_first) & 1) * rowin_bytes; + const uint8_t * mrow = mtiles ? mtiles + s * tile_bytes : tctx->mask + c0 * elem_size; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + if (is_f16) { + top_k_mask_scatter_tile_f16(tctx, slot, mrow, tile, tmp, c0, ncols, 0, kp->n_top, true); + } else { + top_k_mask_scatter_tile_f32(tctx, slot, mrow, tile, tmp, c0, ncols, 0, kp->n_top, true); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + + const uint32_t d_off = r * kp->u.rows.rs_dst + c0 * elem_size; + const uint32_t nb = ncols * elem_size; + dma_queue_push(dma_q, dma_make_data(dst_data + d_off, tile), nb, nb, nb, 1); + + if (it + 2 < n_items) { + n_in[s] = top_k_mask_push_item(tctx, dma_q, pr, pct, rowin + ((pr - row_first) & 1) * rowin_bytes, mtiles ? mtiles + s * tile_bytes : NULL); + top_k_mask_next(&pr, &pct, n_col_tiles); + } + top_k_mask_next(&r, &ct, n_col_tiles); + } + + const uint32_t n_wb = MIN(n_items, 2); + for (uint32_t i = 0; i < n_wb; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } +} + +// Pass 2 (ROWS): whole rows split across threads, the col tiles of a row in order +static void top_k_mask_thread_rows(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_top_k_mask_context * tctx = (const struct htp_top_k_mask_context *) data; + + const uint32_t ir0 = tctx->row0 + ith * tctx->rows_per_hvx; + const uint32_t ir1 = MIN(ir0 + tctx->rows_per_hvx, tctx->row1); + if (ir0 >= ir1) { + return; + } + + top_k_mask_run_items(tctx, th, ith, ir0, ir1); +} + +static inline void top_k_mask_barrier(atomic_uint * b, uint32_t target, struct htp_thread_trace * tr, uint16_t info) { + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, info); + atomic_fetch_add_explicit(b, 1, memory_order_release); + while (atomic_load_explicit(b, memory_order_acquire) < target) { + hex_pause(); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, info); +} + +// PICKS, one pass over n_virt = kp->n_hvx virtual workers (host slices); real worker ith runs the virtual workers ith, ith + nth, ... +// A: shared preload and a slice of the flag clear per virtual worker, the mask slice of this worker into the row tile; +// barrier; B: scatter keep flags of the picks of each virtual worker; barrier; C: select mask or -inf in this worker's slice and write it back +static void top_k_mask_thread_picks(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + struct htp_ops_context * octx = tctx->octx; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + + const struct htp_tensor * top_k = octx->src[1]; + const struct htp_tensor * score = octx->src[2]; + const struct htp_tensor * extra = octx->src[4]; + + uint8_t * slot = tctx->rowin; + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + uint8_t * out = tctx->tiles; + uint8_t * flags = tctx->tiles + tile_bytes; + + const uint32_t n_virt = kp->n_hvx; + + // this worker's slice of the device columns: the mask goes straight into the row tile, it is popped after the scan. + // The device share of the columns depends on mdev, so this slice is sized here. + const uint32_t out_b0 = tctx->col0 * kp->elem_size; + const uint32_t out_nb = tctx->ncols_out * kp->elem_size; + const uint32_t wslice = hex_round_up(fastdiv(out_nb + nth - 1, &octx->n_hvx_div), 128); + const uint32_t wb0 = out_b0 + ith * wslice; + const uint32_t wnb = ith * wslice < out_nb ? MIN(wslice, out_nb - ith * wslice) : 0; + + const uint32_t fslice = kp->u.picks.flags_slice_u * HTP_TOP_K_MASK_UNIT; + + for (uint32_t v = ith; v < n_virt; v += nth) { + uint32_t n_pushed = 0; + if (kp->blocks_bytes) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) tctx->blocks, octx->src[3]->data, kp->blocks_bytes, kp->u.picks.blocks_slice_u * HTP_TOP_K_MASK_UNIT); + } + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) slot, top_k->data, kp->n_top * sizeof(int32_t), kp->u.picks.top_slice_u * HTP_TOP_K_MASK_UNIT); + if (kp->flags & HTP_TOP_K_MASK_HAS_EXTRA) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) (slot + tctx->off_extra), extra->data, kp->n_extra * sizeof(int32_t), kp->u.picks.extra_slice_u * HTP_TOP_K_MASK_UNIT); + } + if (kp->flags & HTP_TOP_K_MASK_HAS_SCORE) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) (slot + tctx->off_score), score->data, kp->score_bytes, kp->u.picks.score_slice_u * HTP_TOP_K_MASK_UNIT); + } + + // the mask slice follows the last preload in the queue, so the pops below leave it in flight + if (wnb && v + nth >= n_virt) { + dma_queue_push(dma_q, dma_make_data(out + wb0, octx->src[0]->data + wb0), wnb, wnb, wnb, 1); + } + + const uint32_t fb0 = v * fslice; + if (fb0 < tile_bytes) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + top_k_mask_fill(flags + fb0, MIN(fslice, tile_bytes - fb0), 0); // clear the flags + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + } + + for (uint32_t i = 0; i < n_pushed; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } + } + + top_k_mask_barrier(&tctx->barrier, nth, tr, 0); + + HVX_Vector * tmp = (HVX_Vector *) (tctx->scratch + ith * HTP_TOP_K_MASK_SCRATCH_BYTES); + + for (uint32_t v = ith; v < n_virt; v += nth) { + const uint32_t j0 = MIN(v * kp->u.picks.picks_per_hvx, kp->n_top); + const uint32_t j1 = MIN(j0 + kp->u.picks.picks_per_hvx, kp->n_top); + const bool do_extra = v == n_virt - 1; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + if (kp->flags & HTP_TOP_K_MASK_F16) { + top_k_mask_flag_tile_f16(tctx, slot, flags, tmp, j0, j1, do_extra); + } else { + top_k_mask_flag_tile_f32(tctx, slot, flags, tmp, j0, j1, do_extra); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + } + + top_k_mask_barrier(&tctx->barrier, 2 * nth, tr, 1); + + if (wnb) { + top_k_mask_pop(dma_q, tr, (uint16_t) ith); + // wb0 is 128B aligned (col0 is a multiple of 64 columns), the select may run past the row inside the tile + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + top_k_mask_select(out + wb0, flags + wb0, hex_round_up(wnb, 128), kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + dma_queue_push(dma_q, dma_make_data(octx->dst->data + wb0, out + wb0), wnb, wnb, wnb, 1); + top_k_mask_pop(dma_q, tr, (uint16_t) ith); + } +} + +int op_top_k_mask(struct htp_ops_context * octx) { + const struct htp_top_k_mask_kernel_params * kp = (const struct htp_top_k_mask_kernel_params *) octx->kernel_params; + + if (kp->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const bool picks = kp->kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS; + + // mdev: ROWS gives this device a contiguous share of the rows, DMA writes are exempt from the 128B rule. + // PICKS: every device scans the row, the write back is split by 64 column chunks. + uint32_t dev_row0 = 0; + uint32_t dev_rows = picks ? 1 : kp->u.rows.total_rows; + uint32_t dev_c0 = 0; + uint32_t dev_cols = kp->n_kv; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = picks ? + htp_tensor_mdev_partition(kp->n_kv, 64, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div) : + htp_tensor_mdev_partition(kp->u.rows.total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + if (range.count == 0) { + return HTP_STATUS_OK; + } + if (picks) { + dev_c0 = range.start; + dev_cols = range.count; + } else { + dev_row0 = range.start; + dev_rows = range.count; + } + } + + // fewer workers than kparams->n_hvx always fit the VTCM layout: ROWS runs one row range per worker, PICKS loops over the host slices + const uint32_t max_hvx = MIN((uint32_t) kp->n_hvx, octx->ctx->n_hvx); + if (!htp_ops_context_set_n_hvx(octx, picks ? max_hvx : MIN(max_hvx, dev_rows))) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t n_hvx = octx->n_hvx; + + uint8_t * vtcm = octx->ctx->vtcm_base; + + struct htp_top_k_mask_context tctx = { + .octx = octx, + .kparams = kp, + .off_extra = kp->off_extra_units * HTP_TOP_K_MASK_UNIT, + .off_score = kp->off_score_units * HTP_TOP_K_MASK_UNIT, + .row0 = dev_row0, + .row1 = dev_row0 + dev_rows, + .col0 = dev_c0, + .ncols_out = dev_cols, + .blocks = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_blocks_u * HTP_TOP_K_MASK_UNIT), + .mask = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_mask_u * HTP_TOP_K_MASK_UNIT), + .rowin = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_rowin_u * HTP_TOP_K_MASK_UNIT), + .tiles = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_tiles_u * HTP_TOP_K_MASK_UNIT), + .mtiles = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_mtiles_u * HTP_TOP_K_MASK_UNIT), + .scratch = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_scratch_u * HTP_TOP_K_MASK_UNIT), + }; + + if (picks) { + atomic_init(&tctx.barrier, 0); + work_queue_run(octx->ctx->hvx_queue, top_k_mask_thread_picks, &tctx, n_hvx); + return HTP_STATUS_OK; + } + + tctx.rows_per_hvx = fastdiv(dev_rows + n_hvx - 1, &octx->n_hvx_div); + + work_queue_run(octx->ctx->hvx_queue, top_k_mask_preload, &tctx, n_hvx); + work_queue_run(octx->ctx->hvx_queue, top_k_mask_thread_rows, &tctx, n_hvx); + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h new file mode 100644 index 000000000..9dcdfaa48 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h @@ -0,0 +1,131 @@ +#ifndef HTP_TOP_K_MASK_OPS_H +#define HTP_TOP_K_MASK_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include <stdbool.h> +#include <stdint.h> +#include <string.h> + +// HVX gather results land here (gathers write to VTCM only): 10 vectors per thread +#define HTP_TOP_K_MASK_SCRATCH_BYTES 1280 + +enum htp_top_k_mask_kernel { + HTP_TOP_K_MASK_KERNEL_ROWS = 0, // threads split rows, row inputs streamed per row + HTP_TOP_K_MASK_KERNEL_PICKS = 1, // one row with one resident mask row: threads split the picks, keep flags select mask or -inf +}; + +#define HTP_TOP_K_MASK_F16 (1u << 0) +#define HTP_TOP_K_MASK_HAS_SCORE (1u << 1) +#define HTP_TOP_K_MASK_HAS_BLOCKS (1u << 2) +#define HTP_TOP_K_MASK_HAS_EXTRA (1u << 3) +#define HTP_TOP_K_MASK_MASK_RESIDENT (1u << 4) // mask has exactly 1 row + +#define HTP_TOP_K_MASK_UNIT 128 // unit of the compact u16 sizes in kernel_params + +struct htp_top_k_mask_shape { + uint32_t n_kv, elem_size, n_top, n_pick, n_blk, n_extra; + uint32_t has_score, has_blocks, mask_resident; +}; + +struct htp_top_k_mask_vtcm_layout { + size_t total_bytes; + size_t off_blocks, off_mask, off_rowin, off_tiles, off_mtiles; // 0 when unused + size_t off_scratch; // HTP_TOP_K_MASK_SCRATCH_BYTES per thread + size_t rowin_bytes; // one slot of top_k + extra + score row, parts at htp_top_k_mask_rowin_off_* + size_t tile_bytes; // one output tile slot, 128B aligned + size_t blocks_bytes, mask_bytes; // exact DMA sizes, 0 when unused +}; + +static inline uint64_t htp_top_k_mask_align128(uint64_t n) { + return (n + 127) & ~(uint64_t) 127; +} + +// Parts of one row input slot: top_k at 0, then extra, then score; used by the host layout +static inline uint64_t htp_top_k_mask_rowin_off_extra(uint32_t n_top) { + return htp_top_k_mask_align128((uint64_t) n_top * 4); +} + +static inline uint64_t htp_top_k_mask_rowin_off_score(uint32_t n_top, uint32_t n_extra) { + return htp_top_k_mask_rowin_off_extra(n_top) + htp_top_k_mask_align128((uint64_t) n_extra * 4); +} + +// the caller compares total_bytes against the VTCM size +static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_layout * L, + const struct htp_top_k_mask_shape * S, uint32_t kernel_type, uint32_t n_hvx, uint32_t tile_cols) { + memset(L, 0, sizeof(*L)); + + // PICKS streams the mask straight into its row tile + const bool picks = kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS; + const uint64_t blocks_exact = S->has_blocks ? (uint64_t) S->n_blk * S->n_pick * 4 : 0; + const uint64_t blocks_bytes = htp_top_k_mask_align128(blocks_exact); + const uint64_t mask_exact = S->mask_resident && !picks ? (uint64_t) S->n_kv * S->elem_size : 0; + const uint64_t mask_bytes = htp_top_k_mask_align128(mask_exact); + const uint64_t rowin_bytes = htp_top_k_mask_rowin_off_score(S->n_top, S->n_extra) + + (S->has_score ? htp_top_k_mask_align128((uint64_t) S->n_pick * 4) : 0); + const uint64_t tile_bytes = htp_top_k_mask_align128((uint64_t) tile_cols * S->elem_size); + + // ROWS streams row inputs into 2 slots per thread. PICKS: one row, a row tile and a flag tile. + const uint64_t rowin_total = (picks ? 1 : 2 * (uint64_t) n_hvx) * rowin_bytes; + const uint64_t tiles_total = picks ? 2 * tile_bytes : 2 * (uint64_t) n_hvx * tile_bytes; + const uint64_t mtiles_total = S->mask_resident ? 0 : tiles_total; + const uint64_t scratch = (uint64_t) n_hvx * HTP_TOP_K_MASK_SCRATCH_BYTES; + + size_t off = 0; + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_blocks, (size_t) blocks_bytes, blocks_bytes > 0); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_mask, (size_t) mask_bytes, mask_bytes > 0); + VTCM_LAYOUT_ALLOC(off, off_rowin, (size_t) rowin_total); + VTCM_LAYOUT_ALLOC(off, off_tiles, (size_t) tiles_total); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_mtiles, (size_t) mtiles_total, !S->mask_resident); + VTCM_LAYOUT_ALLOC(off, off_scratch, (size_t) scratch); + + L->rowin_bytes = (size_t) rowin_bytes; + L->tile_bytes = (size_t) tile_bytes; + L->blocks_bytes = (size_t) blocks_exact; + L->mask_bytes = (size_t) mask_exact; + L->total_bytes = off; +} + +// Set by the host, used as is by the NPU: no tensor shape, stride or layout is derived on the NPU. +// Sizes of one slot or tile, the slot parts and the VTCM offsets are in HTP_TOP_K_MASK_UNIT bytes as u16. +// Row strides: byte offset of flat row r of top_k, extra, score and dst is r * stride (the host requires row flattenable inputs). +// PICKS splits the work in n_hvx virtual workers; fewer real workers loop over them. +struct htp_top_k_mask_kernel_params { + uint8_t n_hvx, kernel_type, flags, elem_size; + uint32_t vtcm_size; + uint32_t n_kv, n_top, n_pick, n_extra; + uint32_t ninf; // -inf cell pattern of elem_size + uint32_t blocks_bytes, score_bytes; // exact DMA sizes, gather regions are [0, bytes) + uint32_t tile_cols, n_col_tiles; + uint16_t n_blk, blocks_nb1; // blocks_nb1 = n_blk * 4, fits a signed 16-bit multiply + uint16_t off_blocks_u, off_mask_u, off_rowin_u, off_tiles_u, off_mtiles_u, off_scratch_u; + uint16_t rowin_units, tile_units, off_extra_units, off_score_units; + union { + struct { + uint32_t total_rows; + uint32_t rs_top, rs_extra, rs_score, rs_dst; // flat row strides (bytes of row 1) + uint32_t mask_nb1; // mask row stride, rows of the mask are dense above it + union { + uint32_t mask_bytes; // resident mask (1 row) + struct { // broadcast mask: row r -> (i1, i2, i3) -> mask row (i mod mne) + uint32_t mp[5]; // fastdiv multipliers of dst ne1, ne2 and mask ne1, ne2, ne3 + uint16_t ne[5]; // the divisors + uint8_t l[5]; // fastdiv shifts + } dec; + } m; + } rows; + struct { + uint32_t picks_per_hvx; // per virtual worker, multiple of the lanes of one vector + uint16_t blocks_slice_u, top_slice_u, extra_slice_u, score_slice_u, flags_slice_u; // per virtual worker slices + } picks; + } u; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_top_k_mask_kernel_params) <= 128, "htp_top_k_mask_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_top_k_mask_kernel_params) <= 128, "htp_top_k_mask_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_TOP_K_MASK_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index b63fd4c29..19ade22f3 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -21,7 +21,7 @@ #include "htp-ops.h" #include "htp-tensor.h" #include "htp-vtcm.h" -#include "hex-profile.h" +#include "htp-trace.h" struct htp_unary_context; @@ -36,6 +36,13 @@ typedef void (*unary_rms_norm_mul_compute_fn_t)(const void * restrict src, uint32_t num_rows, const struct htp_unary_context * uctx); +typedef void (*unary_rms_norm_mul_add_compute_fn_t)(const void * restrict src, + const void * restrict weight, + const void * restrict add, + void * restrict dst, + uint32_t num_rows, + const struct htp_unary_context * uctx); + typedef void (*unary_tri_compute_fn_t)(const void * restrict src, void * restrict dst, uint32_t num_rows, @@ -63,6 +70,7 @@ struct htp_unary_context { dma_addr_t data_src0; dma_addr_t data_src1; // weight/scale tensor for RMS_NORM_MUL + dma_addr_t data_src2; // residual tensor for RMS_NORM_MUL_ADD dma_addr_t data_dst; size_t src0_data_row_size; // actual data bytes per row @@ -79,7 +87,7 @@ struct htp_unary_context { uint32_t block; uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; uint32_t nc; uint32_t col_tile; // tiled mode @@ -87,11 +95,13 @@ struct htp_unary_context { uint8_t * vtcm_src0; uint8_t * vtcm_src1; + uint8_t * vtcm_src2; uint8_t * vtcm_dst; - size_t vtcm_src0_size_per_thread; - size_t vtcm_src1_size_per_thread; - size_t vtcm_dst_size_per_thread; + size_t vtcm_src0_size_per_hvx; + size_t vtcm_src1_size_per_hvx; + size_t vtcm_src2_size_per_hvx; + size_t vtcm_dst_size_per_hvx; }; // Convert flat row index to DDR byte offset using the tensor's actual strides. @@ -109,18 +119,17 @@ static inline size_t unary_row_offset(uint32_t ir, return i1 * nb1 + i2 * nb2 + i3 * nb3; } -// Safe DMA block size from row `ir`: clamp to the tighter dim-1 slice -// boundary of src and dst so the nb1 stride stays valid for all rows. +// Safe DMA block size from row `ir`: clamp to the dim-1 slice boundary +// when non-contiguous so the nb1 stride stays valid for all rows. static inline uint32_t unary_block_size(uint32_t ir, uint32_t end_row, uint32_t block, - bool src_contig, - bool dst_contig, + bool contig, uint32_t ne1, const struct fastdiv_values * div_ne1) { uint32_t limit = MIN(block, end_row - ir); - if (!src_contig || !dst_contig) { + if (!contig) { const uint32_t slice_end = (fastdiv(ir, div_ne1) + 1) * ne1; limit = MIN(limit, slice_end - ir); } @@ -213,13 +222,15 @@ static void rms_norm_f32(const void * restrict src, const struct htp_unary_context * uctx) { htp_unary_op_preamble; float epsilon = 0.f; - memcpy(&epsilon, op_params, sizeof(float)); + float scale = 0.f; + memcpy(&epsilon, &op_params[0], sizeof(float)); + memcpy(&scale, &op_params[1], sizeof(float)); for (uint32_t ir = 0; ir < num_rows; ir++) { const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon); + hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon, scale); } } @@ -230,14 +241,39 @@ static void rms_norm_mul_f32(const void * restrict src, const struct htp_unary_context * uctx) { htp_unary_op_preamble; float epsilon = 0.f; - memcpy(&epsilon, op_params, sizeof(float)); + float scale = 0.f; + memcpy(&epsilon, &op_params[0], sizeof(float)); + memcpy(&scale, &op_params[1], sizeof(float)); for (uint32_t ir = 0; ir < num_rows; ir++) { const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); const uint8_t * restrict w_local = (const uint8_t *)weight + (uctx->broadcast_weight ? 0 : ir * uctx->src1_row_size_aligned); uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon); + hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon, scale); + } +} + +static void rms_norm_mul_add_f32(const void * restrict src, + const void * restrict weight, + const void * restrict add, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + float scale = 0.f; + memcpy(&epsilon, &op_params[0], sizeof(float)); + memcpy(&scale, &op_params[1], sizeof(float)); + + const uint8_t * restrict w_local = (const uint8_t *) weight; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *) src + (ir * src0_row_size_aligned); + const uint8_t * restrict add_local = (const uint8_t *) add + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *) dst + (ir * dst_row_size_aligned); + + hvx_fast_rms_norm_mul_add_f32(src_local, w_local, add_local, dst_local, ne0, epsilon, scale); } } @@ -423,6 +459,34 @@ static void step_f16(const void * restrict src, } } +static void ceil_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_ceil_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void sgn_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sgn_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + static void l2_norm_f16(const void * restrict src, void * restrict dst, const uint32_t num_rows, @@ -497,7 +561,72 @@ static void silu_f32(const void * restrict src, } } -// gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference) +// GELU uses the tanh approximation. +static __attribute__((noinline)) HVX_Vector hvx_vec_gelu_f32(HVX_Vector x) { + const HVX_Vector half = hvx_vec_splat_f32(0.5f); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + + HVX_Vector inner = hvx_vec_mul_f32_f32(x, x); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.044715f)); + inner = hvx_vec_add_f32_f32(inner, one); + inner = hvx_vec_mul_f32_f32(inner, x); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.7978845608028654f)); + + return hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(half, x), + hvx_vec_add_f32_f32(one, hvx_vec_tanh_f32(inner))); +} + +static inline void hvx_gelu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + + uint32_t i = 0; + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_gelu_f32(vsrc[i]); + } + if (nloe) { + hvx_vec_store_a(&vdst[i], nloe * sizeof(float), hvx_vec_gelu_f32(vsrc[i])); + } +} + +// GELU_QUICK uses x * sigmoid(1.702 * x). +static __attribute__((noinline)) HVX_Vector hvx_vec_gelu_quick_f32(HVX_Vector x) { + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + const HVX_Vector max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.0f); + const HVX_Vector scaled = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(1.702f)); + const HVX_Vector sigmoid = hvx_vec_fast_sigmoid_f32_guard(scaled, one, max_exp, min_exp); + + return hvx_vec_mul_f32_f32(x, sigmoid); +} + +static inline void hvx_gelu_quick_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + + uint32_t i = 0; + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_gelu_quick_f32(vsrc[i]); + } + if (nloe) { + hvx_vec_store_a(&vdst[i], nloe * sizeof(float), hvx_vec_gelu_quick_f32(vsrc[i])); + } +} + static void gelu_f32(const void * restrict src, void * restrict dst, const uint32_t num_rows, @@ -508,9 +637,21 @@ static void gelu_f32(const void * restrict src, const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_mul_scalar_f32(dst_local, src_local, 1.702f, ne0); - hvx_sigmoid_f32_aa(dst_local, dst_local, ne0); - hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0); + hvx_gelu_f32_aa(dst_local, src_local, ne0); + } +} + +static void gelu_quick_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_gelu_quick_f32_aa(dst_local, src_local, ne0); } } @@ -540,10 +681,11 @@ static void tri_f32(const void * restrict src, const uint32_t nloe = ne0 % VLEN_FP32; const uint32_t ne01 = uctx->octx->src[0]->ne[1]; + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; for (uint32_t b = 0; b < num_rows; b++) { const uint32_t abs_row = ir + b; - const uint32_t i01 = abs_row % ne01; + const uint32_t i01 = fastmodulo(abs_row, ne01, div_ne01); const HVX_Vector * restrict v_src = (const HVX_Vector *) ((const uint8_t *) src + b * src0_row_size_aligned); HVX_Vector * restrict v_dst = (HVX_Vector *) ((uint8_t *) dst + b * dst_row_size_aligned); @@ -613,22 +755,48 @@ static void tri_f32(const void * restrict src, } } +// softplus(x) = log(1 + exp(x)) +// Match CPU reference: return x for x > 20 to avoid overflow. +static inline void hvx_softplus_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const HVX_Vector twenty = hvx_vec_splat_f32(20.0f); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector x = v_src[i]; + HVX_VectorPred pred_gt20 = Q6_Q_vcmp_gt_VsfVsf(x, twenty); + HVX_Vector expx = hvx_vec_exp_f32(x); + HVX_Vector logv = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx)); + v_dst[i] = Q6_V_vmux_QVV(pred_gt20, x, logv); + } + + if (nloe > 0) { + HVX_Vector x = v_src[nvec]; + HVX_VectorPred pred_gt20 = Q6_Q_vcmp_gt_VsfVsf(x, twenty); + HVX_Vector expx = hvx_vec_exp_f32(x); + HVX_Vector logv = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx)); + HVX_Vector sp = Q6_V_vmux_QVV(pred_gt20, x, logv); + hvx_vec_store_a((void *) &v_dst[nvec], nloe * sizeof(float), sp); + } +} + static void softplus_f32(const void * restrict src, void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; - // softplus(x) = log(1 + exp(x)) - // Match CPU reference: ggml_compute_softplus_f32() in ggml-impl.h + for (uint32_t ir = 0; ir < num_rows; ir++) { - const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned)); - float * restrict dst_f = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned)); + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - for (uint32_t i = 0; i < ne0; i++) { - float x = src_f[i]; - // For x > 20: softplus(x) ~ x (avoids exp overflow) - dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); - } + hvx_softplus_f32_aa(dst_local, src_local, ne0); } } @@ -648,6 +816,143 @@ static void l2_norm_f32(const void * restrict src, } } +static inline HVX_VectorPair hvx_fwht_stage_pair(HVX_Vector v0, HVX_Vector v1, int32_t b) { + HVX_VectorPair p = Q6_W_vshuff_VVR(v1, v0, b); + HVX_Vector s = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p)); + HVX_Vector d = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p)); + return Q6_W_vshuff_VVR(d, s, b); +} + +static inline HVX_VectorPair hvx_fwht32_pair(HVX_Vector v0_sf, HVX_Vector v1_sf, HVX_Vector v_scale) { + HVX_Vector v0 = Q6_Vqf32_vmpy_VsfVsf(v0_sf, v_scale); + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(v1_sf, v_scale); + HVX_VectorPair p; + p = hvx_fwht_stage_pair(v0, v1, 4); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 8); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 16); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 32); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 64); + return p; +} + +static inline void hvx_fwht_store32(HVX_Vector * restrict v_dst, HVX_VectorPair p) { + v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); +} + +static inline void hvx_fwht_store64(HVX_Vector * restrict v_dst, HVX_VectorPair p) { + HVX_Vector u = Q6_V_lo_W(p); + HVX_Vector v = Q6_V_hi_W(p); + v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(u, v)); + v_dst[1] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(u, v)); +} + +static inline void hvx_fwht_tail_f32(HVX_Vector * restrict v_dst, uint32_t nvec) { + uint32_t len_vec = 2; + for (; len_vec * 2 < nvec; len_vec <<= 2) { + const uint32_t step = len_vec * 4; + for (uint32_t i = 0; i < nvec; i += step) { + for (uint32_t j = 0; j < len_vec; j++) { + HVX_Vector a0 = v_dst[i + j]; + HVX_Vector a1 = v_dst[i + j + len_vec]; + HVX_Vector a2 = v_dst[i + j + 2 * len_vec]; + HVX_Vector a3 = v_dst[i + j + 3 * len_vec]; + + HVX_Vector t0 = Q6_Vqf32_vadd_VsfVsf(a0, a1); + HVX_Vector t1 = Q6_Vqf32_vsub_VsfVsf(a0, a1); + HVX_Vector t2 = Q6_Vqf32_vadd_VsfVsf(a2, a3); + HVX_Vector t3 = Q6_Vqf32_vsub_VsfVsf(a2, a3); + + v_dst[i + j] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t0, t2)); + v_dst[i + j + len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t1, t3)); + v_dst[i + j + 2 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t0, t2)); + v_dst[i + j + 3 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t1, t3)); + } + } + } + + if (len_vec < nvec) { + for (uint32_t i = 0; i < nvec; i += 2 * len_vec) { + for (uint32_t j = 0; j < len_vec; j++) { + HVX_Vector u = v_dst[i + j]; + HVX_Vector v = v_dst[i + len_vec + j]; + v_dst[i + j] = hvx_vec_add_f32_f32(u, v); + v_dst[i + len_vec + j] = hvx_vec_sub_f32_f32(u, v); + } + } + } +} + +static inline void hvx_fwht_row_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const uint32_t nvec = n / VLEN_FP32; + + if (nvec == 1) { + HVX_VectorPair p = hvx_fwht32_pair(v_src[0], v_src[0], v_scale); + hvx_fwht_store32(v_dst, p); + return; + } + + for (uint32_t i = 0; i < nvec; i += 2) { + HVX_VectorPair p = hvx_fwht32_pair(v_src[i], v_src[i + 1], v_scale); + hvx_fwht_store64(&v_dst[i], p); + } + + hvx_fwht_tail_f32(v_dst, nvec); +} + +static inline void hvx_fwht_row_f16(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const uint32_t nvec = n / VLEN_FP32; + + if (nvec == 1) { + HVX_Vector v0 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src[0])); + HVX_VectorPair p = hvx_fwht32_pair(v0, v0, v_scale); + hvx_fwht_store32(v_dst, p); + return; + } + + const uint32_t nvec_f16 = n / VLEN_FP16; + for (uint32_t i = 0; i < nvec_f16; i++) { + HVX_VectorPair f32p = hvx_vec_f16_to_f32(v_src[i]); + HVX_VectorPair p = hvx_fwht32_pair(Q6_V_lo_W(f32p), Q6_V_hi_W(f32p), v_scale); + hvx_fwht_store64(&v_dst[2 * i], p); + } + + hvx_fwht_tail_f32(v_dst, nvec); +} + +static void hadamard_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned); + + hvx_fwht_row_f32(src_row, dst_row, ne0, v_scale); + } +} + +static void hadamard_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned); + + hvx_fwht_row_f16(src_row, dst_row, ne0, v_scale); + } +} + static void tanh_f32(const void * restrict src, void * restrict dst, const uint32_t num_rows, @@ -704,6 +1009,34 @@ static void step_f32(const void * restrict src, } } +static void ceil_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_ceil_f32_aa(dst_local, src_local, ne0); + } +} + +static void sgn_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sgn_f32_aa(dst_local, src_local, ne0); + } +} + static void log_f32(const void * restrict src, void * restrict dst, const uint32_t num_rows, @@ -774,9 +1107,12 @@ static void tile_silu_f32(void * restrict dst, const void * restrict src, uint32 static void tile_gelu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { (void) uctx; - hvx_mul_scalar_f32((uint8_t *) dst, (const uint8_t *) src, 1.702f, tw); - hvx_sigmoid_f32_aa((uint8_t *) dst, (uint8_t *) dst, tw); - hvx_mul_f32_aaa((uint8_t *) dst, (const uint8_t *) src, (uint8_t *) dst, tw); + hvx_gelu_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_gelu_quick_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_gelu_quick_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); } static void tile_gelu_erf_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { @@ -786,12 +1122,7 @@ static void tile_gelu_erf_f32(void * restrict dst, const void * restrict src, ui static void tile_softplus_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { (void) uctx; - const float * restrict sf = (const float *) src; - float * restrict df = (float *) dst; - for (uint32_t i = 0; i < tw; i++) { - float x = sf[i]; - df[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); - } + hvx_softplus_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); } static void tile_tanh_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { @@ -819,6 +1150,16 @@ static void tile_step_f32(void * restrict dst, const void * restrict src, uint32 hvx_step_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); } +static void tile_ceil_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_ceil_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_sgn_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sgn_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + static void tri_apply_tile_f32(const void * restrict src, void * restrict dst, uint32_t tile_elems, uint32_t col_start, uint32_t i01, uint32_t ne0, int32_t ttype) { @@ -892,25 +1233,25 @@ static void tri_apply_tile_f32(const void * restrict src, void * restrict dst, } // 1. Standard row-block unary task (F32 and F16). -static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_row_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -919,41 +1260,29 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && - (nb03 == (size_t)ne02 * nb02); - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && - (nb3 == (size_t)ne2 * nb2); + const bool src0_contig = uctx->kparams->src0_contig; + const uint32_t BLOCK = uctx->block; const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); - const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); - const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); - if (BLOCK == 0) { - FARF(ERROR, "unary-row-block : current VTCM reservation %zu is too small, needed at least %zu\n", - uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { - const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, - ne01, div_ne01); + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), nb1, dst_row_size_aligned, dst_data_row_size, 0); - const size_t src0_off = src0_contig ? (ir * nb01) : + const size_t src0_off = src0_contig ? ((size_t) ir * nb01) : unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), @@ -965,8 +1294,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da unary_compute_fn_t compute = (unary_compute_fn_t) uctx->compute; for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { - const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, - ne01, div_ne01); + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; @@ -975,8 +1303,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da compute(src0_vtcm, dst_vtcm, block_size, uctx); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); - const size_t dst_off = dst_contig ? (ir * nb1) : - unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3); + const size_t dst_off = (size_t) ir * nb1; dma_queue_push(dma_q, dma_make_data(data_dst + dst_off, dst_vtcm), nb1, dst_row_size_aligned, dst_data_row_size, block_size); @@ -984,12 +1311,12 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da const uint32_t next_ir = ir + block_size; if (next_ir < src0_end_row) { const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, - dst_contig, ne01, div_ne01); + ne01, div_ne01); const uint32_t pref_ir = next_ir + next_block_size; if (pref_ir < src0_end_row) { const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, - dst_contig, ne01, div_ne01); - const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : + ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) : unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); dma_queue_push(dma_q, dma_make_data(src0_vtcm, data_src + src0_pref_off), @@ -1003,25 +1330,25 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da } // 2. RMS_NORM_MUL row-block task with weight buffer. -static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_rms_norm_mul_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1035,42 +1362,25 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo const uint32_t nb11 = src1->nb[1]; const uint32_t nb12 = src1->nb[2]; const uint32_t nb13 = src1->nb[3]; - const uint32_t nb11_bc = (src1->ne[1] > 1) ? nb11 : 0; - const uint32_t nb12_bc = (src1->ne[2] > 1) ? nb12 : 0; - const uint32_t nb13_bc = (src1->ne[3] > 1) ? nb13 : 0; - const bool src1_contig = ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12)); - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL; - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_hvx)) : NULL; + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size; const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && - (nb03 == (size_t)ne02 * nb02); - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && - (nb3 == (size_t)ne2 * nb2); + const bool src0_contig = uctx->kparams->src0_contig; + const bool src1_contig = uctx->kparams->src1_contig; + const bool block_contig = src0_contig && src1_contig; + const uint32_t BLOCK = uctx->block; const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - const bool src1_needs_row_clip = !uctx->broadcast_weight && !src1_contig; - const bool block_src0_contig = src0_contig && !src1_needs_row_clip; - const bool block_dst_contig = dst_contig && !src1_needs_row_clip; - - const uint32_t src0_max_block = block_src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); - const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); - const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); - if (BLOCK == 0) { - FARF(ERROR, "unary-rms-norm-mul : current VTCM reservation %zu is too small, needed at least %zu\n", - uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; if (uctx->broadcast_weight) { dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1), @@ -1079,22 +1389,21 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo } for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { - const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, - ne01, div_ne01); + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_contig, ne01, div_ne01); dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), nb1, dst_row_size_aligned, dst_data_row_size, 0); - const size_t src0_off = src0_contig ? (ir * nb01) : + const size_t src0_off = src0_contig ? ((size_t) ir * nb01) : unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), src0_row_size_aligned, nb01, src0_data_row_size, block_size); if (!uctx->broadcast_weight) { - const size_t src1_off = src1_contig ? (ir * nb11) : - unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, nb13_bc); + const size_t src1_off = src1_contig ? ((size_t) ir * nb11) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); dma_queue_push(dma_q, dma_make_data(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off), uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size); @@ -1106,8 +1415,7 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo unary_rms_norm_mul_compute_fn_t compute = (unary_rms_norm_mul_compute_fn_t) uctx->compute; for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { - const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, - ne01, div_ne01); + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_contig, ne01, div_ne01); void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; @@ -1121,30 +1429,28 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo compute(src0_vtcm, w, dst_vtcm, block_size, uctx); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); - const size_t dst_off = dst_contig ? (ir * nb1) : - unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3); + const size_t dst_off = (size_t) ir * nb1; dma_queue_push(dma_q, dma_make_data(data_dst + dst_off, dst_vtcm), nb1, dst_row_size_aligned, dst_data_row_size, block_size); const uint32_t next_ir = ir + block_size; if (next_ir < src0_end_row) { - const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_src0_contig, - block_dst_contig, ne01, div_ne01); + const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_contig, + ne01, div_ne01); const uint32_t pref_ir = next_ir + next_block_size; if (pref_ir < src0_end_row) { - const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_src0_contig, - block_dst_contig, ne01, div_ne01); - const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : + const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_contig, + ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) : unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); dma_queue_push(dma_q, dma_make_data(src0_vtcm, data_src + src0_pref_off), src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); if (!uctx->broadcast_weight) { - const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) : - unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, - nb13_bc); + const size_t src1_pref_off = src1_contig ? ((size_t) pref_ir * nb11) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); dma_queue_push(dma_q, dma_make_data(src1_vtcm, data_src1 + src1_pref_off), uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size); @@ -1157,92 +1463,90 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo dma_queue_flush(dma_q); } -// 3. TRI row-block task with row index ir. -static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data) { +// 2b. RMS_NORM_MUL_ADD row-block task with broadcast weight and residual add. +static void unary_thread_rms_norm_mul_add_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; - const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; } - const dma_addr_t data_src = uctx->data_src0; - const dma_addr_t data_dst = uctx->data_dst; + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_src1 = uctx->data_src1; + const dma_addr_t data_src2 = uctx->data_src2; + const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_data = uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_hvx); + uint8_t * src2_vtcm_data = uctx->vtcm_src2 + (ith * uctx->vtcm_src2_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && - (nb03 == (size_t)ne02 * nb02); - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && - (nb3 == (size_t)ne2 * nb2); + const bool src0_contig = uctx->kparams->src0_contig; + const uint32_t BLOCK = uctx->block; const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); - const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); - const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); - if (BLOCK == 0) { - FARF(ERROR, "unary-tri : current VTCM reservation %zu is too small, needed at least %zu\n", - uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); - return; - } + dma_queue * dma_q = th->dma_q; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1), + uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1); + dma_queue_flush(dma_q); for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { - const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, - ne01, div_ne01); + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), nb1, dst_row_size_aligned, dst_data_row_size, 0); - const size_t src0_off = src0_contig ? (ir * nb01) : + const size_t src0_off = src0_contig ? ((size_t) ir * nb01) : unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), src0_row_size_aligned, nb01, src0_data_row_size, block_size); + dma_queue_push(dma_q, + dma_make_data(src2_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src2 + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); ir += block_size; } - unary_tri_compute_fn_t compute = (unary_tri_compute_fn_t) uctx->compute; + unary_rms_norm_mul_add_compute_fn_t compute = (unary_rms_norm_mul_add_compute_fn_t) uctx->compute; for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { - const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, - ne01, div_ne01); + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + void * src2_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); - compute(src0_vtcm, dst_vtcm, block_size, ir, uctx); + compute(src0_vtcm, src1_vtcm_data, src2_vtcm, dst_vtcm, block_size, uctx); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); - const size_t dst_off = dst_contig ? (ir * nb1) : - unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3); + const size_t dst_off = (size_t) ir * nb1; dma_queue_push(dma_q, dma_make_data(data_dst + dst_off, dst_vtcm), nb1, dst_row_size_aligned, dst_data_row_size, block_size); @@ -1250,16 +1554,106 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data const uint32_t next_ir = ir + block_size; if (next_ir < src0_end_row) { const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, - dst_contig, ne01, div_ne01); + ne01, div_ne01); const uint32_t pref_ir = next_ir + next_block_size; if (pref_ir < src0_end_row) { const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, - dst_contig, ne01, div_ne01); - const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : + ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) : unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); dma_queue_push(dma_q, dma_make_data(src0_vtcm, data_src + src0_pref_off), src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + dma_queue_push(dma_q, + dma_make_data(src2_vtcm, data_src2 + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + } + } + ir += block_size; + } + + dma_queue_flush(dma_q); +} + +// 3. TRI row-block task with row index ir. +static void unary_thread_tri_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = th->trace; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; + const size_t src0_data_row_size = uctx->src0_data_row_size; + const size_t dst_data_row_size = uctx->dst_data_row_size; + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); + + const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; + const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; + const uint32_t BLOCK = uctx->block; + + dma_queue * dma_q = th->dma_q; + + for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); + + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), + nb1, dst_row_size_aligned, dst_data_row_size, 0); + + const size_t src0_off = (size_t) ir * nb01; + dma_queue_push(dma_q, + dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + + ir += block_size; + } + + unary_tri_compute_fn_t compute = (unary_tri_compute_fn_t) uctx->compute; + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); + + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + compute(src0_vtcm, dst_vtcm, block_size, ir, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + const size_t dst_off = (size_t) ir * nb1; + dma_queue_push(dma_q, + dma_make_data(data_dst + dst_off, dst_vtcm), + nb1, dst_row_size_aligned, dst_data_row_size, block_size); + + const uint32_t next_ir = ir + block_size; + if (next_ir < src0_end_row) { + const uint32_t next_block_size = MIN(BLOCK, src0_end_row - next_ir); + const uint32_t pref_ir = next_ir + next_block_size; + if (pref_ir < src0_end_row) { + const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_ir); + const size_t src0_pref_off = (size_t) pref_ir * nb01; + dma_queue_push(dma_q, + dma_make_data(src0_vtcm, data_src + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); } } ir += block_size; @@ -1269,22 +1663,22 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data } // 4. Pointwise tiled unary task. -static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const uint32_t col_tile = uctx->col_tile; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1293,34 +1687,30 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_half = uctx->src0_vtcm_half_size; const size_t dst_half = uctx->dst_vtcm_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; - const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; - - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && - (nb03 == (size_t)ne02 * nb02); - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && - (nb3 == (size_t)ne2 * nb2); + const uint32_t tiles_per_row = uctx->kparams->tiles_per_row; + const bool src0_contig = uctx->kparams->src0_contig; const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { - const uint32_t row = src0_start_row + t / tiles_per_row; - const uint32_t col = (t % tiles_per_row) * col_tile; + const uint32_t row = src0_start_row + fastdiv(t, div_tpr); + const uint32_t col = fastmodulo(t, tiles_per_row, div_tpr) * col_tile; const uint32_t tw = MIN(col_tile, ne0 - col); const size_t tb = (size_t) tw * sizeof(float); - const size_t soff = (src0_contig ? (row * nb01) : + const size_t soff = (src0_contig ? ((size_t) row * nb01) : unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + (size_t) col * sizeof(float); @@ -1348,9 +1738,7 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) compute(dst_vtcm, src_vtcm, tw, uctx); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); - const size_t doff = (dst_contig ? (row * nb1) : - unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + - (size_t) col * sizeof(float); + const size_t doff = (size_t) row * nb1 + (size_t) col * sizeof(float); const size_t tb = (size_t) tw * sizeof(float); dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1); @@ -1358,7 +1746,7 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) if (pt < total_tiles) { const uint32_t ptw = MIN(col_tile, ne0 - pcol); const size_t ptb = (size_t) ptw * sizeof(float); - const size_t psoff = (src0_contig ? (prow * nb01) : + const size_t psoff = (src0_contig ? ((size_t) prow * nb01) : unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + (size_t) pcol * sizeof(float); @@ -1386,23 +1774,23 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) } // 5. TRI tiled task. -static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_tiled_tri_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const int32_t * op_params = octx->op_params; const uint32_t col_tile = uctx->col_tile; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1411,37 +1799,28 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_half = uctx->src0_vtcm_half_size; const size_t dst_half = uctx->dst_vtcm_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; - const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; - const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; - const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; - const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; + const uint32_t tiles_per_row = uctx->kparams->tiles_per_row; const int32_t tri_ttype = op_params[0]; - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && - (nb03 == (size_t)ne02 * nb02); - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && - (nb3 == (size_t)ne2 * nb2); - const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { - const uint32_t row = src0_start_row + t / tiles_per_row; - const uint32_t col = (t % tiles_per_row) * col_tile; + const uint32_t row = src0_start_row + fastdiv(t, div_tpr); + const uint32_t col = fastmodulo(t, tiles_per_row, div_tpr) * col_tile; const uint32_t tw = MIN(col_tile, ne0 - col); const size_t tb = (size_t) tw * sizeof(float); - const size_t soff = (src0_contig ? (row * nb01) : - unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + - (size_t) col * sizeof(float); + const size_t soff = (size_t) row * nb01 + (size_t) col * sizeof(float); dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1); @@ -1468,20 +1847,15 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void compute(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); - const size_t doff = (dst_contig ? (row * nb1) : - unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + - (size_t) col * sizeof(float); + const size_t doff = (size_t) row * nb1 + (size_t) col * sizeof(float); const size_t tb = (size_t) tw * sizeof(float); dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1); const uint32_t pt = t + 2; if (pt < total_tiles) { - const uint32_t ptw = MIN(col_tile, ne0 - pcol); - const size_t ptb = (size_t) ptw * sizeof(float); - const size_t psoff = (src0_contig ? (prow * nb01) : - unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, - nb03)) + - (size_t) pcol * sizeof(float); + const uint32_t ptw = MIN(col_tile, ne0 - pcol); + const size_t ptb = (size_t) ptw * sizeof(float); + const size_t psoff = (size_t) prow * nb01 + (size_t) pcol * sizeof(float); dma_queue_push(dma_q, dma_make_data(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); } @@ -1520,9 +1894,10 @@ static int execute_op_unary(struct htp_ops_context * octx) { const char * op_type = NULL; switch (octx->op) { - case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break; - case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break; - case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; + case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break; + case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break; + case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; + case HTP_OP_RMS_NORM_MUL_ADD: op_type = "rmsnorm-mul-add-f32"; break; case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break; case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break; case HTP_OP_LEAKY_RELU: op_type = "leaky-relu-f32"; break; @@ -1533,6 +1908,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break; case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break; case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break; + case HTP_OP_UNARY_GELU_QUICK: op_type = "gelu-quick-f32"; break; case HTP_OP_UNARY_GELU_ERF: op_type = "gelu-erf-f32"; break; case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break; case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; @@ -1540,51 +1916,34 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_LOG: op_type = is_f16 ? "log-f16" : "log-f32"; break; case HTP_OP_UNARY_RELU: op_type = "relu-f32"; break; case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break; + case HTP_OP_UNARY_CEIL: op_type = is_f16 ? "ceil-f16" : "ceil-f32"; break; + case HTP_OP_UNARY_SGN: op_type = is_f16 ? "sgn-f16" : "sgn-f32"; break; case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break; case HTP_OP_TRI: op_type = "tri-f32"; break; + case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break; default: FARF(ERROR, "Unsupported unary Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; } - // F16 only has row-block kernels for this subset of ops (see the dispatch switch - // below) - reject everything else up front, before touching kparams/VTCM. - if (is_f16) { - switch (octx->op) { - case HTP_OP_NORM: - case HTP_OP_RMS_NORM: - case HTP_OP_SCALE: - case HTP_OP_CLAMP: - case HTP_OP_SQR: - case HTP_OP_SQRT: - case HTP_OP_L2_NORM: - case HTP_OP_UNARY_ABS: - case HTP_OP_UNARY_LOG: - case HTP_OP_UNARY_STEP: - break; - default: - FARF(ERROR, "unary-%s: not supported for F16\n", op_type); - return HTP_STATUS_NO_SUPPORT; - } - } - const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } + const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); + const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float); const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); - const size_t src0_data_row_size = src0->ne[0] * elem_size; - const size_t dst_data_row_size = dst->ne[0] * elem_size; + const size_t src0_data_row_size = src0->ne[0] * src0_elem_size; + const size_t dst_data_row_size = dst->ne[0] * dst_elem_size; uint32_t row_start = 0; uint32_t nrows = src0_nrows; if (octx->ctx->mdev.count > 1) { uint32_t rows_per_chunk = 0; - htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst_elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk); const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); row_start = range.start; nrows = range.count; @@ -1594,29 +1953,28 @@ static int execute_op_unary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; - const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; - const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; - - // Always 0 for F16 - htp_unary_vtcm_layout_build() keeps F16 on the row-block path, + // Always 0 for F16 - htp_unary_solve_layout() keeps F16 on the row-block path, // since only F32 has unary_task_f32_tiled_* kernels. const uint32_t col_tile = kparams->col_tile; size_t src1_data_row_size = 0; - size_t src1_row_size_aligned = kparams->src1_row_size_aligned; bool broadcast_weight = kparams->broadcast_weight; const struct htp_tensor * src1 = NULL; - // RMS_NORM_MUL fusion is F32-only (its weight tensor is always F32; see + const bool has_weight = (octx->op == HTP_OP_RMS_NORM_MUL || octx->op == HTP_OP_RMS_NORM_MUL_ADD); + const bool has_src2 = (octx->op == HTP_OP_RMS_NORM_MUL_ADD); + + // RMS_NORM_MUL* fusion is F32-only (its weight tensor is always F32; see // try_fuse_node()'s type guard), so this never triggers when is_f16 is true. - if (octx->op == HTP_OP_RMS_NORM_MUL) { + if (has_weight) { src1 = octx->src[1]; src1_data_row_size = src1->ne[0] * sizeof(float); } - if (octx->ctx->vtcm_size < (size_t)kparams->vtcm_size) { - FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t)kparams->vtcm_size); + if ((!col_tile && kparams->block == 0) || octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) { + FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t) kparams->vtcm_size); return HTP_STATUS_VTCM_TOO_SMALL; } @@ -1624,51 +1982,58 @@ static int execute_op_unary(struct htp_ops_context * octx) { octx->src1_spad.src = NULL; octx->dst_spad.src = NULL; - FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %u src1-vtcm-size %u dst-vtcm-size %u\n", op_type, + struct htp_unary_vtcm_layout L; + htp_unary_vtcm_layout_build(&L, octx->op, src0_data_row_size, dst_data_row_size, src1_data_row_size, + src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_hvx); + + FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - kparams->vtcm_src0_size, kparams->vtcm_src1_size, kparams->vtcm_dst_size); + L.src0_bytes * n_hvx, L.src1_bytes * n_hvx, L.dst_bytes * n_hvx); uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; struct htp_unary_context uctx = { .octx = octx, .kparams = kparams, - .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .src0_nrows = nrows, .row_start = row_start, .data_src0 = src0->data, - .data_src1 = (octx->op == HTP_OP_RMS_NORM_MUL) ? src1->data : 0, + .data_src1 = has_weight ? src1->data : 0, + .data_src2 = has_src2 ? octx->src[2]->data : 0, .data_dst = dst->data, .src0_data_row_size = src0_data_row_size, .src1_data_row_size = src1_data_row_size, .dst_data_row_size = dst_data_row_size, - .src0_row_size_aligned = src0_row_size_aligned, - .src1_row_size_aligned = src1_row_size_aligned, - .dst_row_size_aligned = dst_row_size_aligned, + .src0_row_size_aligned = L.src0_row_size_aligned, + .src1_row_size_aligned = L.src1_row_size_aligned, + .dst_row_size_aligned = L.dst_row_size_aligned, - .src0_vtcm_half_size = kparams->vtcm_src0_size_per_thread / 2, - .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (kparams->vtcm_src1_size_per_thread / (broadcast_weight ? 1 : 2)) : 0, - .dst_vtcm_half_size = kparams->vtcm_dst_size_per_thread / 2, + .src0_vtcm_half_size = L.src0_bytes >> 1, + .src1_vtcm_half_size = has_weight ? (L.src1_bytes >> (broadcast_weight ? 0 : 1)) : 0, + .dst_vtcm_half_size = L.dst_bytes >> 1, .block = kparams->block, .nc = src0->ne[0], .col_tile = col_tile, .broadcast_weight = broadcast_weight, - .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0), - .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, kparams->vtcm_src0_size, kparams->vtcm_src1_size > 0), - .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->vtcm_src0_size + kparams->vtcm_src1_size), + .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0), + .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src1, L.src1_bytes > 0), + .vtcm_src2 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src2, L.src2_bytes > 0), + .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst), - .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread, - .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread, - .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread, + .vtcm_src0_size_per_hvx = L.src0_bytes, + .vtcm_src1_size_per_hvx = L.src1_bytes, + .vtcm_src2_size_per_hvx = L.src2_bytes, + .vtcm_dst_size_per_hvx = L.dst_bytes, }; FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile); - worker_callback_t task_func = NULL; + work_queue_func_t task_func = NULL; void * compute_func = NULL; if (col_tile) { @@ -1684,6 +2049,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_SIGMOID: compute_func = (void *) tile_sigmoid_f32; break; case HTP_OP_UNARY_SILU: compute_func = (void *) tile_silu_f32; break; case HTP_OP_UNARY_GELU: compute_func = (void *) tile_gelu_f32; break; + case HTP_OP_UNARY_GELU_QUICK: compute_func = (void *) tile_gelu_quick_f32; break; case HTP_OP_UNARY_GELU_ERF: compute_func = (void *) tile_gelu_erf_f32; break; case HTP_OP_UNARY_SOFTPLUS: compute_func = (void *) tile_softplus_f32; break; case HTP_OP_UNARY_TANH: compute_func = (void *) tile_tanh_f32; break; @@ -1691,6 +2057,8 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_LOG: compute_func = (void *) tile_log_f32; break; case HTP_OP_UNARY_RELU: compute_func = (void *) tile_relu_f32; break; case HTP_OP_UNARY_STEP: compute_func = (void *) tile_step_f32; break; + case HTP_OP_UNARY_CEIL: compute_func = (void *) tile_ceil_f32; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) tile_sgn_f32; break; case HTP_OP_TRI: task_func = unary_thread_tiled_tri_f32; compute_func = (void *) tri_apply_tile_f32; @@ -1710,6 +2078,9 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_ABS: compute_func = (void *) abs_f16; break; case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break; case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break; + case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f16; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f16; break; + case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break; default: break; } } else { @@ -1721,6 +2092,10 @@ static int execute_op_unary(struct htp_ops_context * octx) { task_func = unary_thread_rms_norm_mul_f32; compute_func = (void *) rms_norm_mul_f32; break; + case HTP_OP_RMS_NORM_MUL_ADD: + task_func = unary_thread_rms_norm_mul_add_f32; + compute_func = (void *) rms_norm_mul_add_f32; + break; case HTP_OP_SCALE: compute_func = (void *) scale_f32; break; case HTP_OP_CLAMP: compute_func = (void *) clamp_f32; break; case HTP_OP_LEAKY_RELU: compute_func = (void *) leaky_relu_f32; break; @@ -1731,6 +2106,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_SIGMOID: compute_func = (void *) sigmoid_f32; break; case HTP_OP_UNARY_SILU: compute_func = (void *) silu_f32; break; case HTP_OP_UNARY_GELU: compute_func = (void *) gelu_f32; break; + case HTP_OP_UNARY_GELU_QUICK: compute_func = (void *) gelu_quick_f32; break; case HTP_OP_UNARY_GELU_ERF: compute_func = (void *) gelu_erf_f32; break; case HTP_OP_UNARY_SOFTPLUS: compute_func = (void *) softplus_f32; break; case HTP_OP_UNARY_TANH: compute_func = (void *) tanh_f32; break; @@ -1738,7 +2114,10 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_LOG: compute_func = (void *) log_f32; break; case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break; case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break; + case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f32; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f32; break; case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break; + case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break; case HTP_OP_TRI: task_func = unary_thread_tri_f32; compute_func = (void *) tri_f32; @@ -1753,7 +2132,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { } uctx.compute = compute_func; - work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, task_func, &uctx, n_hvx); return err; } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 7b73cf10f..e03b2ac73 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -2,29 +2,22 @@ #define HTP_UNARY_OPS_H #include "hex-common.h" +#include "hex-fastdiv.h" #include "htp-ops.h" +#include "htp-vtcm.h" // Op-specific struct for precomputed unary params struct htp_unary_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t col_tile; - uint32_t vtcm_row_per_thread; uint32_t block; uint32_t broadcast_weight; - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_src1_size_per_thread; - uint32_t vtcm_dst_size_per_thread; - - uint32_t vtcm_src0_size; - uint32_t vtcm_src1_size; - uint32_t vtcm_dst_size; - - uint32_t src0_row_size_aligned; - uint32_t src1_row_size_aligned; - uint32_t dst_row_size_aligned; + uint32_t src0_contig; + uint32_t src1_contig; uint32_t vtcm_size; + uint32_t tiles_per_row; // Fastdiv helpers struct fastdiv_values div_ne01; @@ -46,6 +39,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_NORM: case HTP_OP_RMS_NORM: case HTP_OP_RMS_NORM_MUL: + case HTP_OP_RMS_NORM_MUL_ADD: case HTP_OP_SCALE: case HTP_OP_SQR: case HTP_OP_SQRT: @@ -54,6 +48,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_UNARY_SIGMOID: case HTP_OP_UNARY_SILU: case HTP_OP_UNARY_GELU: + case HTP_OP_UNARY_GELU_QUICK: case HTP_OP_UNARY_GELU_ERF: case HTP_OP_UNARY_SOFTPLUS: case HTP_OP_UNARY_TANH: @@ -61,120 +56,176 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_UNARY_LOG: case HTP_OP_UNARY_RELU: case HTP_OP_UNARY_STEP: + case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: case HTP_OP_L2_NORM: case HTP_OP_TRI: + case HTP_OP_HADAMARD: return true; default: return false; } } +static inline bool htp_unary_is_elementwise(uint32_t opcode) { + if (!htp_op_is_unary(opcode)) { + return false; + } + switch (opcode) { + case HTP_OP_NORM: + case HTP_OP_RMS_NORM: + case HTP_OP_RMS_NORM_MUL: + case HTP_OP_RMS_NORM_MUL_ADD: + case HTP_OP_L2_NORM: + case HTP_OP_HADAMARD: + case HTP_OP_TRI: + return false; + default: + return true; + } +} + struct htp_unary_vtcm_layout { size_t total_bytes; size_t off_src0; size_t off_src1; + size_t off_src2; size_t off_dst; + size_t src0_row_size_aligned; + size_t src1_row_size_aligned; + size_t dst_row_size_aligned; + size_t src0_bytes; size_t src1_bytes; + size_t src2_bytes; size_t dst_bytes; }; static inline void htp_unary_vtcm_layout_build( + struct htp_unary_vtcm_layout * L, + uint32_t op, + size_t src0_data_row_size, + size_t dst_data_row_size, + size_t src1_data_row_size, + size_t elem_size, + uint32_t col_tile, + uint32_t block, + bool broadcast_weight, + uint32_t n_hvx +) { + const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); + + L->src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); + L->dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); + L->src1_row_size_aligned = has_weight ? hex_round_up(src1_data_row_size, 128) : 0; + + if (col_tile > 0) { + const size_t col_tile_bytes = col_tile * elem_size; + L->src0_bytes = col_tile_bytes * 2; + L->dst_bytes = col_tile_bytes * 2; + L->src1_bytes = 0; + L->src2_bytes = 0; + } else { + L->src0_bytes = L->src0_row_size_aligned * block * 2; + L->src2_bytes = (op == HTP_OP_RMS_NORM_MUL_ADD) ? L->src0_bytes : 0; + L->dst_bytes = L->dst_row_size_aligned * block * 2; + if (has_weight) { + L->src1_bytes = broadcast_weight ? L->src1_row_size_aligned : (L->src1_row_size_aligned * block * 2); + } else { + L->src1_bytes = 0; + } + } + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_hvx, L->src1_bytes > 0); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src2, L->src2_bytes * n_hvx, L->src2_bytes > 0); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_hvx); + L->total_bytes = off; +} + +static inline void htp_unary_solve_layout( struct htp_unary_vtcm_layout * L, uint32_t op, uint32_t ne00, uint32_t ne10, uint32_t ne11, + uint32_t max_block, bool broadcast_weight, - uint32_t n_threads, + uint32_t n_hvx, size_t vtcm_size, size_t elem_size, uint32_t * out_col_tile, - uint32_t * out_vtcm_row_per_thread + uint32_t * out_vtcm_row_per_hvx ) { + const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); + + const size_t dst_elem_size = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size; const size_t src0_data_row_size = ne00 * elem_size; - const size_t dst_data_row_size = ne10 * elem_size; + const size_t dst_data_row_size = ne10 * dst_elem_size; + const size_t src1_data_row_size = has_weight ? (ne11 * sizeof(float)) : 0; const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); - - size_t src1_row_size_aligned = 0; - if (op == HTP_OP_RMS_NORM_MUL) { - // RMS_NORM_MUL fusion is F32-only; its weight tensor is always F32. - const size_t src1_data_row_size = ne11 * sizeof(float); - src1_row_size_aligned = hex_round_up(src1_data_row_size, 128); - } + const size_t src1_row_size_aligned = has_weight ? hex_round_up(src1_data_row_size, 128) : 0; size_t vtcm_size_per_row = 0; - size_t vtcm_row_per_thread = 0; - - if (op == HTP_OP_RMS_NORM_MUL) { + size_t vtcm_row_per_hvx = 0; + + if (op == HTP_OP_RMS_NORM_MUL_ADD) { + size_t src1_vtcm_total = n_hvx * src1_row_size_aligned; + size_t available_vtcm = (vtcm_size > src1_vtcm_total) ? (vtcm_size - src1_vtcm_total) : 0; + vtcm_size_per_row = 2 * (2 * src0_row_size_aligned + dst_row_size_aligned); + vtcm_row_per_hvx = available_vtcm / (n_hvx * vtcm_size_per_row); + } else if (op == HTP_OP_RMS_NORM_MUL) { if (broadcast_weight) { size_t available_vtcm = vtcm_size; - size_t src1_vtcm_total = n_threads * src1_row_size_aligned; + size_t src1_vtcm_total = n_hvx * src1_row_size_aligned; if (available_vtcm > src1_vtcm_total) { available_vtcm -= src1_vtcm_total; } else { available_vtcm = 0; } vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); - vtcm_row_per_thread = available_vtcm / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = available_vtcm / (n_hvx * vtcm_size_per_row); } else { vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned + src1_row_size_aligned); - vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = vtcm_size / (n_hvx * vtcm_size_per_row); } } else { vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); - vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = vtcm_size / (n_hvx * vtcm_size_per_row); + } + + if (max_block > 0 && vtcm_row_per_hvx > max_block) { + vtcm_row_per_hvx = max_block; } const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM || - op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM); + op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD || + op == HTP_OP_L2_NORM || op == HTP_OP_HADAMARD); // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*); // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops. - // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM + // NOTE: if F16 ends up with vtcm_row_per_hvx == 0 here (row too large for the VTCM // budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op // (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known // limitation; supporting it would require adding F16 tiled kernels. const bool is_f16 = (elem_size == sizeof(_Float16)); uint32_t col_tile = 0; - if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) { - const size_t per_thread_budget = vtcm_size / n_threads; - const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128); + if (vtcm_row_per_hvx == 0 && !is_reduction && !is_f16) { + const size_t per_hvx_budget = vtcm_size / n_hvx; + const size_t col_tile_bytes = hex_align_down(per_hvx_budget / 4, 128); col_tile = (uint32_t) (col_tile_bytes / elem_size); - - L->src0_bytes = col_tile_bytes * 2; - L->dst_bytes = col_tile_bytes * 2; - L->src1_bytes = 0; - } else { - L->src0_bytes = src0_row_size_aligned * vtcm_row_per_thread * 2; - L->dst_bytes = dst_row_size_aligned * vtcm_row_per_thread * 2; - if (op == HTP_OP_RMS_NORM_MUL) { - if (broadcast_weight) { - L->src1_bytes = src1_row_size_aligned; - } else { - L->src1_bytes = src1_row_size_aligned * vtcm_row_per_thread * 2; - } - } else { - L->src1_bytes = 0; - } - } - - L->off_src0 = 0; - if (op == HTP_OP_RMS_NORM_MUL) { - L->off_src1 = L->off_src0 + L->src0_bytes * n_threads; - L->off_dst = L->off_src1 + L->src1_bytes * n_threads; - } else { - L->off_src1 = 0; - L->off_dst = L->off_src0 + L->src0_bytes * n_threads; } - L->total_bytes = L->off_dst + L->dst_bytes * n_threads; + const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_hvx; + htp_unary_vtcm_layout_build(L, op, src0_data_row_size, dst_data_row_size, src1_data_row_size, + elem_size, col_tile, block, broadcast_weight, n_hvx); *out_col_tile = col_tile; - *out_vtcm_row_per_thread = vtcm_row_per_thread; + *out_vtcm_row_per_hvx = (uint32_t) vtcm_row_per_hvx; } #endif /* HTP_UNARY_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.c b/ggml/src/ggml-hexagon/htp/upscale-ops.c new file mode 100644 index 000000000..59ab7914a --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/upscale-ops.c @@ -0,0 +1,958 @@ +#pragma clang diagnostic ignored "-Wunused-variable" + +#include <float.h> +#include <math.h> +#include <HAP_farf.h> + +#include "hex-common.h" +#include "dma-queue.h" +#include "htp-trace.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-types.h" +#include "hvx-base.h" +#include "hvx-utils.h" +#include "upscale-ops.h" + +static inline int32_t clamp_i32(int32_t v, int32_t lo, int32_t hi) { + return v < lo ? lo : (v > hi ? hi : v); +} + +static inline float clamp_f32(float v, float lo, float hi) { + return v < lo ? lo : (v > hi ? hi : v); +} + +static inline float bicubic_weight1(float x) { + const float a = -0.75f; + return ((a + 2.0f) * x - (a + 3.0f)) * x * x + 1.0f; +} + +static inline float bicubic_weight2(float x) { + const float a = -0.75f; + return ((a * x - 5.0f * a) * x + 8.0f * a) * x - 4.0f * a; +} + +static inline void bicubic_weights(float x, float w[4]) { + w[0] = bicubic_weight2(x + 1.0f); + w[1] = bicubic_weight1(x); + w[2] = bicubic_weight1(1.0f - x); + w[3] = bicubic_weight2(2.0f - x); +} + +static void upscale_init_luts_nearest( + uint32_t * lut_x, + uint32_t * lut_y, + const struct htp_tensor * src0, + const struct htp_tensor * dst, + const struct htp_upscale_kernel_params * p +) { + const uint32_t src_x = src0->ne[0]; + const uint32_t src_y = src0->ne[1]; + const uint32_t dst_x = dst->ne[0]; + const uint32_t dst_y = dst->ne[1]; + const uint32_t dst_x_aligned = (dst_x + VLEN_FP32 - 1) & ~(VLEN_FP32 - 1); + + for (uint32_t ox = 0; ox < dst_x_aligned; ++ox) { + if (ox < dst_x) { + uint32_t x0 = (uint32_t) ((float) ox * p->inv_sf0); + lut_x[ox] = MIN(x0, src_x - 1) * sizeof(float); + } else { + lut_x[ox] = 0; + } + } + for (uint32_t oy = 0; oy < dst_y; ++oy) { + uint32_t y0 = (uint32_t) ((float) oy * p->inv_sf1); + lut_y[oy] = MIN(y0, src_y - 1); + } +} + +static void upscale_init_luts_bilinear( + struct upscale_bilinear_x_chunk * lut_x, + struct upscale_bilinear_y * lut_y, + const struct htp_tensor * src0, + const struct htp_tensor * dst, + const struct htp_upscale_kernel_params * p +) { + const uint32_t src_x = src0->ne[0]; + const uint32_t src_y = src0->ne[1]; + const uint32_t dst_x = dst->ne[0]; + const uint32_t dst_y = dst->ne[1]; + const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32; + + for (uint32_t c = 0; c < n_chunks; ++c) { + struct upscale_bilinear_x_chunk * chunk = &lut_x[c]; + const uint32_t ox_start = c * VLEN_FP32; + for (uint32_t l = 0; l < VLEN_FP32; ++l) { + const uint32_t ox = ox_start + l; + if (ox < dst_x) { + float xf = ((float) ox + p->pixel_offset) * p->inv_sf0 - p->pixel_offset; + int32_t x0 = (int32_t) floorf(xf); + int32_t x1 = x0 + 1; + x0 = clamp_i32(x0, 0, (int32_t) src_x - 1); + x1 = clamp_i32(x1, 0, (int32_t) src_x - 1); + float dx = clamp_f32(xf - (float) x0, 0.0f, 1.0f); + + chunk->off0[l] = x0 * (int32_t) sizeof(float); + chunk->off1[l] = x1 * (int32_t) sizeof(float); + chunk->w0[l] = 1.0f - dx; + chunk->dx[l] = dx; + } else { + chunk->off0[l] = 0; + chunk->off1[l] = 0; + chunk->w0[l] = 0.0f; + chunk->dx[l] = 0.0f; + } + } + } + for (uint32_t oy = 0; oy < dst_y; ++oy) { + float yf = ((float) oy + p->pixel_offset) * p->inv_sf1 - p->pixel_offset; + int32_t y0 = (int32_t) floorf(yf); + int32_t y1 = y0 + 1; + y0 = clamp_i32(y0, 0, (int32_t) src_y - 1); + y1 = clamp_i32(y1, 0, (int32_t) src_y - 1); + float dy = clamp_f32(yf - (float) y0, 0.0f, 1.0f); + + lut_y[oy].y0 = (uint32_t) y0; + lut_y[oy].y1 = (uint32_t) y1; + lut_y[oy].wy1 = dy; + lut_y[oy].wy0 = 1.0f - dy; + } +} + +static void upscale_init_luts_bicubic( + struct upscale_bicubic_x * lut_x, + struct upscale_bicubic_y * lut_y, + const struct htp_tensor * src0, + const struct htp_tensor * dst, + const struct htp_upscale_kernel_params * p +) { + const uint32_t src_x = src0->ne[0]; + const uint32_t src_y = src0->ne[1]; + const uint32_t dst_x = dst->ne[0]; + const uint32_t dst_y = dst->ne[1]; + + for (uint32_t ox = 0; ox < dst_x; ++ox) { + float xf = ((float) ox + p->pixel_offset) * p->inv_sf0 - p->pixel_offset; + int32_t x0 = (int32_t) floorf(xf); + float dx = xf - (float) x0; + bicubic_weights(dx, lut_x[ox].w); + + lut_x[ox].x[0] = clamp_i32(x0 - 1, 0, (int32_t) src_x - 1); + lut_x[ox].x[1] = clamp_i32(x0 + 0, 0, (int32_t) src_x - 1); + lut_x[ox].x[2] = clamp_i32(x0 + 1, 0, (int32_t) src_x - 1); + lut_x[ox].x[3] = clamp_i32(x0 + 2, 0, (int32_t) src_x - 1); + } + for (uint32_t oy = 0; oy < dst_y; ++oy) { + float yf = ((float) oy + p->pixel_offset) * p->inv_sf1 - p->pixel_offset; + int32_t y0 = (int32_t) floorf(yf); + float dy = yf - (float) y0; + bicubic_weights(dy, lut_y[oy].w); + + lut_y[oy].y[0] = clamp_i32(y0 - 1, 0, (int32_t) src_y - 1); + lut_y[oy].y[1] = clamp_i32(y0 + 0, 0, (int32_t) src_y - 1); + lut_y[oy].y[2] = clamp_i32(y0 + 1, 0, (int32_t) src_y - 1); + lut_y[oy].y[3] = clamp_i32(y0 + 2, 0, (int32_t) src_y - 1); + } +} + +static void upscale_init_luts_antialias( + struct upscale_antialias_x * lut_x, + struct upscale_antialias_y * lut_y, + const struct htp_tensor * src0, + const struct htp_tensor * dst, + const struct htp_upscale_kernel_params * p +) { + const uint32_t src_x = src0->ne[0]; + const uint32_t src_y = src0->ne[1]; + const uint32_t dst_x = dst->ne[0]; + const uint32_t dst_y = dst->ne[1]; + + const float filter_radius0 = p->filter_radius0; + const float filter_radius1 = p->filter_radius1; + + for (uint32_t ox = 0; ox < dst_x; ++ox) { + const float xf = ((float) ox + p->pixel_offset) * p->inv_sf0; + int32_t x_min = (int32_t) MAX(xf - filter_radius0 + p->pixel_offset, 0.0f); + int32_t x_max = (int32_t) MIN(xf + filter_radius0 + p->pixel_offset, (float) src_x); + lut_x[ox].x_min = clamp_i32(x_min, 0, (int32_t) src_x); + lut_x[ox].x_max = clamp_i32(x_max, 0, (int32_t) src_x); + lut_x[ox].xf = xf; + } + for (uint32_t oy = 0; oy < dst_y; ++oy) { + const float yf = ((float) oy + p->pixel_offset) * p->inv_sf1; + int32_t y_min = (int32_t) MAX(yf - filter_radius1 + p->pixel_offset, 0.0f); + int32_t y_max = (int32_t) MIN(yf + filter_radius1 + p->pixel_offset, (float) src_y); + lut_y[oy].y_min = clamp_i32(y_min, 0, (int32_t) src_y); + lut_y[oy].y_max = clamp_i32(y_max, 0, (int32_t) src_y); + lut_y[oy].yf = yf; + } +} + +static inline void get_src_y_window( + const struct htp_tensor * src0, + const struct htp_tensor * dst, + const struct htp_upscale_kernel_params * p, + const void * lut_y, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t * out_src_y_start, + uint32_t * out_src_y_count +) { + const uint32_t src_y = src0->ne[1]; + const uint32_t dst_y = dst->ne[1]; + + if (dst_y_count == dst_y) { + *out_src_y_start = 0; + *out_src_y_count = src_y; + return; + } + + const uint32_t last_oy = dst_y_start + dst_y_count - 1; + + if (p->mode == HTP_UPSCALE_MODE_NEAREST) { + const uint32_t * ly = (const uint32_t *) lut_y; + const uint32_t y0 = ly[dst_y_start]; + const uint32_t y1 = ly[last_oy]; + *out_src_y_start = y0; + *out_src_y_count = y1 - y0 + 1; + } else if (p->mode == HTP_UPSCALE_MODE_BICUBIC) { + const struct upscale_bicubic_y * ly = (const struct upscale_bicubic_y *) lut_y; + const int32_t y0 = ly[dst_y_start].y[0]; + const int32_t y1 = ly[last_oy].y[3]; + *out_src_y_start = (uint32_t) y0; + *out_src_y_count = (uint32_t) (y1 - y0 + 1); + } else if (p->antialias) { + const struct upscale_antialias_y * ly = (const struct upscale_antialias_y *) lut_y; + const int32_t y0 = ly[dst_y_start].y_min; + const int32_t y1 = ly[last_oy].y_max; + *out_src_y_start = (uint32_t) y0; + *out_src_y_count = (uint32_t) (y1 - y0); + } else { + const struct upscale_bilinear_y * ly = (const struct upscale_bilinear_y *) lut_y; + const uint32_t y0 = ly[dst_y_start].y0; + const uint32_t y1 = ly[last_oy].y1; + *out_src_y_start = y0; + *out_src_y_count = y1 - y0 + 1; + } +} + +static inline void resolve_dst_slice( + const struct htp_tensor * dst, + const struct htp_upscale_kernel_params * p, + uint32_t chunk_idx, + dma_addr_t * out_slice_dst_off, + uint32_t * out_dst_ys, + uint32_t * out_dst_yc +) { + const uint32_t s_idx = fastdiv(chunk_idx, &p->div_chunks_per_slice); + const uint32_t c_idx = chunk_idx - s_idx * p->chunks_per_slice; + const uint32_t d_ys = c_idx * p->dst_nrows_per_chunk; + const uint32_t d_yc = MIN(p->dst_nrows_per_chunk, dst->ne[1] - d_ys); + const uint32_t i3 = fastdiv(s_idx, &p->div_dst_z); + const uint32_t i2 = s_idx - i3 * dst->ne[2]; + + *out_slice_dst_off = (dma_addr_t) i2 * dst->nb[2] + (dma_addr_t) i3 * dst->nb[3]; + *out_dst_ys = d_ys; + *out_dst_yc = d_yc; +} + +static inline void resolve_src_slice( + const struct htp_tensor * src0, + const struct htp_tensor * dst, + const struct htp_upscale_kernel_params * p, + const void * lut_y, + uint32_t chunk_idx, + dma_addr_t * out_slice_src_off, + uint32_t * out_src_ys, + uint32_t * out_src_yc +) { + const uint32_t s_idx = fastdiv(chunk_idx, &p->div_chunks_per_slice); + const uint32_t c_idx = chunk_idx - s_idx * p->chunks_per_slice; + const uint32_t d_ys = c_idx * p->dst_nrows_per_chunk; + const uint32_t d_yc = MIN(p->dst_nrows_per_chunk, dst->ne[1] - d_ys); + uint32_t s_ys, s_yc; + get_src_y_window(src0, dst, p, lut_y, d_ys, d_yc, &s_ys, &s_yc); + + const uint32_t i3 = fastdiv(s_idx, &p->div_dst_z); + const uint32_t i2 = s_idx - i3 * dst->ne[2]; + const uint32_t i03 = (uint32_t) ((float) i3 * p->inv_sf3); + const uint32_t i02 = (uint32_t) ((float) i2 * p->inv_sf2); + + *out_slice_src_off = (dma_addr_t) i02 * src0->nb[2] + (dma_addr_t) i03 * src0->nb[3]; + *out_src_ys = s_ys; + *out_src_yc = s_yc; +} + +struct upscale_context; + +typedef void (*upscale_chunk_fn_t)( + const struct upscale_context * ctx, + const float * srcb, + float * dstb, + float * R_buf, + float * gather_tmp, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t src_y_start +); + +struct upscale_context { + struct htp_ops_context * octx; + const struct htp_upscale_kernel_params * kparams; + struct htp_upscale_vtcm_layout layout; + upscale_chunk_fn_t compute_chunk; + const void * lut_x; + const void * lut_y; + uint32_t chunk_start; + uint32_t chunk_count; + uint32_t chunks_per_hvx; +}; + +static void compute_nearest_chunk( + const struct upscale_context * ctx, + const float * srcb, + float * dstb, + float * R_buf, + float * gather_tmp, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t src_y_start +) { + const uint32_t src_x = ctx->octx->src[0]->ne[0]; + const uint32_t dst_x = ctx->octx->dst->ne[0]; + const uint32_t * lut_x = (const uint32_t *) ctx->lut_x; + const uint32_t * lut_y = (const uint32_t *) ctx->lut_y; + const uint32_t region_size = hex_round_up(src_x * sizeof(float), 128); + + for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) { + const uint32_t y0 = lut_y[oy]; + const float * r0 = srcb + (y0 - src_y_start) * src_x; + float * dst_row = dstb + (oy - dst_y_start) * dst_x; + + for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) { + *(HVX_Vector *) (R_buf + ix) = hvx_vmemu(r0 + ix); + } + + for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) { + const uint32_t rem = dst_x - ox; + const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN; + const HVX_Vector voff = hvx_vmem(lut_x + ox); + Q6_vgather_ARMVw((HVX_Vector *) gather_tmp, (size_t) R_buf, region_size, voff); + hvx_gather_sync(gather_tmp); + hvx_vec_store_u(dst_row + ox, nbytes, hvx_vmem(gather_tmp)); + } + } +} + +static void compute_nearest_2x_chunk( + const struct upscale_context * ctx, + const float * srcb, + float * dstb, + float * R_buf, + float * gather_tmp, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t src_y_start +) { + (void) R_buf; + (void) gather_tmp; + const uint32_t src_x = ctx->octx->src[0]->ne[0]; + const uint32_t dst_x = ctx->octx->dst->ne[0]; + + for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) { + const uint32_t sy = oy >> 1; + const float * r0 = srcb + (sy - src_y_start) * src_x; + float * dst_row = dstb + (oy - dst_y_start) * dst_x; + + for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) { + const HVX_Vector vin = hvx_vmemu(r0 + ix); + + const HVX_VectorPair p = Q6_W_vshuff_VVR(vin, vin, -4); + const HVX_Vector v_lo = Q6_V_lo_W(p); + const HVX_Vector v_hi = Q6_V_hi_W(p); + + const uint32_t ox0 = 2 * ix; + const uint32_t rem_out0 = (dst_x > ox0) ? (dst_x - ox0) : 0; + const uint32_t nbytes_out0 = (rem_out0 < VLEN_FP32) ? (rem_out0 * sizeof(float)) : VLEN; + hvx_vec_store_u(dst_row + ox0, nbytes_out0, v_lo); + + const uint32_t ox1 = ox0 + VLEN_FP32; + if (dst_x > ox1) { + const uint32_t rem_out1 = dst_x - ox1; + const uint32_t nbytes_out1 = (rem_out1 < VLEN_FP32) ? (rem_out1 * sizeof(float)) : VLEN; + hvx_vec_store_u(dst_row + ox1, nbytes_out1, v_hi); + } + } + } +} + +static void compute_nearest_4x_chunk( + const struct upscale_context * ctx, + const float * srcb, + float * dstb, + float * R_buf, + float * gather_tmp, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t src_y_start +) { + (void) R_buf; + (void) gather_tmp; + const uint32_t src_x = ctx->octx->src[0]->ne[0]; + const uint32_t dst_x = ctx->octx->dst->ne[0]; + + for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) { + const uint32_t sy = oy >> 2; + const float * r0 = srcb + (sy - src_y_start) * src_x; + float * dst_row = dstb + (oy - dst_y_start) * dst_x; + + for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) { + const HVX_Vector vin = hvx_vmemu(r0 + ix); + + const HVX_VectorPair p = Q6_W_vshuff_VVR(vin, vin, -4); + const HVX_Vector v_lo = Q6_V_lo_W(p); + const HVX_Vector v_hi = Q6_V_hi_W(p); + + const HVX_VectorPair p0 = Q6_W_vshuff_VVR(v_lo, v_lo, -4); + const HVX_VectorPair p1 = Q6_W_vshuff_VVR(v_hi, v_hi, -4); + + const HVX_Vector out0 = Q6_V_lo_W(p0); + const HVX_Vector out1 = Q6_V_hi_W(p0); + const HVX_Vector out2 = Q6_V_lo_W(p1); + const HVX_Vector out3 = Q6_V_hi_W(p1); + + const uint32_t ox = 4 * ix; + if (dst_x > ox) { + const uint32_t rem = dst_x - ox; + hvx_vec_store_u(dst_row + ox, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out0); + } + if (dst_x > ox + VLEN_FP32) { + const uint32_t rem = dst_x - (ox + VLEN_FP32); + hvx_vec_store_u(dst_row + ox + VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out1); + } + if (dst_x > ox + 2 * VLEN_FP32) { + const uint32_t rem = dst_x - (ox + 2 * VLEN_FP32); + hvx_vec_store_u(dst_row + ox + 2 * VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out2); + } + if (dst_x > ox + 3 * VLEN_FP32) { + const uint32_t rem = dst_x - (ox + 3 * VLEN_FP32); + hvx_vec_store_u(dst_row + ox + 3 * VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out3); + } + } + } +} + +static inline void compute_bilinear_chunk_32( + float * restrict dst_chunk, + const float * restrict R_buf, + float * restrict tmp_a, + float * restrict tmp_b, + uint32_t region_size, + const struct upscale_bilinear_x_chunk * restrict chunk, + uint32_t nbytes +) { + const HVX_Vector voff0 = hvx_vmem(chunk->off0); + const HVX_Vector voff1 = hvx_vmem(chunk->off1); + const HVX_Vector vw0 = hvx_vmem(chunk->w0); + const HVX_Vector vdx = hvx_vmem(chunk->dx); + + Q6_vgather_ARMVw((HVX_Vector *) tmp_a, (size_t) R_buf, region_size, voff0); + Q6_vgather_ARMVw((HVX_Vector *) tmp_b, (size_t) R_buf, region_size, voff1); + hvx_gather_sync(tmp_b); + + const HVX_Vector va = hvx_vmem(tmp_a); + const HVX_Vector vb = hvx_vmem(tmp_b); + + const HVX_Vector val = hvx_vec_add_f32_f32( + hvx_vec_mul_f32_f32(vw0, va), + hvx_vec_mul_f32_f32(vdx, vb) + ); + + hvx_vec_store_u(dst_chunk, nbytes, val); +} + +static void compute_bilinear_chunk( + const struct upscale_context * ctx, + const float * srcb, + float * dstb, + float * R_buf, + float * gather_tmp, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t src_y_start +) { + const uint32_t src_x = ctx->octx->src[0]->ne[0]; + const uint32_t dst_x = ctx->octx->dst->ne[0]; + const struct upscale_bilinear_x_chunk * lut_x = (const struct upscale_bilinear_x_chunk *) ctx->lut_x; + const struct upscale_bilinear_y * lut_y = (const struct upscale_bilinear_y *) ctx->lut_y; + const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32; + const uint32_t region_size = hex_round_up(src_x * sizeof(float), 128); + float * tmp_a = gather_tmp; + float * tmp_b = gather_tmp + VLEN_FP32; + + for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) { + const struct upscale_bilinear_y * ey = &lut_y[oy]; + const float * r0 = srcb + (ey->y0 - src_y_start) * src_x; + const float * r1 = srcb + (ey->y1 - src_y_start) * src_x; + float * dst_row = dstb + (oy - dst_y_start) * dst_x; + + if (ey->y0 == ey->y1) { + for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) { + *(HVX_Vector *) (R_buf + ix) = hvx_vmemu(r0 + ix); + } + } else { + const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->wy0); + const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->wy1); + for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) { + const HVX_Vector vr0 = hvx_vmemu(r0 + ix); + const HVX_Vector vr1 = hvx_vmemu(r1 + ix); + *(HVX_Vector *) (R_buf + ix) = hvx_vec_add_f32_f32( + hvx_vec_mul_f32_f32(vwy0, vr0), + hvx_vec_mul_f32_f32(vwy1, vr1) + ); + } + } + + for (uint32_t c = 0; c < n_chunks; ++c) { + const uint32_t ox = c * VLEN_FP32; + const uint32_t rem = dst_x - ox; + const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN; + compute_bilinear_chunk_32(dst_row + ox, R_buf, tmp_a, tmp_b, region_size, &lut_x[c], nbytes); + } + } +} + +static void compute_bilinear_2x_chunk( + const struct upscale_context * ctx, + const float * srcb, + float * dstb, + float * R_buf, + float * gather_tmp, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t src_y_start +) { + (void) R_buf; + (void) gather_tmp; + const uint32_t src_x = ctx->octx->src[0]->ne[0]; + const uint32_t dst_x = ctx->octx->dst->ne[0]; + const struct upscale_bilinear_y * lut_y = (const struct upscale_bilinear_y *) ctx->lut_y; + const HVX_Vector c_075 = hvx_vec_splat_f32(0.75f); + const HVX_Vector c_025 = hvx_vec_splat_f32(0.25f); + + for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) { + const struct upscale_bilinear_y * ey = &lut_y[oy]; + const float * r0 = srcb + (ey->y0 - src_y_start) * src_x; + const float * r1 = srcb + (ey->y1 - src_y_start) * src_x; + float * dst_row = dstb + (oy - dst_y_start) * dst_x; + + const bool same_row = (ey->y0 == ey->y1); + const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->wy0); + const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->wy1); + + HVX_Vector v_curr = hvx_vmem(r0); + if (!same_row) { + const HVX_Vector v1 = hvx_vmem(r1); + v_curr = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, v_curr), hvx_vec_mul_f32_f32(vwy1, v1)); + } + + HVX_Vector v_prev = Q6_V_vror_VR(v_curr, 4); + + for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) { + HVX_Vector v_next_chunk; + const uint32_t next_ix = ix + VLEN_FP32; + if (next_ix < src_x) { + v_next_chunk = hvx_vmem(r0 + next_ix); + if (!same_row) { + const HVX_Vector v1 = hvx_vmem(r1 + next_ix); + v_next_chunk = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, v_next_chunk), hvx_vec_mul_f32_f32(vwy1, v1)); + } + } else { + v_next_chunk = Q6_V_vror_VR(v_curr, 124); + } + + const HVX_Vector v_shl = Q6_V_valign_VVR(v_curr, v_prev, 124); + const HVX_Vector v_shr = Q6_V_valign_VVR(v_next_chunk, v_curr, 4); + + const HVX_Vector ve = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(c_075, v_curr), hvx_vec_mul_f32_f32(c_025, v_shl)); + const HVX_Vector vo = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(c_075, v_curr), hvx_vec_mul_f32_f32(c_025, v_shr)); + + const HVX_VectorPair p = Q6_W_vshuff_VVR(vo, ve, -4); + const HVX_Vector out0 = Q6_V_lo_W(p); + const HVX_Vector out1 = Q6_V_hi_W(p); + + const uint32_t ox0 = 2 * ix; + hvx_vmem(dst_row + ox0) = out0; + hvx_vmem(dst_row + ox0 + VLEN_FP32) = out1; + + v_prev = v_curr; + v_curr = v_next_chunk; + } + } +} + +static void compute_bilinear_antialias_chunk( + const struct upscale_context * ctx, + const float * srcb, + float * dstb, + float * R_buf, + float * gather_tmp, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t src_y_start +) { + (void) R_buf; + (void) gather_tmp; + const uint32_t src_x = ctx->octx->src[0]->ne[0]; + const uint32_t dst_x = ctx->octx->dst->ne[0]; + const struct htp_upscale_kernel_params * p = ctx->kparams; + const struct upscale_antialias_x * lut_x = (const struct upscale_antialias_x *) ctx->lut_x; + const struct upscale_antialias_y * lut_y = (const struct upscale_antialias_y *) ctx->lut_y; + + const float invscale1 = p->invscale1; + const float invscale0 = p->invscale0; + + for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) { + const struct upscale_antialias_y * ey = &lut_y[oy]; + const float yf = ey->yf; + const int32_t y_min = ey->y_min; + const int32_t y_max = ey->y_max; + + float * dst_row = dstb + (oy - dst_y_start) * dst_x; + + for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) { + const uint32_t rem = dst_x - ox; + const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN; + float tmp[VLEN_FP32] __attribute__((aligned(128))); + const uint32_t cnt = rem < VLEN_FP32 ? rem : VLEN_FP32; + + for (uint32_t l = 0; l < cnt; ++l) { + const uint32_t cur_ox = ox + l; + const struct upscale_antialias_x * ex = &lut_x[cur_ox]; + const float xf = ex->xf; + const int32_t x_min = ex->x_min; + const int32_t x_max = ex->x_max; + + float val = 0.0f; + float total_weight = 0.0f; + + for (int32_t sy = y_min; sy < y_max; ++sy) { + const float diff_y = fabsf((float) sy - yf + p->pixel_offset) * invscale1; + const float weight_y = MAX(1.0f - diff_y, 0.0f); + if (weight_y <= 0.0f) { + continue; + } + const float * r = srcb + (uint32_t) (sy - (int32_t) src_y_start) * src_x; + + for (int32_t sx = x_min; sx < x_max; ++sx) { + const float diff_x = fabsf((float) sx - xf + p->pixel_offset) * invscale0; + const float weight_x = MAX(1.0f - diff_x, 0.0f); + const float weight = weight_x * weight_y; + if (weight <= 0.0f) { + continue; + } + val += r[sx] * weight; + total_weight += weight; + } + } + + tmp[l] = (total_weight > 0.0f) ? (val / total_weight) : 0.0f; + } + + hvx_vec_store_u(dst_row + ox, nbytes, *(const HVX_Vector *) tmp); + } + } +} + +static void compute_bicubic_chunk( + const struct upscale_context * ctx, + const float * srcb, + float * dstb, + float * R_buf, + float * gather_tmp, + uint32_t dst_y_start, + uint32_t dst_y_count, + uint32_t src_y_start +) { + (void) R_buf; + (void) gather_tmp; + const uint32_t src_x = ctx->octx->src[0]->ne[0]; + const uint32_t dst_x = ctx->octx->dst->ne[0]; + const struct upscale_bicubic_x * lut_x = (const struct upscale_bicubic_x *) ctx->lut_x; + const struct upscale_bicubic_y * lut_y = (const struct upscale_bicubic_y *) ctx->lut_y; + + for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) { + const struct upscale_bicubic_y * ey = &lut_y[oy]; + const float * r0 = srcb + (uint32_t) (ey->y[0] - (int32_t) src_y_start) * src_x; + const float * r1 = srcb + (uint32_t) (ey->y[1] - (int32_t) src_y_start) * src_x; + const float * r2 = srcb + (uint32_t) (ey->y[2] - (int32_t) src_y_start) * src_x; + const float * r3 = srcb + (uint32_t) (ey->y[3] - (int32_t) src_y_start) * src_x; + + float * dst_row = dstb + (oy - dst_y_start) * dst_x; + + const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->w[0]); + const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->w[1]); + const HVX_Vector vwy2 = hvx_vec_splat_f32(ey->w[2]); + const HVX_Vector vwy3 = hvx_vec_splat_f32(ey->w[3]); + + for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) { + const uint32_t rem = dst_x - ox; + const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN; + + float v0[VLEN_FP32] __attribute__((aligned(128))); + float v1[VLEN_FP32] __attribute__((aligned(128))); + float v2[VLEN_FP32] __attribute__((aligned(128))); + float v3[VLEN_FP32] __attribute__((aligned(128))); + + const uint32_t cnt = rem < VLEN_FP32 ? rem : VLEN_FP32; + for (uint32_t l = 0; l < cnt; ++l) { + const struct upscale_bicubic_x * e = &lut_x[ox + l]; + v0[l] = r0[e->x[0]] * e->w[0] + r0[e->x[1]] * e->w[1] + r0[e->x[2]] * e->w[2] + r0[e->x[3]] * e->w[3]; + v1[l] = r1[e->x[0]] * e->w[0] + r1[e->x[1]] * e->w[1] + r1[e->x[2]] * e->w[2] + r1[e->x[3]] * e->w[3]; + v2[l] = r2[e->x[0]] * e->w[0] + r2[e->x[1]] * e->w[1] + r2[e->x[2]] * e->w[2] + r2[e->x[3]] * e->w[3]; + v3[l] = r3[e->x[0]] * e->w[0] + r3[e->x[1]] * e->w[1] + r3[e->x[2]] * e->w[2] + r3[e->x[3]] * e->w[3]; + } + for (uint32_t l = cnt; l < VLEN_FP32; ++l) { + v0[l] = v1[l] = v2[l] = v3[l] = 0.0f; + } + + const HVX_Vector vv0 = *(const HVX_Vector *) v0; + const HVX_Vector vv1 = *(const HVX_Vector *) v1; + const HVX_Vector vv2 = *(const HVX_Vector *) v2; + const HVX_Vector vv3 = *(const HVX_Vector *) v3; + + const HVX_Vector acc0 = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, vv0), hvx_vec_mul_f32_f32(vwy1, vv1)); + const HVX_Vector acc1 = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy2, vv2), hvx_vec_mul_f32_f32(vwy3, vv3)); + const HVX_Vector val = hvx_vec_add_f32_f32(acc0, acc1); + + hvx_vec_store_u(dst_row + ox, nbytes, val); + } + } +} + +static inline void dma_push_src_chunk( + dma_queue * dma_q, + float * spad, + const struct htp_tensor * src0, + dma_addr_t slice_src_off, + uint32_t src_ys, + uint32_t src_yc +) { + const uint32_t src_x = src0->ne[0]; + const dma_addr_t src_addr = src0->data + slice_src_off + (dma_addr_t) src_ys * src0->nb[1]; + + if (src0->nb[0] == sizeof(float)) { + const uint32_t row_size = src_x * sizeof(float); + const uint32_t dst_stride = row_size; + const uint32_t src_stride = src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad, src_addr), dst_stride, src_stride, row_size, src_yc); + } else { + const uint32_t row_size = sizeof(float); + const uint32_t dst_stride = sizeof(float); + const uint32_t src_stride = src0->nb[0]; + const uint32_t nrows = src_x * src_yc; + dma_queue_push(dma_q, dma_make_data(spad, src_addr), dst_stride, src_stride, row_size, nrows); + } +} + +static void upscale_worker(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct upscale_context * ctx = (struct upscale_context *) data; + const struct htp_upscale_kernel_params * p = ctx->kparams; + const struct htp_upscale_vtcm_layout * L = &ctx->layout; + const struct htp_tensor * src0 = ctx->octx->src[0]; + const struct htp_tensor * dst = ctx->octx->dst; + + const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_hvx; + const uint32_t last = MIN(first + ctx->chunks_per_hvx, ctx->chunk_start + ctx->chunk_count); + + if (first >= last) { + return; + } + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size; + const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; + + uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base; + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; + float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_hvx / sizeof(float)); + float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_hvx / sizeof(float)); + const void * lut_y = ctx->lut_y; + + float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; + float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) }; + + const uint32_t dst_row_size = dst->ne[0] * sizeof(float); + const uint32_t dst_stride = dst->nb[1]; + + const uint32_t total = last - first; + + // Warmup: prime pipeline with initial transfers + for (uint32_t i = 0; i < total && i < 2; ++i) { + dma_queue_push(dma_q, dma_make_data(dst->data, dstb2[i]), dst_stride, dst_row_size, dst_row_size, 0); + + dma_addr_t slice_src_off; + uint32_t src_ys, src_yc; + resolve_src_slice(src0, dst, p, lut_y, first + i, &slice_src_off, &src_ys, &src_yc); + + dma_push_src_chunk(dma_q, srcb2[i], src0, slice_src_off, src_ys, src_yc); + } + + // Steady state loop + for (uint32_t i = 0; i < total; ++i) { + const uint32_t buf = i & 1u; + float * srcb = srcb2[buf]; + float * dstb = dstb2[buf]; + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + dma_addr_t slice_dst_off; + uint32_t dst_ys, dst_yc; + resolve_dst_slice(dst, p, first + i, &slice_dst_off, &dst_ys, &dst_yc); + + uint32_t src_ys, src_yc; + get_src_y_window(src0, dst, p, lut_y, dst_ys, dst_yc, &src_ys, &src_yc); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (first + i)); + ctx->compute_chunk(ctx, srcb, dstb, R_buf, gather_tmp, dst_ys, dst_yc, src_ys); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (first + i)); + + const dma_addr_t dst_addr = dst->data + slice_dst_off + (dma_addr_t) dst_ys * dst_stride; + dma_queue_push(dma_q, dma_make_data(dst_addr, dstb), dst_stride, dst_row_size, dst_row_size, dst_yc); + + const uint32_t next = i + 2; + if (next < total) { + dma_addr_t next_src_off; + uint32_t next_src_ys, next_src_yc; + resolve_src_slice(src0, dst, p, lut_y, first + next, &next_src_off, &next_src_ys, &next_src_yc); + + dma_push_src_chunk(dma_q, srcb, src0, next_src_off, next_src_ys, next_src_yc); + } + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +int op_upscale(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + const struct htp_upscale_kernel_params * p = + (const struct htp_upscale_kernel_params *) octx->kernel_params; + + const uint32_t src_x = src0->ne[0]; + const uint32_t src_y = src0->ne[1]; + const uint32_t dst_x = dst->ne[0]; + const uint32_t dst_y = dst->ne[1]; + + uint32_t chunk_start = 0; + uint32_t chunk_count = p->total_chunks; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + chunk_count, htp_tensor_mdev_data_aligned(dst) ? p->chunks_per_group : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + chunk_start = range.start; + chunk_count = range.count; + } + + if (chunk_count == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_hvx = MIN(p->n_hvx, chunk_count); + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t chunks_per_hvx = fastdiv(chunk_count + n_hvx - 1, &octx->n_hvx_div); + + size_t lut_x_size = 0; + size_t elem_y_sz = sizeof(uint32_t); + + if (p->mode == HTP_UPSCALE_MODE_BICUBIC) { + lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_bicubic_x), 128); + elem_y_sz = sizeof(struct upscale_bicubic_y); + } else if (p->antialias) { + lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_antialias_x), 128); + elem_y_sz = sizeof(struct upscale_antialias_y); + } else if (p->mode == HTP_UPSCALE_MODE_BILINEAR) { + const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32; + lut_x_size = hex_round_up(n_chunks * sizeof(struct upscale_bilinear_x_chunk), 128); + elem_y_sz = sizeof(struct upscale_bilinear_y); + } else { + lut_x_size = hex_round_up(dst_x * sizeof(uint32_t), 128); + } + + const size_t lut_y_size = hex_round_up(dst_y * elem_y_sz, 128); + const size_t total_lut_size = lut_x_size + lut_y_size; + + if (total_lut_size > octx->ctx->ddr_spad_size) { + return HTP_STATUS_NO_SUPPORT; + } + + void * base_scratch = octx->ctx->ddr_spad_base; + void * lut_x = base_scratch; + void * lut_y = (uint8_t *) base_scratch + lut_x_size; + + if (p->mode == HTP_UPSCALE_MODE_NEAREST) { + upscale_init_luts_nearest((uint32_t *) lut_x, (uint32_t *) lut_y, src0, dst, p); + } else if (p->mode == HTP_UPSCALE_MODE_BICUBIC) { + upscale_init_luts_bicubic((struct upscale_bicubic_x *) lut_x, (struct upscale_bicubic_y *) lut_y, src0, dst, p); + } else if (p->antialias) { + upscale_init_luts_antialias((struct upscale_antialias_x *) lut_x, (struct upscale_antialias_y *) lut_y, src0, dst, p); + } else { + upscale_init_luts_bilinear((struct upscale_bilinear_x_chunk *) lut_x, (struct upscale_bilinear_y *) lut_y, src0, dst, p); + } + + upscale_chunk_fn_t compute_chunk; + switch (p->kernel_type) { + case HTP_UPSCALE_KERNEL_NEAREST: + compute_chunk = compute_nearest_chunk; + break; + case HTP_UPSCALE_KERNEL_NEAREST_2X: + compute_chunk = compute_nearest_2x_chunk; + break; + case HTP_UPSCALE_KERNEL_NEAREST_4X: + compute_chunk = compute_nearest_4x_chunk; + break; + case HTP_UPSCALE_KERNEL_BILINEAR: + compute_chunk = compute_bilinear_chunk; + break; + case HTP_UPSCALE_KERNEL_BILINEAR_2X: + compute_chunk = compute_bilinear_2x_chunk; + break; + case HTP_UPSCALE_KERNEL_BILINEAR_AA: + compute_chunk = compute_bilinear_antialias_chunk; + break; + case HTP_UPSCALE_KERNEL_BICUBIC: + compute_chunk = compute_bicubic_chunk; + break; + default: + return HTP_STATUS_INVAL_PARAMS; + } + + struct upscale_context ctx = { + .octx = octx, + .kparams = p, + .compute_chunk = compute_chunk, + .lut_x = lut_x, + .lut_y = lut_y, + .chunk_start = chunk_start, + .chunk_count = chunk_count, + .chunks_per_hvx = chunks_per_hvx, + }; + + htp_upscale_vtcm_layout_build(&ctx.layout, src_x, src_y, dst_x, dst_y, + p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_hvx); + + work_queue_run(octx->ctx->hvx_queue, upscale_worker, &ctx, n_hvx); + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.h b/ggml/src/ggml-hexagon/htp/upscale-ops.h new file mode 100644 index 000000000..592bf7f00 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/upscale-ops.h @@ -0,0 +1,204 @@ +#ifndef HTP_UPSCALE_OPS_H +#define HTP_UPSCALE_OPS_H + +#include <stdint.h> +#include <stddef.h> +#include <stdbool.h> +#include <math.h> + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" + +enum htp_upscale_mode { + HTP_UPSCALE_MODE_NEAREST = 0, + HTP_UPSCALE_MODE_BILINEAR = 1, + HTP_UPSCALE_MODE_BICUBIC = 2, +}; + +enum htp_upscale_kernel_type { + HTP_UPSCALE_KERNEL_NEAREST = 0, + HTP_UPSCALE_KERNEL_NEAREST_2X = 1, + HTP_UPSCALE_KERNEL_NEAREST_4X = 2, + HTP_UPSCALE_KERNEL_BILINEAR = 3, + HTP_UPSCALE_KERNEL_BILINEAR_2X = 4, + HTP_UPSCALE_KERNEL_BILINEAR_AA = 5, + HTP_UPSCALE_KERNEL_BICUBIC = 6, +}; + +struct htp_upscale_kernel_params { + uint8_t kernel_type; + uint8_t mode; + uint8_t align_corners; + uint8_t antialias; + + float inv_sf0; + float inv_sf1; + float inv_sf2; + float inv_sf3; + float pixel_offset; + + float filter_radius0; + float filter_radius1; + float invscale0; + float invscale1; + + uint32_t n_hvx; + uint32_t chunks_per_group; + + uint32_t dst_nrows_per_chunk; + uint32_t chunks_per_slice; + uint32_t total_chunks; + uint32_t vtcm_size; + + struct fastdiv_values div_chunks_per_slice; + struct fastdiv_values div_dst_z; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_upscale_kernel_params) <= 128, "htp_upscale_kernel_params is too large"); +#else +_Static_assert(sizeof(struct htp_upscale_kernel_params) <= 128, "htp_upscale_kernel_params is too large"); +#endif + +struct __attribute__((aligned(128))) upscale_bilinear_x_chunk { + int32_t off0[32]; + int32_t off1[32]; + float w0[32]; + float dx[32]; +}; + +struct upscale_bilinear_y { + uint32_t y0; + uint32_t y1; + float wy0; + float wy1; +}; + +struct upscale_bicubic_x { + int32_t x[4]; + float w[4]; +}; + +struct upscale_bicubic_y { + int32_t y[4]; + float w[4]; +}; + +struct upscale_antialias_x { + int32_t x_min; + int32_t x_max; + float xf; +}; + +struct upscale_antialias_y { + int32_t y_min; + int32_t y_max; + float yf; +}; + +struct htp_upscale_vtcm_layout { + size_t total_bytes; + size_t off_src; + size_t off_dst; + size_t off_r; + size_t off_gather; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; + size_t r_bytes_per_hvx; + size_t gather_bytes_per_hvx; + size_t src_spad_half_size; + size_t dst_spad_half_size; + uint32_t max_src_chunk_rows; + uint32_t dst_chunk_rows; +}; + +static inline void htp_upscale_vtcm_layout_build( + struct htp_upscale_vtcm_layout * L, + uint32_t src_x, + uint32_t src_y, + uint32_t dst_x, + uint32_t dst_y, + uint32_t dst_chunk_rows, + float inv_sf1, + float filter_radius1, + uint32_t n_hvx +) { + uint32_t max_src_rows; + if (dst_chunk_rows >= dst_y) { + max_src_rows = src_y; + } else { + const float est = (float) dst_chunk_rows * inv_sf1 + 2.0f * filter_radius1; + max_src_rows = (uint32_t) ceilf(est) + 4; + if (max_src_rows > src_y) { + max_src_rows = src_y; + } + } + + const uint32_t src_chunk_bytes = src_x * max_src_rows * sizeof(float); + const uint32_t dst_chunk_bytes = dst_x * dst_chunk_rows * sizeof(float); + + L->max_src_chunk_rows = max_src_rows; + L->dst_chunk_rows = dst_chunk_rows; + L->src_spad_half_size = hex_round_up(src_chunk_bytes + 256, 128); + L->dst_spad_half_size = hex_round_up(dst_chunk_bytes, 128); + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; + L->r_bytes_per_hvx = hex_round_up(src_x * sizeof(float), 128); + L->gather_bytes_per_hvx = 256; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_hvx * n_hvx); + L->total_bytes = off; +} + +static inline bool htp_upscale_solve_layout( + struct htp_upscale_vtcm_layout * layout, + uint32_t src_x, + uint32_t src_y, + uint32_t dst_x, + uint32_t dst_y, + float inv_sf1, + float filter_radius1, + uint32_t n_hvx, + size_t vtcm_budget, + uint32_t * out_dst_chunk_rows +) { + // 1. Try full plane + uint32_t chunk_rows = dst_y; + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); + if (layout->total_bytes <= vtcm_budget) { + *out_dst_chunk_rows = chunk_rows; + return true; + } + + // 2. Try smaller chunk bands: 64, 32, 16, 8, 4, 2, 1 + chunk_rows = 64; + while (chunk_rows > 1) { + if (chunk_rows < dst_y) { + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); + if (layout->total_bytes <= vtcm_budget) { + *out_dst_chunk_rows = chunk_rows; + return true; + } + } + chunk_rows /= 2; + } + + // 3. Fallback to 1 row + chunk_rows = 1; + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); + if (layout->total_bytes <= vtcm_budget) { + *out_dst_chunk_rows = chunk_rows; + return true; + } + + return false; +} + +int op_upscale(struct htp_ops_context * octx); + +#endif // HTP_UPSCALE_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/work-queue.c b/ggml/src/ggml-hexagon/htp/work-queue.c index 6c74445f7..5d274274a 100644 --- a/ggml/src/ggml-hexagon/htp/work-queue.c +++ b/ggml/src/ggml-hexagon/htp/work-queue.c @@ -16,8 +16,8 @@ // internal structure kept in thread-local storage per instance of work queue typedef struct { - work_queue_t queue; - unsigned int id; + htp_thread_t th; + work_queue_t queue; } worker_context_t; struct work_queue_task_s { @@ -35,15 +35,15 @@ struct work_queue_s { uint32_t idx_mask; uint32_t capacity; - qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers - worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts - void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers - unsigned int n_threads; // total threads (workers + main) - unsigned int n_workers; // number of active threads (just workers) + qurt_thread_t thread[HTP_MAX_NTHREADS]; // thread ID's of the workers + worker_context_t context[HTP_MAX_NTHREADS]; // worker contexts (0 = main, 1..n_workers = workers) + void * stack[HTP_MAX_NTHREADS]; // thread stack pointers + unsigned int n_threads; // total threads (workers + main) + unsigned int n_workers; // number of active threads (just workers) - atomic_uint active; // workers are polling/active - atomic_bool killed; // threads need to exit - bool external_mem; // memory owned externally + atomic_uint active; // workers are polling/active + atomic_bool killed; // threads need to exit + bool external_mem; // memory owned externally struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE))); }; @@ -51,8 +51,9 @@ struct work_queue_s { static void work_queue_thread(void * context) { worker_context_t * me = (worker_context_t *) context; work_queue_t q = me->queue; + unsigned int id = (unsigned int) (me - q->context); - FARF(HIGH, "work-queue: thread %u started", me->id); + FARF(HIGH, "work-queue: thread %u (tid %u) started", id, me->th.tid); unsigned int prev_seqn = 0; @@ -62,6 +63,7 @@ static void work_queue_thread(void * context) { if (atomic_load_explicit(&q->active, memory_order_relaxed)) { hex_pause(); } else { + qurt_hvx_unlock(); qurt_futex_wait(&q->active, 0); } continue; @@ -77,9 +79,8 @@ static void work_queue_thread(void * context) { struct work_queue_task_s * task = &q->queue[ir]; unsigned int n = task->n_threads; - unsigned int i = me->id; - if (i < n) { - task->func(n, i, task->data); + if (id < n) { + task->func(&me->th, n, id, task->data); atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); } else { @@ -92,10 +93,15 @@ static void work_queue_thread(void * context) { } } - FARF(HIGH, "work-queue: thread %u stopped", me->id); + FARF(HIGH, "work-queue: thread %u (tid %u) stopped", id, me->th.tid); } -bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { +bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n <= 1) { + func(&q->context[0].th, n, 0, data); + return true; + } + if (n > q->n_threads) { FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads); return false; @@ -121,7 +127,7 @@ bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, u atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); // main thread runs job #0 - func(n, 0, data); + func(&q->context[0].th, n, 0, data); atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); @@ -148,7 +154,7 @@ size_t work_queue_alignof(void) { return 4096; } -work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size, dma_queue_t * dma, struct htp_thread_trace * trace) { capacity = hex_ceil_pow2(capacity); uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; unsigned char * mem_blob = (unsigned char *) ptr; @@ -161,11 +167,14 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, q->external_mem = true; q->capacity = capacity; + for (unsigned int i = 0; i < n_threads; i++) { + htp_thread_init(&q->context[i].th, dma, trace); + q->context[i].queue = q; + } + for (unsigned int i = 0; i < n_workers; i++) { q->stack[i] = mem_blob; mem_blob += stack_size; q->thread[i] = 0; - q->context[i].id = i + 1; - q->context[i].queue = q; } atomic_init(&q->idx_read, 0); @@ -190,7 +199,7 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, qurt_thread_attr_set_stack_size(&attr, stack_size); char thread_name[32]; - snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i); + snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i + 1); qurt_thread_attr_set_name(&attr, thread_name); // set up priority - by default, match the creating thread's prio @@ -204,7 +213,7 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, qurt_thread_attr_set_priority(&attr, prio); - int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]); + int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i + 1]); if (err) { FARF(ERROR, "Could not launch worker threads!"); work_queue_free(q); diff --git a/ggml/src/ggml-hexagon/htp/work-queue.h b/ggml/src/ggml-hexagon/htp/work-queue.h index 09ca4b1f4..446b72c46 100644 --- a/ggml/src/ggml-hexagon/htp/work-queue.h +++ b/ggml/src/ggml-hexagon/htp/work-queue.h @@ -5,34 +5,21 @@ #include <stdint.h> #include <stddef.h> -typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *); +#include "htp-thread.h" + +typedef void (*work_queue_func_t)(const htp_thread_t * th, unsigned int n, unsigned int i, void *); struct work_queue_s; typedef struct work_queue_s * work_queue_t; -#define WORK_QUEUE_MAX_N_THREADS 10 - size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size); size_t work_queue_alignof(void); -work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size, dma_queue_t * dma, struct htp_thread_trace * trace); void work_queue_free(work_queue_t q); void work_queue_wakeup(work_queue_t q); void work_queue_suspend(work_queue_t q); -bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); - -static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { - if (n <= 1) { - func(n, 0, data); - return true; - } - return work_queue_run_async(q, func, data, n); -} - -// Legacy compatibility -typedef work_queue_func_t worker_callback_t; -#define worker_pool_run_func work_queue_run -#define worker_pool work_queue +bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); #endif // #ifndef HTP_WORK_QUEUE_H diff --git a/ggml/src/ggml-hexagon/libggml-htp.inf b/ggml/src/ggml-hexagon/libggml-htp.inf index 874dde1b8..ff6ac27cd 100644 --- a/ggml/src/ggml-hexagon/libggml-htp.inf +++ b/ggml/src/ggml-hexagon/libggml-htp.inf @@ -18,6 +18,7 @@ libggml-htp-v73.so = 1 libggml-htp-v75.so = 1 libggml-htp-v79.so = 1 libggml-htp-v81.so = 1 +libggml-htp-v85.so = 1 [ControlFlags] ExcludeFromSelect = * @@ -30,6 +31,7 @@ libggml-htp-v73.so,,,0x10 ;COPYFLG_NO_OVERWRITE libggml-htp-v75.so,,,0x10 ;COPYFLG_NO_OVERWRITE libggml-htp-v79.so,,,0x10 ;COPYFLG_NO_OVERWRITE libggml-htp-v81.so,,,0x10 ;COPYFLG_NO_OVERWRITE +libggml-htp-v85.so,,,0x10 ;COPYFLG_NO_OVERWRITE [Strings] GGML = 'GGML' diff --git a/ggml/src/ggml-hip/CMakeLists.txt b/ggml/src/ggml-hip/CMakeLists.txt index 9b963dd5f..8dec5eedc 100644 --- a/ggml/src/ggml-hip/CMakeLists.txt +++ b/ggml/src/ggml-hip/CMakeLists.txt @@ -12,7 +12,7 @@ list(APPEND CMAKE_PREFIX_PATH ${ROCM_PATH}) list(APPEND CMAKE_PREFIX_PATH "${ROCM_PATH}/lib64/cmake") if (NOT DEFINED CMAKE_HIP_FLAGS_DEBUG) - set(CMAKE_HIP_FLAGS_DEBUG "-g -O2") + set(CMAKE_HIP_FLAGS_DEBUG "-g -Xarch_device -O2 -Xarch_host -O0") endif() # CMake on Windows doesn't support the HIP language yet diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index ae26e0c23..7c6a605af 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -107,6 +107,13 @@ static inline bool ggml_impl_is_view(const struct ggml_tensor * t) { static inline float ggml_compute_softplus_f32(float input) { return (input > 20.0f) ? input : logf(1 + expf(input)); } + +// compute correction dims for YaRN RoPE scaling +GGML_API void ggml_rope_yarn_corr_dims( + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, bool truncate, float dims[2]); + +GGML_API bool ggml_rope_get_truncate(const struct ggml_tensor * a); + // // logging // @@ -512,6 +519,7 @@ static inline float ggml_e8m0_to_fp32_half(uint8_t x) { // UE4M3: unsigned, 4 exp bits (bias=7), 3 mantissa bits // Returns value * 0.5 to match kvalues_mxfp4 convention (kvalues = 2 * E2M1_float) static inline float ggml_ue4m3_to_fp32(uint8_t x) { + // TODO: reject invalid unsigned E4M3 scales instead of decoding NaN as zero or ignoring the sign bit. if (x == 0 || x == 0x7F) { return 0.0f; } @@ -527,6 +535,7 @@ static inline float ggml_ue4m3_to_fp32(uint8_t x) { } static inline uint8_t ggml_fp32_to_ue4m3(float x) { + // TODO: use ties-to-even rounding, allow subnormal carry, and preserve finite values with exponent field 15. if (!(x > 0.0f)) { return 0; } @@ -564,6 +573,62 @@ static inline uint8_t ggml_fp32_to_ue4m3(float x) { return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man); } +static inline float ggml_f8_e4m3_to_fp32(uint8_t x) { + const uint8_t ax = x & 0x7F; + if (ax == 0x7F) { + // OCP FP8 defines both 0x7F and 0xFF as NaN; the sign need not be preserved. + return NAN; + } + + const int exp = (ax >> 3) & 0xF; + const int man = ax & 0x7; + // OCP FP8 rev. 1.1, section 5.1: subnormal magnitude = M * 2^-9; normal magnitude = (1 + M / 8) * 2^(E - 7). + const float value = exp == 0 + ? ldexpf((float) man, -9) + : ldexpf(1.0f + (float) man / 8.0f, exp - 7); + return x & 0x80 ? -value : value; +} + +static inline int ggml_round_to_nearest_even(float x) { + const int value = (int) floorf(x); + const float fraction = x - value; + return fraction > 0.5f || (fraction == 0.5f && (value & 1)) ? value + 1 : value; +} + +// Supports saturating conversion only; infinities and overflow clamp to +/-448. +static inline uint8_t ggml_fp32_to_f8_e4m3(float x) { + const uint8_t sign = signbit(x) ? 0x80 : 0; + x = fabsf(x); + + if (isnan(x)) { + // OCP FP8 leaves NaN sign handling to the implementation; use canonical 0x7F. + return 0x7F; + } + if (x == 0.0f) { + return sign; + } + if (isinf(x) || x >= 448.0f) { + return sign | 0x7E; + } + if (x < 0.015625f) { + // OCP FP8 rev. 1.1, section 5.1: subnormal magnitude = M * 2^(1 - bias - m) = M / 512. + return sign | (uint8_t) ggml_round_to_nearest_even(x * 512.0f); + } + + int exp; + const float mantissa = frexpf(x, &exp) * 2.0f; + int encoded_exp = exp + 6; + int encoded_man = ggml_round_to_nearest_even((mantissa - 1.0f) * 8.0f); + if (encoded_man == 8) { + encoded_man = 0; + encoded_exp++; + } + if (encoded_exp > 15 || (encoded_exp == 15 && encoded_man > 6)) { + return sign | 0x7E; + } + return sign | (uint8_t) (encoded_exp << 3) | (uint8_t) encoded_man; +} + /** * Converts brain16 to float32. * diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt index 05cdea156..91691b223 100644 --- a/ggml/src/ggml-metal/CMakeLists.txt +++ b/ggml/src/ggml-metal/CMakeLists.txt @@ -45,6 +45,7 @@ set(METALLIB_KERNEL_SOURCES kernels/fa_q5_0.metal kernels/fa_q5_1.metal kernels/fa_q8_0.metal + kernels/fa_tensor.metal kernels/fa_vec_f16.metal kernels/fa_vec_f32.metal kernels/fa_vec_q4_0.metal @@ -233,12 +234,12 @@ else() VERBATIM ) - set(AIR_FA_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/fa_f16_tensor.air") + set(AIR_FA_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/fa_tensor_tensor.air") add_custom_command( OUTPUT ${AIR_FA_TENSOR} - COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/fa_f16.metal -o ${AIR_FA_TENSOR} - DEPENDS kernels/fa_f16.metal ${METALLIB_KERNELS_FA_SHARED} kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h - COMMENT "Compiling kernels/fa_f16.metal (tensor API)" + COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/fa_tensor.metal -o ${AIR_FA_TENSOR} + DEPENDS kernels/fa_tensor.metal kernels/common.h ${METALLIB_COMMON} ggml-metal-impl.h + COMMENT "Compiling kernels/fa_tensor.metal (tensor API)" VERBATIM ) diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp index 8b065f54a..f36312254 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.cpp +++ b/ggml/src/ggml-metal/ggml-metal-common.cpp @@ -38,14 +38,15 @@ bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgr return !ggml_is_transposed(op->src[0]) && !ggml_is_transposed(op->src[1]) && - has_simdgroup_mm && ne00 >= 64 && ne11 > 8; + has_simdgroup_mm && ne11 > 8 && ne00 >= 32; } bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) { const int64_t ne00 = op->src[0]->ne[0]; + const int64_t ne02 = op->src[0]->ne[2]; const int64_t ne21 = op->src[2]->ne[1]; - return has_simdgroup_mm && ne00 >= 64 && ne21 >= 32; + return has_simdgroup_mm && ne00 >= 64 && ne02 <= 1024 && ne21 >= 32; } // the most src1 rows of the few-row MMA kernels @@ -80,6 +81,22 @@ static bool ggml_metal_mul_mv_mma_type_supported(enum ggml_type type) { case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: + case GGML_TYPE_BF16: + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_TQ2_0: return true; default: return false; @@ -105,10 +122,18 @@ static int64_t ggml_metal_mul_mv_mma_rows_min(enum ggml_type type) { switch (type) { case GGML_TYPE_F32: return 6; + case GGML_TYPE_TQ2_0: + return 5; + case GGML_TYPE_BF16: + return 4; case GGML_TYPE_F16: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: + case GGML_TYPE_Q2_K: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: return 3; default: return 2; diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index e9064666f..e12c55a39 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -53,9 +53,9 @@ // command buffer state int n_cb; // number of extra threads used to submit the command buffers - int n_nodes_0; // number of nodes submitted by the main thread - int n_nodes_1; // remaining number of nodes submitted by the n_cb threads - int n_nodes_per_cb; + + // first node of each extra command buffer, then the end of the graph + int cb_start[GGML_METAL_MAX_COMMAND_BUFFERS + 1]; struct ggml_cgraph * gf; @@ -491,7 +491,7 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * ggml_metal_device_rsets_keep_alive(ctx->dev); // submit the ggml compute graph to the GPU by creating command buffers and encoding the ops in them - // the first n_nodes_0 are encoded and submitted for processing directly by the calling thread + // the first n_main nodes are encoded and submitted for processing directly by the calling thread // while these nodes are processing, we start n_cb threads to enqueue the rest of the nodes // each thread creates it's own command buffer and enqueues the ops in parallel // @@ -500,16 +500,22 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; - if (ctx->n_cb == 0) { - // single-threaded encoding: the whole graph is encoded by one command buffer - ctx->n_nodes_0 = gf->n_nodes; - ctx->n_nodes_1 = 0; - ctx->n_nodes_per_cb = 0; - } else { - ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); - ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; - - ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + // the main thread encodes [0, cb_start[0]) and the extra command buffer i encodes [cb_start[i], cb_start[i + 1]) + // a fusion cannot span two command buffers, so each boundary moves to the end of the fusable group it falls + // into and the encoders fuse the same nodes as with a single command buffer + { + const int n_rest = gf->n_nodes - MIN(n_main, gf->n_nodes); + const int n_per_cb = n_cb > 0 ? (n_rest + n_cb - 1) / n_cb : 0; + + int i = 0; + for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) { + const int target = MIN(n_main + cb_idx*n_per_cb, gf->n_nodes); + while (i < target) { + i += ggml_metal_fusion_max(gf, i); + } + ctx->cb_start[cb_idx] = MIN(i, gf->n_nodes); + } + ctx->cb_start[n_cb] = gf->n_nodes; } if (ctx->capture_compute >= 0) { @@ -539,8 +545,14 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * descriptor.outputURL = [NSURL fileURLWithPath:path]; NSError * error = nil; - if (![[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]) { + const bool ok = [[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]; + + [descriptor release]; + + if (!ok) { GGML_LOG_ERROR("%s: error: unable to start capture '%s' (did you set METAL_CAPTURE_ENABLED=1 ?)\n", __func__, [[error localizedDescription] UTF8String]); + [ctx->capture_scope release]; + ctx->capture_scope = nil; } else { [ctx->capture_scope beginScope]; ctx->capture_started = true; @@ -600,6 +612,8 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * // enter here only when capturing in order to wait for all computation to finish // otherwise, we leave the graph to compute asynchronously if (use_capture && ctx->capture_started) { + enum ggml_status res = GGML_STATUS_SUCCESS; + // wait for completion and check status of each command buffer // needed to detect if the device ran out-of-memory for example (#1881) { @@ -613,11 +627,11 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * GGML_LOG_INFO("error: %s\n", [[cmd_buf error].localizedDescription UTF8String]); } - return GGML_STATUS_FAILED; + res = GGML_STATUS_FAILED; } } - for (int i = 0; i < n_cb; ++i) { + for (int i = 0; i < n_cb && res == GGML_STATUS_SUCCESS; ++i) { id<MTLCommandBuffer> cmd_buf = ctx->cmd_bufs[i].obj; [cmd_buf waitUntilCompleted]; @@ -627,8 +641,8 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * if (status == MTLCommandBufferStatusError) { GGML_LOG_INFO("error: %s\n", [[cmd_buf error].localizedDescription UTF8String]); } - - return GGML_STATUS_FAILED; + res = GGML_STATUS_FAILED; + break; } id<MTLCommandBuffer> next_buffer = (i + 1 < n_cb ? ctx->cmd_bufs[i + 1].obj : nil); @@ -643,7 +657,8 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * if (ctx->abort_callback && ctx->abort_callback(ctx->abort_callback_data)) { GGML_LOG_INFO("%s: command buffer %d aborted", __func__, i); - return GGML_STATUS_ABORTED; + res = GGML_STATUS_ABORTED; + break; } [next_buffer commit]; @@ -652,7 +667,14 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * [ctx->capture_scope endScope]; [[MTLCaptureManager sharedCaptureManager] stopCapture]; + [ctx->capture_scope release]; + ctx->capture_scope = nil; + ctx->capture_started = false; + + if (res != GGML_STATUS_SUCCESS) { + return res; + } } } @@ -728,17 +750,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { const int cb_idx = iter; const int n_cb_l = ctx->n_cb; - const int n_nodes_0 = ctx->n_nodes_0; - const int n_nodes_1 = ctx->n_nodes_1; - - const int n_nodes_per_cb = ctx->n_nodes_per_cb; - int idx_start = 0; - int idx_end = n_nodes_0; + int idx_end = ctx->cb_start[0]; if (cb_idx < n_cb_l) { - idx_start = n_nodes_0 + ( (cb_idx + 0) * n_nodes_per_cb); - idx_end = n_nodes_0 + (MIN((cb_idx == n_cb_l - 1) ? n_nodes_1 : (cb_idx + 1) * n_nodes_per_cb, n_nodes_1)); + idx_start = ctx->cb_start[cb_idx]; + idx_end = ctx->cb_start[cb_idx + 1]; } id<MTLCommandBuffer> cmd_buf = ctx->cmd_bufs[cb_idx].obj; diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 8ac6e0830..0d18a5f01 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -507,6 +507,47 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexe return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_mask( + ggml_metal_library_t lib, + const ggml_tensor * op, + bool fill, + bool has_score, + bool has_blocks, + bool has_extra, + bool vec) { + GGML_ASSERT(op->op == GGML_OP_TOP_K_MASK); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_top_k_mask%s_%s", fill ? "_fill" : "", ggml_type_name(op->type)); + + ggml_metal_pipeline_with_params res; + + ggml_metal_cv_t cv = ggml_metal_cv_init(); + + if (fill) { + snprintf(name, 256, "%s_vec=%d", base, vec ? 1 : 0); + + ggml_metal_cv_set_bool(cv, vec, FC_TOP_K_MASK + 3); + } else { + snprintf(name, 256, "%s_sc=%d_bl=%d_ex=%d", base, has_score ? 1 : 0, has_blocks ? 1 : 0, has_extra ? 1 : 0); + + ggml_metal_cv_set_bool(cv, has_score, FC_TOP_K_MASK + 0); + ggml_metal_cv_set_bool(cv, has_blocks, FC_TOP_K_MASK + 1); + ggml_metal_cv_set_bool(cv, has_extra, FC_TOP_K_MASK + 2); + } + + res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + } + + ggml_metal_cv_free(cv); + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, const ggml_tensor * op) { char name[256]; const char * base = nullptr; @@ -1062,7 +1103,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta { nsg = N_SG_MXFP4; nr0 = N_R0_MXFP4; - smem = 32*sizeof(float); } break; case GGML_TYPE_Q2_K: { @@ -1225,55 +1265,36 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(ggml_metal_library_t lib, int ne02, int ne20) { char base[256]; char name[256]; - snprintf(base, 256, "kernel_mul_mm_id_amax_part_f32"); - snprintf(name, 256, "%s", base); + snprintf(base, 256, "kernel_mul_mm_id_map0_ne20_%d", ne20); + snprintf(name, 256, "%s_ne02=%d", base, ne02); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); - } - - res.smem = 32*sizeof(float); - - return res; -} - -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib) { - char base[256]; - char name[256]; + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_int32(cv, ne02, FC_MUL_MM + 7); - snprintf(base, 256, "kernel_mul_mm_id_amax_f32"); - snprintf(name, 256, "%s", base); + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); - ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); - if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + ggml_metal_cv_free(cv); } - res.smem = 32*sizeof(float); + // per expert: entry count + base offset, followed by the staging of the ids of ntg tokens + res.smem = 2*(size_t) ne02*sizeof(uint32_t) + (size_t) ne02*ne20*sizeof(uint16_t); + res.smem = GGML_PAD(res.smem, 16); return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(ggml_metal_library_t lib, int ne02, int ne20) { - char base[256]; - char name[256]; - - snprintf(base, 256, "kernel_mul_mm_id_map0_ne20_%d", ne20); - snprintf(name, 256, "%s_ne02=%d", base, ne02); - - ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_zero(ggml_metal_library_t lib) { + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, "kernel_mul_mm_id_zero"); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + res = ggml_metal_library_compile_pipeline(lib, "kernel_mul_mm_id_zero", "kernel_mul_mm_id_zero", nullptr); } - res.smem = (size_t) ne02*ne20*sizeof(uint16_t); - res.smem = GGML_PAD(res.smem, 16); - return res; } @@ -1286,18 +1307,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id(ggml_m const bool bc_inp = op->src[0]->ne[0] % 32 != 0; - // src1 prec [TAG_GGML_PREC] - const bool amax = ggml_get_op_params_i32(op, 3) == GGML_PREC_F32; - snprintf(base, 256, "kernel_mul_mm_id_%s_%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1)); - snprintf(name, 256, "%s_bci=%d_amax=%d", base, bc_inp, amax); + snprintf(name, 256, "%s_bci=%d", base, bc_inp); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); ggml_metal_cv_set_bool(cv, bc_inp, FC_MUL_MM + 0); - ggml_metal_cv_set_bool(cv, amax, FC_MUL_MM + 6); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -1381,7 +1398,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m { nsg = N_SG_MXFP4; nr0 = N_R0_MXFP4; - smem = 32*sizeof(float); } break; case GGML_TYPE_Q2_K: { @@ -1561,6 +1577,28 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax(ggml_meta return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax_part(ggml_metal_library_t lib) { + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, "kernel_argmax_part_f32"); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, "kernel_argmax_part_f32", "kernel_argmax_part_f32", nullptr); + } + + res.smem = 32*(sizeof(float) + sizeof(int32_t)); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax_final(ggml_metal_library_t lib) { + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, "kernel_argmax_final_f32"); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, "kernel_argmax_final_f32", "kernel_argmax_final_f32", nullptr); + } + + res.smem = 32*(sizeof(float) + sizeof(int32_t)); + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_ARGSORT); @@ -1700,12 +1738,13 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge(ggml } ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe( - ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm) { + ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm, int32_t gating, bool pbias) { char base[256]; char name[256]; snprintf(base, 256, "kernel_topk_moe_f32"); - snprintf(name, 256, "%s_n_expert=%d_top_k=%d_with_norm=%d", base, n_expert, top_k, with_norm ? 1 : 0); + snprintf(name, 256, "%s_n_expert=%d_top_k=%d_with_norm=%d_gating=%d_pbias=%d", + base, n_expert, top_k, with_norm ? 1 : 0, gating, pbias ? 1 : 0); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { @@ -1713,6 +1752,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe( ggml_metal_cv_set_bool (cv, with_norm, FC_TOPK_MOE + 0); ggml_metal_cv_set_int32(cv, n_expert, FC_TOPK_MOE + 1); ggml_metal_cv_set_int32(cv, top_k, FC_TOPK_MOE + 2); + ggml_metal_cv_set_int32(cv, gating, FC_TOPK_MOE + 3); + ggml_metal_cv_set_bool (cv, pbias, FC_TOPK_MOE + 4); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -1860,7 +1901,9 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_t const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - snprintf(base, 256, "kernel_flash_attn_ext_tensor_f16_dk%d_dv%d", dk, dv); + const char * type = op->src[1]->type == GGML_TYPE_BF16 ? "bf16" : "f16"; + + snprintf(base, 256, "kernel_flash_attn_ext_tensor_%s_dk%d_dv%d", type, dk, dv); snprintf(name, 256, "%s_mask=%d_sinks=%d_bias=%d_scap=%d", base, has_mask, has_sinks, has_bias, has_scap); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); @@ -1947,6 +1990,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_rows( + ggml_metal_library_t lib, + const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + const char * base = "kernel_flash_attn_ext_rows"; + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, base); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, base, nullptr); + } + + GGML_UNUSED(op); + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx( ggml_metal_library_t lib, const ggml_tensor * op) { @@ -1976,6 +2036,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v bool has_scap, bool has_kvpad, bool has_sparse, + bool has_rows, int32_t nqpsg, int32_t ne, int32_t nsg, @@ -2005,7 +2066,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v dv, qne_suffix); - snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_sparse=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d", + snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_sparse=%d_rows=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d", base, has_mask, has_sinks, @@ -2013,6 +2074,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v has_scap, has_kvpad, has_sparse, + has_rows, ns10, ns20, nsg, nwg); @@ -2027,6 +2089,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v ggml_metal_cv_set_bool(cv, has_scap, FC_FLASH_ATTN_EXT_VEC + 3); ggml_metal_cv_set_bool(cv, has_kvpad, FC_FLASH_ATTN_EXT_VEC + 4); ggml_metal_cv_set_bool(cv, has_sparse, FC_FLASH_ATTN_EXT_VEC + 5); + ggml_metal_cv_set_bool(cv, has_rows, FC_FLASH_ATTN_EXT_VEC + 6); ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT_VEC + 20); ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT_VEC + 21); @@ -2178,17 +2241,34 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_l2_norm(ggml_met ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_group_norm(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_GROUP_NORM); - GGML_ASSERT(ggml_is_contiguous(op->src[0])); + GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); char base[256]; char name[256]; snprintf(base, 256, "kernel_group_norm_f32"); - snprintf(name, 256, "%s", base); + + const ggml_tensor * src0 = op->src[0]; + + const bool flat = src0->nb[1] == src0->ne[0]*src0->nb[0] && + src0->nb[2] == src0->nb[1]*src0->ne[1] && + op->nb[1] == op->ne[0] *op->nb[0] && + op->nb[2] == op->nb[1] *op->ne[1]; + + if (flat) { + snprintf(name, 256, "%s", base); + } else { + snprintf(name, 256, "%s_strided", base); + } ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool(cv, flat, FC_GROUP_NORM + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); } res.smem = 32*sizeof(float); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 4012b9d5f..4053a624e 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -126,6 +126,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_ad struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_mask (ggml_metal_library_t lib, const struct ggml_tensor * op, bool fill, bool has_score, bool has_blocks, bool has_extra, bool vec); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t nc, bool use_silu); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs, int32_t nc, bool use_silu); @@ -140,11 +141,12 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_mm struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0 (ggml_metal_library_t lib, int ne02, int ne20); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_zero (ggml_metal_library_t lib); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax_part (ggml_metal_library_t lib); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax_final (ggml_metal_library_t lib); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht (ggml_metal_library_t lib, int n, enum ggml_type tsrc); @@ -152,7 +154,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_radix (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe (ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe (ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm, int32_t gating, bool pbias); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_moe_reduce (ggml_metal_library_t lib, int32_t n_expert_used); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin_one (ggml_metal_library_t lib, enum ggml_op op); @@ -217,6 +219,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att int32_t ns10, int32_t ns20); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_rows( + ggml_metal_library_t lib, + const struct ggml_tensor * op); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx( ggml_metal_library_t lib, const struct ggml_tensor * op); @@ -230,6 +236,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_scap, bool has_kvpad, bool has_sparse, + bool has_rows, int32_t nqpsg, int32_t ne, int32_t nsg, @@ -360,7 +367,10 @@ bool ggml_metal_buffer_is_shared(ggml_metal_buffer_t buf); void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); void ggml_metal_buffer_set_tensor (ggml_metal_buffer_t buf, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); void ggml_metal_buffer_get_tensor (ggml_metal_buffer_t buf, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); +void ggml_metal_buffer_set_tensor_n (ggml_metal_buffer_t buf, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); +void ggml_metal_buffer_get_tensor_n (ggml_metal_buffer_t buf, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); bool ggml_metal_buffer_cpy_tensor (ggml_metal_buffer_t buf, const struct ggml_tensor * src, struct ggml_tensor * dst); +bool ggml_metal_buffer_cpy_tensor_n (ggml_metal_buffer_t buf, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors); void ggml_metal_buffer_clear (ggml_metal_buffer_t buf, uint8_t value); // finds the Metal buffer that contains the tensor data on the GPU device diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 827d92622..0a07a76bf 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -118,6 +118,7 @@ int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_wi X(FA_Q5_0, fa_q5_0) \ X(FA_Q5_1, fa_q5_1) \ X(FA_Q8_0, fa_q8_0) \ + X(FA_TENSOR, fa_tensor) \ X(FA_VEC_F16, fa_vec_f16) \ X(FA_VEC_F32, fa_vec_f32) \ X(FA_VEC_Q4_0, fa_vec_q4_0) \ @@ -1509,12 +1510,25 @@ void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t } } +static bool ggml_metal_supports_mul_mat_src1_prec(const struct ggml_tensor * op, bool has_tensor) { + const enum ggml_prec prec = ggml_prec_get_src(op, 1); + + if (prec == GGML_PREC_F32) { + return false; + } + if (prec == GGML_PREC_BF16) { + return has_tensor && op->op == GGML_OP_MUL_MAT && op->src[0]->type != GGML_TYPE_F32; + } + return true; +} + static bool ggml_metal_supports_mul_mat_op( bool has_simdgroup_reduction, const struct ggml_tensor * op, bool src0_f16_has_mv, bool mm_path) { if (!has_simdgroup_reduction || + op->src[0]->type == GGML_TYPE_F8_E4M3 || op->src[0]->type == GGML_TYPE_NVFP4 || op->src[0]->type == GGML_TYPE_TQ1_0) { return false; @@ -1554,6 +1568,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te switch (op->op) { case GGML_OP_SCALE: + return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16); case GGML_OP_FILL: case GGML_OP_CLAMP: case GGML_OP_SQR: @@ -1678,9 +1693,13 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_CUMSUM: case GGML_OP_MEAN: case GGML_OP_SOFT_MAX: - case GGML_OP_GROUP_NORM: case GGML_OP_L2_NORM: return has_simdgroup_reduction && ggml_is_contiguous_rows(op->src[0]); + case GGML_OP_GROUP_NORM: + return has_simdgroup_reduction && + op->src[0]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + ggml_is_contiguous_rows(op->src[0]); case GGML_OP_COUNT_EQUAL: return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_I32 && @@ -1693,7 +1712,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return has_simdgroup_reduction && (ggml_is_contiguous_rows(op->src[0])); case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: - return true; + return ggml_rope_get_truncate(op); case GGML_OP_IM2COL: return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32); case GGML_OP_CONV_2D: @@ -1724,6 +1743,39 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_ROLL: return ggml_is_contiguous(op->src[0]); case GGML_OP_FLASH_ATTN_EXT: + // K/V row lists (ggml_flash_attn_ext_rows): one list of cache rows per q slice + if (op->src[5] != NULL) { + const struct ggml_tensor * kv_rows = op->src[5]; + + if (kv_rows->type != GGML_TYPE_I32 || + !ggml_is_contiguous(kv_rows) || + kv_rows->ne[2] != 1 || kv_rows->ne[3] != 1 || + kv_rows->ne[1] != op->src[0]->ne[3] || + !op->src[3] || op->src[1]->ne[3] != 1 || op->src[2]->ne[3] != 1) { + return false; + } + + // the non-vec kernels attend over a contiguous F16 copy of the referenced rows: + // - F16 K/V is gathered directly + // - quantized K/V is first dequantized with kernel_flash_attn_ext_kv_*_f16 + // - other types keep attending to the whole cache (dense mask) + switch (op->src[1]->type) { + case GGML_TYPE_F16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + break; + default: + return false; + } + + // the row gather copies F16 values in groups of 4 + if (op->src[2]->ne[0] % 4 != 0) { + return false; + } + } // for new head sizes, add checks here if (op->src[0]->ne[0] != 32 && op->src[0]->ne[0] != 40 && @@ -1741,10 +1793,13 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[0]->ne[0] != 576) { return false; } - if (op->src[1]->ne[0] == 72 && op->src[1]->ne[0] != op->src[2]->ne[0]) { - return false; - } - if (op->src[1]->ne[0] < op->src[2]->ne[0]) { + // the kernels exist for K == V and for these K > V pairs only + if (op->src[1]->ne[0] != op->src[2]->ne[0] && + !(op->src[1]->ne[0] == 96 && op->src[2]->ne[0] == 64) && + !(op->src[1]->ne[0] == 128 && op->src[2]->ne[0] == 96) && + !(op->src[1]->ne[0] == 192 && op->src[2]->ne[0] == 128) && + !(op->src[1]->ne[0] == 320 && op->src[2]->ne[0] == 256) && + !(op->src[1]->ne[0] == 576 && op->src[2]->ne[0] == 512)) { return false; } if (op->src[1]->type != op->src[2]->type) { @@ -1830,6 +1885,15 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te ggml_is_contiguous_rows(op->src[1]) && ggml_is_contiguous_rows(op->src[2]) && (op->src[3] == NULL || ggml_is_contiguous_rows(op->src[3])); + case GGML_OP_TOP_K_MASK: + return (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32) && + op->src[0]->type == op->type && + op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && + op->nb[0] == ggml_type_size(op->type) && + op->src[1]->type == GGML_TYPE_I32 && + (op->src[2] == NULL || op->src[2]->type == GGML_TYPE_F32) && + (op->src[3] == NULL || op->src[3]->type == GGML_TYPE_I32) && + (op->src[4] == NULL || op->src[4]->type == GGML_TYPE_I32); case GGML_OP_SSM_SCAN: return has_simdgroup_reduction; case GGML_OP_SSM_CONV: @@ -1842,6 +1906,9 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_SOLVE_TRI: return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_MUL_MAT: + if (!ggml_metal_supports_mul_mat_src1_prec(op, dev->props.has_tensor)) { + return false; + } // the FWHT kernels read an F16 source directly; every other F16 src1 path // still goes through ggml_metal_supports_mul_mat_op if (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F16 && @@ -1852,6 +1919,9 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te has_simdgroup_reduction, op, true, ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm)); case GGML_OP_MUL_MAT_ID: + if (!ggml_metal_supports_mul_mat_src1_prec(op, dev->props.has_tensor)) { + return false; + } return ggml_metal_supports_mul_mat_op( has_simdgroup_reduction, op, false, ggml_metal_op_mul_mat_id_use_mm(op, has_simdgroup_mm)); @@ -1919,7 +1989,8 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te } case GGML_OP_GET_ROWS: return op->src[0]->type != GGML_TYPE_NVFP4 && - op->src[0]->type != GGML_TYPE_TQ1_0; + op->src[0]->type != GGML_TYPE_TQ1_0 && + op->src[0]->type != GGML_TYPE_F8_E4M3; case GGML_OP_SET_ROWS: { if (op->src[0]->type == GGML_TYPE_F16) { @@ -2086,19 +2157,11 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) { static void * ggml_metal_host_malloc(size_t n) { void * data = NULL; -#if TARGET_OS_OSX kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE); if (err != KERN_SUCCESS) { GGML_LOG_ERROR("%s: error: vm_allocate failed\n", __func__); return NULL; } -#else - const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n); - if (result != 0) { - GGML_LOG_ERROR("%s: error: posix_memalign failed\n", __func__); - return NULL; - } -#endif return data; } @@ -2280,11 +2343,7 @@ void ggml_metal_buffer_free(ggml_metal_buffer_t buf) { } if (buf->is_shared && buf->owned) { -#if TARGET_OS_OSX vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)buf->all_data, buf->all_size); -#else - free(buf->all_data); -#endif } free(buf); @@ -2422,6 +2481,148 @@ void ggml_metal_buffer_get_tensor(ggml_metal_buffer_t buf, const struct ggml_ten } } +static bool ggml_metal_buffer_data_span(const void * const * data, const size_t * sizes, size_t n_tensors, uintptr_t * min_data, uintptr_t * max_data) { + bool found = false; + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + + const uintptr_t start = (uintptr_t) data[i]; + const uintptr_t end = start + sizes[i]; + if (!found) { + *min_data = start; + *max_data = end; + found = true; + } else { + if (start < *min_data) { + *min_data = start; + } + if (end > *max_data) { + *max_data = end; + } + } + } + + return found; +} + +void ggml_metal_buffer_set_tensor_n(ggml_metal_buffer_t buf, struct ggml_tensor * const * tensors, const void * const * data, + const size_t * offsets, const size_t * sizes, size_t n_tensors) { + if (buf->is_shared) { + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]); + } + return; + } + + uintptr_t min_data = 0, max_data = 0; + if (!ggml_metal_buffer_data_span(data, sizes, n_tensors, &min_data, &max_data)) { + return; + } + + @autoreleasepool { + void * data_ptr = (void *) min_data; + id<MTLBuffer> buf_src = [buf->dev->mtl_device newBufferWithBytesNoCopy:data_ptr + length:max_data - min_data + options:MTLResourceStorageModeShared + deallocator:nil]; + + GGML_ASSERT(buf_src); + + id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences]; + + { + id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder]; + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + + ggml_backend_buffer_t tensor_buf_ggml = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer; + ggml_metal_buffer_t tensor_buf = (ggml_metal_buffer_t) tensor_buf_ggml->context; + struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(tensor_buf, tensors[i]); + bid_dst.offs += offsets[i]; + + [encoder copyFromBuffer:buf_src + sourceOffset:(uintptr_t) data[i] - min_data + toBuffer:bid_dst.metal + destinationOffset:bid_dst.offs + size:sizes[i]]; + } + + [encoder endEncoding]; + } + + [cmd_buf commit]; + [cmd_buf waitUntilCompleted]; + + [buf_src release]; + } +} + +void ggml_metal_buffer_get_tensor_n(ggml_metal_buffer_t buf, const struct ggml_tensor * const * tensors, void * const * data, + const size_t * offsets, const size_t * sizes, size_t n_tensors) { + if (buf->is_shared) { + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + memcpy(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i]); + } + return; + } + + uintptr_t min_data = 0, max_data = 0; + if (!ggml_metal_buffer_data_span((const void * const *) data, sizes, n_tensors, &min_data, &max_data)) { + return; + } + + @autoreleasepool { + id<MTLBuffer> buf_dst = [buf->dev->mtl_device newBufferWithBytesNoCopy:(void *) min_data + length:max_data - min_data + options:MTLResourceStorageModeShared + deallocator:nil]; + + GGML_ASSERT(buf_dst); + + id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences]; + + { + id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder]; + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + + ggml_backend_buffer_t tensor_buf_ggml = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer; + ggml_metal_buffer_t tensor_buf = (ggml_metal_buffer_t) tensor_buf_ggml->context; + struct ggml_metal_buffer_id bid_src = ggml_metal_buffer_get_id(tensor_buf, tensors[i]); + bid_src.offs += offsets[i]; + + [encoder copyFromBuffer:bid_src.metal + sourceOffset:bid_src.offs + toBuffer:buf_dst + destinationOffset:(uintptr_t) data[i] - min_data + size:sizes[i]]; + } + + [encoder endEncoding]; + } + + [cmd_buf commit]; + [cmd_buf waitUntilCompleted]; + + [buf_dst release]; + } +} + bool ggml_metal_buffer_cpy_tensor(ggml_metal_buffer_t buf_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) { ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t)src->buffer->context; @@ -2463,6 +2664,74 @@ bool ggml_metal_buffer_cpy_tensor(ggml_metal_buffer_t buf_dst, const struct ggml return true; } +bool ggml_metal_buffer_cpy_tensor_n(ggml_metal_buffer_t buf_dst, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) { + if (n_tensors == 0) { + return true; + } + + bool all_shared = true; + for (size_t i = 0; i < n_tensors; ++i) { + ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer; + ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t) src_buf->context; + if (!buf_dst->is_shared || !buf_src->is_shared) { + all_shared = false; + break; + } + } + + if (all_shared) { + for (size_t i = 0; i < n_tensors; ++i) { + memcpy(dsts[i]->data, srcs[i]->data, ggml_nbytes(srcs[i])); + } + return true; + } + + // for private buffers, we need to use Metal blit commands + @autoreleasepool { + struct ggml_metal_buffer_id * bid_srcs = malloc(n_tensors * sizeof(struct ggml_metal_buffer_id)); + struct ggml_metal_buffer_id * bid_dsts = malloc(n_tensors * sizeof(struct ggml_metal_buffer_id)); + + for (size_t i = 0; i < n_tensors; ++i) { + ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer; + ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer; + ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t) src_buf->context; + ggml_metal_buffer_t buf_dst_i = (ggml_metal_buffer_t) dst_buf->context; + bid_srcs[i] = ggml_metal_buffer_get_id(buf_src, srcs[i]); + bid_dsts[i] = ggml_metal_buffer_get_id(buf_dst_i, dsts[i]); + + if (bid_srcs[i].metal == nil || bid_dsts[i].metal == nil) { + free(bid_srcs); + free(bid_dsts); + return false; + } + } + + id<MTLCommandBuffer> cmd_buf = [buf_dst->dev->mtl_queue commandBufferWithUnretainedReferences]; + + { + id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder]; + + for (size_t i = 0; i < n_tensors; ++i) { + [encoder copyFromBuffer:bid_srcs[i].metal + sourceOffset:bid_srcs[i].offs + toBuffer:bid_dsts[i].metal + destinationOffset:bid_dsts[i].offs + size:ggml_nbytes(srcs[i])]; + } + + [encoder endEncoding]; + } + + [cmd_buf commit]; + [cmd_buf waitUntilCompleted]; + + free(bid_srcs); + free(bid_dsts); + } + + return true; +} + void ggml_metal_buffer_clear(ggml_metal_buffer_t buf, uint8_t value) { if (buf->is_shared) { memset(buf->all_data, value, buf->all_size); diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp index d78ca7c4b..6766b4bd9 100644 --- a/ggml/src/ggml-metal/ggml-metal-fusion.cpp +++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp @@ -345,9 +345,6 @@ static bool ggml_metal_fusion_check_snake( #define GGML_METAL_TOPK_MOE_MAX_EXPERTS 1024 -// SOFT_MAX + ARGSORT + GET_ROWS (plus optional norm/scale) for MoE routing. -// This is a multi-output elision chain: the fused kernel writes both the selected -// expert ids and the gathered/normalized routing weights. static const std::vector<ggml_op> ops_topk_moe = { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }; @@ -362,6 +359,60 @@ static const std::vector<ggml_op> ops_topk_moe_norm_scale = { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE }; +static const std::vector<ggml_op> ops_topk_moe_norm_bias = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector<ggml_op> ops_topk_moe_norm_scale_bias = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; + +static const std::vector<ggml_op> ops_topk_moe_sp = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS +}; +static const std::vector<ggml_op> ops_topk_moe_sp_scale = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_SCALE +}; +static const std::vector<ggml_op> ops_topk_moe_sp_norm = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector<ggml_op> ops_topk_moe_sp_norm_scale = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; +static const std::vector<ggml_op> ops_topk_moe_sp_norm_bias = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector<ggml_op> ops_topk_moe_sp_norm_scale_bias = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; + +static const std::vector<ggml_op> ops_topk_moe_sig = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS +}; +static const std::vector<ggml_op> ops_topk_moe_sig_scale = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_SCALE +}; +static const std::vector<ggml_op> ops_topk_moe_sig_norm = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector<ggml_op> ops_topk_moe_sig_norm_scale = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; +static const std::vector<ggml_op> ops_topk_moe_sig_norm_bias = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector<ggml_op> ops_topk_moe_sig_norm_scale_bias = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; static bool ggml_metal_fusion_check_topk_moe( const ggml_metal_fusion * fusion, @@ -375,8 +426,11 @@ static bool ggml_metal_fusion_check_topk_moe( const int n_ops = (int) fusion->ops.size(); - const bool with_norm = n_ops >= 6; - const bool with_scale = n_ops == 4 || n_ops == 7; + bool with_norm = false; + for (int j = 0; j < n_ops; ++j) { + with_norm |= fusion->ops[j] == GGML_OP_SUM_ROWS; + } + const bool with_scale = fusion->ops[n_ops - 1] == GGML_OP_SCALE; // the fusion table operates on the non-empty node sequence; the raw graph also // contains the RESHAPE/VIEW nodes that the fused kernel elides. @@ -411,24 +465,70 @@ static bool ggml_metal_fusion_check_topk_moe( } } - const ggml_tensor * softmax = gf->nodes[raw_start]; - const ggml_tensor * probs_reshaped = gf->nodes[raw_start + 1]; - const ggml_tensor * argsort = gf->nodes[raw_start + 2]; - const ggml_tensor * ids = gf->nodes[raw_start + 3]; - const ggml_tensor * get_rows = gf->nodes[raw_start + 4]; - const ggml_tensor * out = gf->nodes[raw_end]; - const ggml_tensor * logits = softmax->src[0]; - - // the fused kernel implements plain softmax only - float scale = 1.0f; - float max_bias = 0.0f; - memcpy(&scale, ((const int32_t *) softmax->op_params) + 0, sizeof(scale)); - memcpy(&max_bias, ((const int32_t *) softmax->op_params) + 1, sizeof(max_bias)); - if (scale != 1.0f || max_bias != 0.0f || softmax->src[1] || softmax->src[2]) { + const ggml_tensor * first = gf->nodes[raw_start]; + const ggml_tensor * out = gf->nodes[raw_end]; + + // the unary chains start at the sigmoid or the softplus node and rank biased scores + const bool with_unary = first->op == GGML_OP_UNARY; + const bool with_sp = with_unary && ggml_get_unary_op(first) == GGML_UNARY_OP_SOFTPLUS; + const bool with_sig = with_unary && ggml_get_unary_op(first) == GGML_UNARY_OP_SIGMOID; + + if (with_unary && !with_sp && !with_sig) { return false; } - if (logits->type != GGML_TYPE_F32 || softmax->type != GGML_TYPE_F32 || + const ggml_tensor * probs; // the scoring output + const ggml_tensor * scores; // the node the argsort ranks (adds the bias for the unary chains) + const ggml_tensor * probs_reshaped; + const ggml_tensor * argsort; + const ggml_tensor * ids; + const ggml_tensor * get_rows; + const ggml_tensor * logits = first->src[0]; + int ids_off; + + if (with_unary) { + // the softplus chains add a SQRT node, the sigmoid chains start at the sigmoid node + const int base = with_sp ? 1 : 0; + + probs = gf->nodes[raw_start + base + 0]; + probs_reshaped = gf->nodes[raw_start + base + 1]; + scores = gf->nodes[raw_start + base + 2]; + argsort = gf->nodes[raw_start + base + 3]; + ids = gf->nodes[raw_start + base + 4]; + get_rows = gf->nodes[raw_start + base + 5]; + ids_off = base + 4; + + if (scores->op != GGML_OP_ADD || scores->src[0] != probs || + (with_sp && (probs->op != GGML_OP_SQRT || probs->src[0] != first))) { + return false; + } + + // the selection bias is a per-expert vector shared by all rows + const ggml_tensor * bias = scores->src[1]; + if (bias->type != GGML_TYPE_F32 || scores->type != GGML_TYPE_F32 || !ggml_is_contiguous(bias) || + bias->ne[0] != probs->ne[0] || bias->ne[1] != 1 || bias->ne[2] != 1 || bias->ne[3] != 1) { + return false; + } + } else { + probs = first; + scores = first; + probs_reshaped = gf->nodes[raw_start + 1]; + argsort = gf->nodes[raw_start + 2]; + ids = gf->nodes[raw_start + 3]; + get_rows = gf->nodes[raw_start + 4]; + ids_off = 3; + + // the fused kernel implements plain softmax only + float scale = 1.0f; + float max_bias = 0.0f; + memcpy(&scale, ((const int32_t *) first->op_params) + 0, sizeof(scale)); + memcpy(&max_bias, ((const int32_t *) first->op_params) + 1, sizeof(max_bias)); + if (scale != 1.0f || max_bias != 0.0f || first->src[1] || first->src[2]) { + return false; + } + } + + if (logits->type != GGML_TYPE_F32 || probs->type != GGML_TYPE_F32 || out->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32) { return false; } @@ -455,38 +555,40 @@ static bool ggml_metal_fusion_check_topk_moe( return false; } - if (probs_reshaped->src[0] != softmax || argsort->src[0] != softmax || + // the top-k gathers the unbiased scores, the argsort ranks the biased ones + if (probs_reshaped->src[0] != probs || argsort->src[0] != scores || ids->src[0] != argsort || get_rows->src[0] != probs_reshaped || get_rows->src[1] != ids) { return false; } if (with_norm) { - const ggml_tensor * weights_reshaped = gf->nodes[raw_start + 5]; - const ggml_tensor * sum_rows = gf->nodes[raw_start + 6]; - const ggml_tensor * clamp = gf->nodes[raw_start + 7]; - const ggml_tensor * div = gf->nodes[raw_start + 8]; - const ggml_tensor * out_reshaped = gf->nodes[raw_start + 9]; + const int norm_off = with_unary ? 6 + (with_sp ? 1 : 0) : 5; + + const ggml_tensor * weights_reshaped = gf->nodes[raw_start + norm_off + 0]; + const ggml_tensor * sum_rows = gf->nodes[raw_start + norm_off + 1]; + const ggml_tensor * denom = gf->nodes[raw_start + norm_off + 2]; + const ggml_tensor * div = gf->nodes[raw_start + norm_off + 3]; + const ggml_tensor * out_reshaped = gf->nodes[raw_start + norm_off + 4]; if (weights_reshaped->src[0] != get_rows || sum_rows->src[0] != weights_reshaped || - clamp->src[0] != sum_rows || div->src[0] != weights_reshaped || div->src[1] != clamp || + denom->src[0] != sum_rows || div->src[0] != weights_reshaped || div->src[1] != denom || out_reshaped->src[0] != div) { return false; } - if (with_scale) { - const ggml_tensor * scale_node = gf->nodes[raw_start + 10]; - if (scale_node->src[0] != out_reshaped) { - return false; - } + // the kernel applies the sum adjustment as a lower clamp or an additive bias, never a scale + if (denom->op == GGML_OP_SCALE && ggml_get_op_params_f32(denom, 0) != 1.0f) { + return false; } - } else if (with_scale) { - const ggml_tensor * scale_node = gf->nodes[raw_start + 5]; - if (scale_node->src[0] != get_rows) { + + if (with_scale && out->src[0] != out_reshaped) { return false; } + } else if (with_scale && out->src[0] != get_rows) { + return false; } - const int outputs[2] = { raw_start + 3, raw_end }; + const int outputs[2] = { raw_start + ids_off, raw_end }; if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, raw_count, ops_all.data(), outputs, 2)) { return false; } @@ -636,6 +738,11 @@ static bool ggml_metal_fusion_check_moe_reduce( if (!match.experts->data || !match.weights->data || !match.dst->data) { return false; } + + // the fused kernel reads experts and weights while writing dst, so dst must not overlap either input + if (ggml_metal_fusion_overlap(match.dst, match.experts) || ggml_metal_fusion_overlap(match.dst, match.weights)) { + return false; + } } return true; @@ -758,33 +865,54 @@ static const std::vector<ggml_op> ops_moe_reduce_8 = { static const std::vector<ggml_op> ops_mul_mat_add = { GGML_OP_MUL_MAT, GGML_OP_ADD }; static const std::vector<ggml_metal_fusion> ggml_metal_fusions = { - { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_SCALE, ops_norm_scale, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_SCALE, ops_rms_norm_scale, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_SNAKE, ops_snake, {}, false, ggml_metal_fusion_check_snake }, - { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, {}, true, ggml_metal_fusion_check_gdn_cache }, - { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe, {1}, true, ggml_metal_fusion_check_topk_moe }, - { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, - { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm, {1}, true, ggml_metal_fusion_check_topk_moe }, - { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_2, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_3, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_4, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_5, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_6, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_7, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_8, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_SSM_CONV_SILU, ops_ssm_conv_silu, {}, false, ggml_metal_fusion_check_ssm_conv_silu }, - { GGML_METAL_FUSION_MUL_MAT_ADD, ops_mul_mat_add, {}, false, ggml_metal_fusion_check_mul_mat_add }, + { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_SCALE, ops_norm_scale, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_SCALE, ops_rms_norm_scale, {}, false, ggml_metal_fusion_check_norm }, + + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, {}, false, ggml_metal_fusion_check_add_chain }, + + { GGML_METAL_FUSION_SNAKE, ops_snake, {}, false, ggml_metal_fusion_check_snake }, + + { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, {}, true, ggml_metal_fusion_check_gdn_cache }, + + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_bias, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_scale_bias, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_scale, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_norm, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_norm_scale, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_norm_bias, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_norm_scale_bias, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_scale, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_norm, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_norm_scale, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_norm_bias, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_norm_scale_bias, {2}, true, ggml_metal_fusion_check_topk_moe }, + + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_2, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_3, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_4, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_5, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_6, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_7, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_8, {}, true, ggml_metal_fusion_check_moe_reduce }, + + { GGML_METAL_FUSION_SSM_CONV_SILU, ops_ssm_conv_silu, {}, false, ggml_metal_fusion_check_ssm_conv_silu }, + + { GGML_METAL_FUSION_MUL_MAT_ADD, ops_mul_mat_add, {}, false, ggml_metal_fusion_check_mul_mat_add }, }; // ---- alloc deps ----------------------------------------------------------- diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index e7c895631..7fd9d97cf 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -14,7 +14,8 @@ #define N_MM_SIMD_GROUP_X 2 #define N_MM_SIMD_GROUP_Y 2 -#define N_MM_NPART_AMAX 256 +#define N_ARGMAX_NPART_MAX 256 +#define N_ARGMAX_CHUNK 4096 // kernel parameters for mat-vec threadgroups // @@ -124,6 +125,8 @@ #define FC_FLASH_ATTN_EXT_TENSOR 2200 #define FC_LIGHTNING_INDEXER 2200 #define FC_MUL_MV_MMA 2300 +#define FC_TOP_K_MASK 2400 +#define FC_GROUP_NORM 2500 // op-specific constants #define OP_FLASH_ATTN_EXT_NQPSG 8 @@ -383,6 +386,19 @@ typedef struct { int32_t nblocks; } ggml_metal_kargs_flash_attn_ext_kv_f16; +// gather the K/V cache rows referenced by the kv_rows lists into contiguous F16 buffers +typedef struct { + int32_t ne11; // number of rows per list (n_kv_rows) + int32_t ne_12_2; // number of K/V heads + int32_t ne10; // DK + int32_t ne20; // DV, set to 0 to skip gathering V (V is a view of K) + uint64_t v_off; // byte offset of the gathered V region in dst + uint64_t nb11; // K source strides (ne13 = 1: the cache is shared by all streams) + uint64_t nb12; + uint64_t nb21; // V source strides (ne23 = 1) + uint64_t nb22; +} ggml_metal_kargs_flash_attn_ext_rows; + typedef struct { int32_t ne11; int32_t ne_12_2; // assume K and V are same shape @@ -560,7 +576,6 @@ typedef struct { } ggml_metal_kargs_mul_mv_ext; typedef struct { - int32_t ne02; int32_t ne10; int32_t ne11; // n_expert_used (bcast) uint64_t nb11; @@ -571,12 +586,11 @@ typedef struct { } ggml_metal_kargs_mul_mm_id_map0; typedef struct { - int32_t ne00; - int32_t ne01; - int32_t ne02; - uint64_t nb01; - uint64_t nb02; -} ggml_metal_kargs_mul_mm_id_amax; + int32_t ne0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb21; +} ggml_metal_kargs_mul_mm_id_zero; typedef struct { int32_t ne00; @@ -590,7 +604,6 @@ typedef struct { uint64_t nb12; uint64_t nb13; int32_t ne20; - int32_t ne21; int32_t ne0; int32_t ne1; int16_t r2; @@ -665,6 +678,10 @@ typedef struct { uint64_t nb00; uint64_t nb01; uint64_t nb02; + uint64_t nb03; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; int32_t ngrp; float eps; } ggml_metal_kargs_group_norm; @@ -1258,6 +1275,7 @@ typedef struct { uint64_t nb01; // logits row stride uint64_t nb1_ids; // ids row stride float clamp; + float bias; float scale; } ggml_metal_kargs_topk_moe; @@ -1297,6 +1315,38 @@ typedef struct { uint64_t nbm3; } ggml_metal_kargs_lightning_indexer; +typedef struct { + int32_t n_kv; + int32_t m1; + int32_t m2; + int32_t m3; + int32_t n_top; + int32_t n_pick; + int32_t n_blk; + int32_t n_extra; + int64_t n_elems; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + uint64_t nbm1; + uint64_t nbm2; + uint64_t nbm3; + uint64_t nbk0; + uint64_t nbk1; + uint64_t nbk2; + uint64_t nbk3; + uint64_t nbs0; + uint64_t nbs1; + uint64_t nbs2; + uint64_t nbs3; + uint64_t nbb0; + uint64_t nbb1; + uint64_t nbe0; + uint64_t nbe1; + uint64_t nbe2; + uint64_t nbe3; +} ggml_metal_kargs_top_k_mask; + typedef struct { int32_t n_tokens; int32_t n_iter; @@ -1381,8 +1431,10 @@ typedef struct { } ggml_metal_kargs_pool_1d; typedef struct { - int64_t ne00; + int64_t ne00; uint64_t nb01; + int64_t npart; + int64_t ne01; } ggml_metal_kargs_argmax; typedef struct { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index dfe46bac6..5c93bc4f0 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -345,6 +345,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_lightning_indexer(ctx, idx); } break; + case GGML_OP_TOP_K_MASK: + { + n_fuse = ggml_metal_op_top_k_mask(ctx, idx); + } break; case GGML_OP_DSV4_HC_COMB: case GGML_OP_DSV4_HC_PRE: case GGML_OP_DSV4_HC_POST: @@ -817,6 +821,16 @@ int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); + // the sigmoid and sqrt(softplus) TOPK_MOE fusions anchor at the gating unary node + if (op->op == GGML_OP_UNARY && ctx->use_fusion() && + (ggml_get_unary_op(op) == GGML_UNARY_OP_SOFTPLUS || ggml_get_unary_op(op) == GGML_UNARY_OP_SIGMOID)) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_TOPK_MOE) { + return ggml_metal_op_topk_moe(ctx, idx); + } + } + ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; @@ -895,7 +909,9 @@ int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) { if (pipeline.cnt) { const int n = pipeline.c4 ? ggml_nelements(op)/4 : ggml_nelements(op); - ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, 1, 1, 1); + const int nth = MIN(256, n); + + ggml_metal_encoder_dispatch_threadgroups(enc, (n + nth - 1)/nth, 1, 1, nth, 1, 1); } else { const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); const int nth = MIN(args.ne00, nth_max); @@ -948,7 +964,9 @@ int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) { const int64_t nrows = ggml_nrows(op->src[0]); - const int32_t nth = std::max(1, std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2)); + // split each row over several threadgroups: with a single row (decode) one threadgroup would run on one GPU core + const int32_t nth = std::max(1, std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), std::min(256, ne0))); + const int32_t nk = (ne0 + nth - 1)/nth; ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); @@ -960,7 +978,7 @@ int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) { } ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); - ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, nk, nrows, 1, nth, 1, 1); return 1; } @@ -1424,6 +1442,124 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) { return 1; } +int ggml_metal_op_top_k_mask(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + const ggml_tensor * mask = op->src[0]; + const ggml_tensor * top_k = op->src[1]; + const ggml_tensor * score = op->src[2]; + const ggml_tensor * blocks = op->src[3]; + const ggml_tensor * extra = op->src[4]; + + GGML_ASSERT(op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32); + GGML_ASSERT(mask->type == op->type); + GGML_ASSERT(top_k->type == GGML_TYPE_I32); + GGML_ASSERT(!score || score->type == GGML_TYPE_F32); + GGML_ASSERT(!blocks || blocks->type == GGML_TYPE_I32); + GGML_ASSERT(!extra || extra->type == GGML_TYPE_I32); + + const int32_t n_kv = mask->ne[0]; + const int32_t n_top = (int32_t) top_k->ne[0]; + const int32_t n_pick = blocks ? blocks->ne[1] : score ? score->ne[0] : n_kv; + const int32_t n_blk = blocks ? blocks->ne[0] : 0; + const int32_t n_extra = extra ? extra->ne[0] : 0; + + const ggml_metal_buffer_id bid = ggml_metal_get_buffer_id(op); + + // the vector fill stores 4 elements at a time + const int32_t ts = ggml_type_size(op->type); + const int64_t n_elems = ggml_nelements(op); + + const bool has_score = score != NULL; + const bool has_blocks = blocks != NULL; + const bool has_extra = extra != NULL; + + const bool vec = n_elems % 4 == 0 && bid.offs % (4*ts) == 0; + + ggml_metal_kargs_top_k_mask args = { + /*.n_kv =*/ n_kv, + /*.m1 =*/ (int32_t) mask->ne[1], + /*.m2 =*/ (int32_t) mask->ne[2], + /*.m3 =*/ (int32_t) mask->ne[3], + /*.n_top =*/ n_top, + /*.n_pick =*/ n_pick, + /*.n_blk =*/ n_blk, + /*.n_extra =*/ n_extra, + /*.n_elems =*/ n_elems, + /*.nb1 =*/ op->nb[1], + /*.nb2 =*/ op->nb[2], + /*.nb3 =*/ op->nb[3], + /*.nbm1 =*/ mask->nb[1], + /*.nbm2 =*/ mask->nb[2], + /*.nbm3 =*/ mask->nb[3], + /*.nbk0 =*/ top_k->nb[0], + /*.nbk1 =*/ top_k->nb[1], + /*.nbk2 =*/ top_k->nb[2], + /*.nbk3 =*/ top_k->nb[3], + /*.nbs0 =*/ score ? score->nb[0] : 0, + /*.nbs1 =*/ score ? score->nb[1] : 0, + /*.nbs2 =*/ score ? score->nb[2] : 0, + /*.nbs3 =*/ score ? score->nb[3] : 0, + /*.nbb0 =*/ blocks ? blocks->nb[0] : 0, + /*.nbb1 =*/ blocks ? blocks->nb[1] : 0, + /*.nbe0 =*/ extra ? extra->nb[0] : 0, + /*.nbe1 =*/ extra ? extra->nb[1] : 0, + /*.nbe2 =*/ extra ? extra->nb[2] : 0, + /*.nbe3 =*/ extra ? extra->nb[3] : 0, + }; + + // the fill covers the whole dst, the scatter overwrites the kept cells with the mask values + { + auto pipeline_fill = ggml_metal_library_get_pipeline_top_k_mask(lib, op, true, false, false, false, vec); + + const int64_t n_fill = n_elems/(vec ? 4 : 1); + const int32_t nth_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline_fill); + + int nth = 32; // SIMD width + + while (nth < nth_max && nth < n_fill) { + nth *= 2; + } + + ggml_metal_encoder_set_pipeline(enc, pipeline_fill); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, bid, 1); + + ggml_metal_encoder_dispatch_threadgroups(enc, (n_fill + nth - 1)/nth, 1, 1, nth, 1, 1); + + ggml_metal_op_concurrency_reset(ctx); + } + + { + auto pipeline = ggml_metal_library_get_pipeline_top_k_mask(lib, op, false, has_score, has_blocks, has_extra, false); + + const int32_t n_max = n_top + n_extra; + const int32_t nth_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline); + + int nth = 32; // SIMD width + + while (nth < nth_max && nth < n_max) { + nth *= 2; + } + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(mask), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(top_k), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(score ? score : mask), 3); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(blocks ? blocks : mask), 4); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(extra ? extra : mask), 5); + ggml_metal_encoder_set_buffer(enc, bid, 6); + + ggml_metal_encoder_dispatch_threadgroups(enc, op->ne[1], op->ne[2], op->ne[3], nth, 1, 1); + } + + return 1; +} + int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2319,13 +2455,18 @@ int ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_pool_1d(lib, op, op_pool); - const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np); - const int ntg = (np + nth - 1) / nth; + const int nth = std::min(256, std::max(1, k0)); + const int ntg = (int) np; + + const size_t smem = op_pool == GGML_OP_POOL_AVG ? + 2*32*sizeof(float) : + 32*sizeof(float); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args_pool_1d, sizeof(args_pool_1d), 0); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); @@ -2476,7 +2617,8 @@ static int ggml_metal_op_mul_mat_mma(ggml_metal_op_t ctx, int idx) { if (fuse_add) { dst = ctx->node(idx + n_fuse - 1); - res = dst->src[0]->op == GGML_OP_MUL_MAT ? dst->src[1] : dst->src[0]; + // the residual is the other operand of the ADD, by identity: it can itself be a MUL_MAT output + res = dst->src[0] == op ? dst->src[1] : dst->src[0]; } auto pipeline = ggml_metal_library_get_pipeline_mul_mv_mma_auto(lib, op, fuse_add); @@ -2771,30 +2913,39 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { return 1; } -size_t ggml_metal_op_mul_mat_id_extra_tpe(const ggml_tensor * op) { - assert(op->op == GGML_OP_MUL_MAT_ID); - +// upper bound on the number of entry tiles emitted by kernel_mul_mm_id_map0: the full tiles +// over all entries plus at most one partial tile per expert +static int64_t ggml_metal_op_mul_mat_id_max_tiles(const ggml_tensor * op) { const int64_t ne02 = op->src[0]->ne[2]; // n_expert + const int64_t ne20 = op->src[2]->ne[0]; // n_expert_used + const int64_t ne21 = op->src[2]->ne[1]; // n_token + + const int64_t n_rows = ne20*ne21; + const int64_t tiles = (n_rows + 31)/32 + (ne02 < n_rows ? ne02 : n_rows) - 1; - return ggml_type_size(GGML_TYPE_I32)*ne02; + return n_rows < tiles ? n_rows : tiles; } -size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) { +size_t ggml_metal_op_mul_mat_id_extra_tpe(const ggml_tensor * op) { assert(op->op == GGML_OP_MUL_MAT_ID); - const int64_t ne02 = op->src[0]->ne[2]; // n_expert - const int64_t ne21 = op->src[2]->ne[1]; // n_token + // row map of kernel_mul_mm_id: total tile count + one + // (expert, first entry, valid rows) descriptor per tile + const int64_t max_tiles = ggml_metal_op_mul_mat_id_max_tiles(op); - return ggml_type_size(GGML_TYPE_I32)*ne02*ne21; + return ggml_type_size(GGML_TYPE_I32)*(1 + 3*max_tiles); } -size_t ggml_metal_op_mul_mat_id_extra_amax(const ggml_tensor * op) { +size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) { assert(op->op == GGML_OP_MUL_MAT_ID); - GGML_UNUSED(op); + const int64_t ne02 = op->src[0]->ne[2]; // n_expert + const int64_t ne20 = op->src[2]->ne[0]; // n_expert_used + const int64_t ne21 = op->src[2]->ne[1]; // n_token - // 2 scaling factors (8 bytes) + N_MM_NPART_AMAX per-threadgroup scales for stage-1 - return 8 + N_MM_NPART_AMAX*sizeof(float); + // one entry per (expert used, token) pair of ids plus one padding slot per expert + // (see kernel_mul_mm_id_map0) + return ggml_type_size(GGML_TYPE_I32)*(ne20*ne21 + ne02); } int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { @@ -2848,42 +2999,8 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_metal_buffer_id bid_ids = bid_tpe; bid_ids.offs += ggml_metal_op_mul_mat_id_extra_tpe(op); - ggml_metal_buffer_id bid_amax = bid_ids; - bid_amax.offs += ggml_metal_op_mul_mat_id_extra_ids(op); - - // src1 prec [TAG_GGML_PREC] - const bool use_amax = ggml_get_op_params_i32(op, 3) == GGML_PREC_F32; - - // src1 rescale factors, computed before the matmul - // ref: https://github.com/ggml-org/llama.cpp/pull/26223 - if (use_amax) { - ggml_metal_kargs_mul_mm_id_amax args = { - /*.ne00 =*/ ne10, - /*.ne01 =*/ ne11, - /*.ne02 =*/ ne12, - /*.nb01 =*/ nb11, - /*.nb02 =*/ nb12, - }; - - auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax_part(lib); - - const size_t smem = pipeline.smem; - - GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); - - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_amax, 2); - - ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - - ggml_metal_encoder_dispatch_threadgroups(enc, N_MM_NPART_AMAX, 1, 1, 256, 1, 1); - } - { ggml_metal_kargs_mul_mm_id_map0 args = { - ne02, ne10, ne11, // n_expert_used (bcast) nb11, @@ -2912,20 +3029,28 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, ne02, 1, 1); } - ggml_metal_op_concurrency_reset(ctx); + // zero the rows that no expert owns - disjoint from what kernel_mul_mm_id writes + { + ggml_metal_kargs_mul_mm_id_zero args = { + /*.ne0 =*/ ne0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb21 =*/ nb21, + }; - if (use_amax) { - auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax(lib); + auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_zero(lib); ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_buffer (enc, bid_amax, 0); - - ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, 32, 1, 1); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_src2, 1); + ggml_metal_encoder_set_buffer (enc, bid_dst, 2); - // the next kernel has to wait for the amax data - ggml_metal_op_concurrency_reset(ctx); + ggml_metal_encoder_dispatch_threadgroups(enc, ne20, ne21, 1, 32, 1, 1); } + // this barrier is always needed because the next kernel has to wait for the id maps to be computed + ggml_metal_op_concurrency_reset(ctx); + { auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id(lib, op); @@ -2941,7 +3066,6 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { /*.nb12 =*/ nb12, /*.nb13 =*/ nb13, /*.ne20 =*/ ne20, // n_expert_used - /*.ne21 =*/ ne21, // n_tokens /*.ne0 =*/ ne0, /*.ne1 =*/ ne1, /*.r2 =*/ r2, @@ -2955,13 +3079,16 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, bid_tpe, 3); ggml_metal_encoder_set_buffer (enc, bid_ids, 4); ggml_metal_encoder_set_buffer (enc, bid_dst, 5); - ggml_metal_encoder_set_buffer (enc, bid_amax, 6); const size_t smem = pipeline.smem; + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - ggml_metal_encoder_dispatch_threadgroups(enc, (ne21 + 31)/32, (ne01 + 63)/64, ne02, 128, 1, 1); + const int max_tiles = (int) ggml_metal_op_mul_mat_id_max_tiles(op); + + ggml_metal_encoder_dispatch_threadgroups(enc, max_tiles, (ne01 + 63)/64, 1, 128, 1, 1); } } else { auto pipeline = ggml_metal_library_get_pipeline_mul_mv_id(lib, op); @@ -3088,7 +3215,9 @@ static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) { // depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial // ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767 // TODO: tune per device - if (op->src[0]->ne[1] < 32) { + // large heads need the upfront dequant to fit the non-vec threadgroup memory + if (op->src[0]->ne[1] < 32 && + (op->src[0]->ne[0] < 512 || ggml_metal_op_flash_attn_ext_use_vec(op))) { return false; } @@ -3138,11 +3267,14 @@ static bool ggml_metal_op_flash_attn_ext_use_tensor(const ggml_tensor * op, bool return false; } - if (op->src[1]->type != GGML_TYPE_F16 && !ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + if (op->src[1]->type != GGML_TYPE_F16 && op->src[1]->type != GGML_TYPE_BF16 && !ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { return false; } - return op->src[1]->ne[1] % OP_FLASH_ATTN_EXT_TENSOR_NCPSG == 0; + // with kv_rows, the attention runs over the compacted row lists (padded to 256), not over the cache + const int64_t ne11_attn = op->src[5] ? op->src[5]->ne[0] : op->src[1]->ne[1]; + + return ne11_attn % OP_FLASH_ATTN_EXT_TENSOR_NCPSG == 0; } // returns the n_kv_max hint if the sparse path is available for this op, or 0 otherwise @@ -3151,6 +3283,11 @@ static bool ggml_metal_op_flash_attn_ext_use_tensor(const ggml_tensor * op, bool static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); + // K/V row lists are handled by the dense kernels, not by the sparse path + if (op->src[5] != nullptr) { + return 0; + } + int32_t n_kv_max = 0; memcpy(&n_kv_max, ((const int32_t *) op->op_params) + 4, sizeof(n_kv_max)); @@ -3177,6 +3314,7 @@ static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) (dk == 96 && dv == 96) || (dk == 96 && dv == 64) || (dk == 128 && dv == 128) || + (dk == 128 && dv == 96) || (dk == 192 && dv == 128) || (dk == 192 && dv == 192) || (dk == 256 && dv == 256) || @@ -3393,6 +3531,32 @@ size_t ggml_metal_op_flash_attn_ext_extra_idx(const ggml_tensor * op) { return GGML_PAD(sizeof(int32_t)*(size_t) n_kv_max_padded*ne31*ne32*ne33, 16); } +// size of the contiguous F16 buffers holding the K/V rows referenced by the kv_rows lists +// the attention over these buffers is a standard dense op (no per-tile gather in the main kernel) +size_t ggml_metal_op_flash_attn_ext_extra_rows(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + const ggml_tensor * kv_rows = op->src[5]; + + if (!kv_rows) { + return 0; + } + + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); + + const int64_t n_rows = kv_rows->ne[0]; + const int64_t n_slices = kv_rows->ne[1]; + + // note: when V is a view of K, the gathered V is read from the gathered K, so only K is gathered + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + + const size_t k_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*n_rows*ne12*n_slices, 16); + const size_t v_size = v_is_view_of_k ? 0 : GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne20*n_rows*ne22*n_slices, 16); + + return k_size + v_size; +} + int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -3425,6 +3589,22 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { GGML_ASSERT(!op->src[3] || op->src[3]->ne[1] >= op->src[0]->ne[1] && "the Flash-Attention Metal kernel requires the mask to be at least n_queries big"); + // K/V row lists (ggml_flash_attn_ext_rows): src[5] holds one list of cache rows per q slice + // the attention loops then run over the mask columns and read K/V rows through the lists + const ggml_tensor * kv_rows = op->src[5]; + const bool has_rows = kv_rows != nullptr; + + GGML_ASSERT(!has_rows || (kv_rows->type == GGML_TYPE_I32 && ggml_is_contiguous(kv_rows))); + GGML_ASSERT(!has_rows || (kv_rows->ne[1] == ne03 && kv_rows->ne[2] == 1 && kv_rows->ne[3] == 1)); + GGML_ASSERT(!has_rows || op->src[3]); + + // the loop bound over the mask columns / K/V row lists + const int64_t ne11_attn = has_rows ? kv_rows->ne[0] : ne11; + + // the row lists are padded by the KV cache to a multiple of 256, so the pad path never triggers + GGML_ASSERT(!has_rows || ne11_attn % OP_FLASH_ATTN_EXT_NCPSG == 0); + GGML_ASSERT(!has_rows || ne11_attn % OP_FLASH_ATTN_EXT_VEC_NCPSG == 0); + float scale; float max_bias; float logit_softcap; @@ -3439,6 +3619,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { const bool has_mask = op->src[3] != NULL; const bool has_sinks = op->src[4] != NULL; + + GGML_ASSERT(!has_rows || has_mask); const bool has_bias = max_bias != 0.0f; const bool has_scap = logit_softcap != 0.0f; @@ -3475,12 +3657,25 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { const bool use_sparse = n_kv_max_sparse > 0; const int n_kv_max_padded = use_sparse ? GGML_PAD(n_kv_max_sparse, OP_FLASH_ATTN_EXT_VEC_NCPSG) : 0; + const bool use_vec = ggml_metal_op_flash_attn_ext_use_vec(op); + + // with kv_rows, the non-vec path gathers the referenced rows into a contiguous F16 buffer and + // attends over it with the F16 kernel - quantized KV is first dequantized with the F16 pass + const bool rows_gather = has_rows && !use_vec; + // the vec kernels dequantize the KV inline; no need for the F16 dequant pass in the sparse path - const bool use_kv_f16 = !use_sparse && ggml_metal_op_flash_attn_ext_use_kv_f16(op); + const bool dequant_kv = !use_sparse && (rows_gather ? + ggml_is_quantized(op->src[1]->type) : ggml_metal_op_flash_attn_ext_use_kv_f16(op)); + + const bool use_kv_f16 = !use_sparse && (rows_gather ? true : dequant_kv); ggml_metal_buffer_id bid_idx = bid_kv_f16; bid_idx.offs += ggml_metal_op_flash_attn_ext_extra_kv_f16(op); + // contiguous K/V buffers holding the rows referenced by the kv_rows lists + ggml_metal_buffer_id bid_rows = bid_idx; + bid_rows.offs += ggml_metal_op_flash_attn_ext_extra_idx(op); + ggml_metal_buffer_id bid_k = bid_src1; ggml_metal_buffer_id bid_v = bid_src2; @@ -3493,7 +3688,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { uint64_t nb22_attn = nb22; uint64_t nb23_attn = nb23; - if (use_kv_f16) { + if (dequant_kv) { assert(ggml_metal_op_flash_attn_ext_extra_kv_f16(op) != 0); const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); @@ -3581,6 +3776,70 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { } } + // with kv_rows, gather the cache rows referenced by the lists into contiguous F16 buffers + // the attention then runs as a standard dense op over the gathered rows (tensor or half8x8 kernel) + if (rows_gather) { + assert(ggml_metal_op_flash_attn_ext_extra_rows(op) != 0); + + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + + GGML_ASSERT(ne20 % 4 == 0); + + const size_t k_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*ne11_attn*ne12*ne03, 16); + + ggml_metal_kargs_flash_attn_ext_rows args0 = { + /*.ne11 =*/ (int32_t) ne11_attn, + /*.ne_12_2 =*/ ne12, + /*.ne10 =*/ ne10, + /*.ne20 =*/ v_is_view_of_k ? 0 : ne20, // the gathered V is a view of the gathered K + /*.v_off =*/ k_size, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + }; + + auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_rows(lib, op); + const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 64); + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(kv_rows), 3); + ggml_metal_encoder_set_buffer (enc, bid_rows, 4); + + ggml_metal_encoder_dispatch_threadgroups(enc, ne11_attn, ne03, 1, nth, 1, 1); + + ggml_metal_op_concurrency_reset(ctx); + + // contiguous F16 layout of the gathered rows: [DK, n_rows, n_head_kv, n_streams] + bid_k = bid_rows; + + nb10_attn = sizeof(ggml_fp16_t); + nb11_attn = nb10_attn*ne10; + nb12_attn = nb11_attn*ne11_attn; + nb13_attn = nb12_attn*ne12; + + // if V is a view of K, the gathered V is read from the gathered K with K's strides + if (v_is_view_of_k) { + bid_v = bid_k; + + nb20_attn = nb10_attn; + nb21_attn = nb11_attn; + nb22_attn = nb12_attn; + nb23_attn = nb13_attn; + } else { + bid_v = bid_rows; + bid_v.offs += k_size; + + nb20_attn = sizeof(ggml_fp16_t); + nb21_attn = nb20_attn*ne20; + nb22_attn = nb21_attn*ne11_attn; + nb23_attn = nb22_attn*ne22; + } + } + if (!use_sparse && ggml_metal_op_flash_attn_ext_use_tensor(op, props_dev->has_tensor)) { // tensor API kernel const int nqptg = ne00 >= 512 ? OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE : OP_FLASH_ATTN_EXT_TENSOR_NQPSG; // queries per threadgroup @@ -3626,9 +3885,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, /*.nb03 =*/ nb03, - /*.ne11 =*/ ne11, + /*.ne11 =*/ (int32_t) ne11_attn, /*.ne_12_2 =*/ ne12, - /*.ne_12_3 =*/ ne13, + /*.ne_12_3 =*/ has_rows ? ne03 : ne13, // the gathered K/V holds one copy per stream /*.ns10 =*/ ns10, /*.nb11 =*/ nb11_attn, /*.nb12 =*/ nb12_attn, @@ -3654,7 +3913,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - // shared memory layout: queries (half), scores (float), probabilities (half), row scale (float), rescale flag (int) + // shared memory layout: queries (16-bit), scores (float), probabilities (16-bit), row scale (float), rescale flag (int) const size_t smem = GGML_PAD(nqptg*ne00*sizeof(ggml_fp16_t) + nqptg*ncpsg*(sizeof(float) + sizeof(ggml_fp16_t)) + nqptg*sizeof(float) + sizeof(int32_t), 16); auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_tensor(lib, op, has_mask, has_sinks, has_bias, has_scap); @@ -3686,7 +3945,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { bool need_sync = false; - const bool has_kvpad = ne11 % ncpsg != 0; + const bool has_kvpad = ne11_attn % ncpsg != 0; if (has_kvpad) { assert(ggml_metal_op_flash_attn_ext_extra_pad(op) != 0); @@ -3786,9 +4045,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, /*.nb03 =*/ nb03, - /*.ne11 =*/ ne11, + /*.ne11 =*/ (int32_t) ne11_attn, /*.ne_12_2 =*/ ne12, - /*.ne_12_3 =*/ ne13, + /*.ne_12_3 =*/ has_rows ? ne03 : ne13, // the gathered K/V holds one copy per stream /*.ns10 =*/ ns10, /*.nb11 =*/ nb11_attn, /*.nb12 =*/ nb12_attn, @@ -3827,6 +4086,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, bid_blk, 7); ggml_metal_encoder_set_buffer (enc, bid_dst, 8); + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nqptg - 1)/nqptg, ne02, ne03, 32, nsg, 1); @@ -3852,7 +4113,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { bool need_sync = false; - const bool has_kvpad = !use_sparse && ne11 % ncpsg != 0; + const bool has_kvpad = !use_sparse && ne11_attn % ncpsg != 0; if (use_sparse) { assert(ggml_metal_op_flash_attn_ext_extra_idx(op) != 0); @@ -3935,13 +4196,13 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { // queries: Q*NSG*PAD2(ne00, 128) // attn + mask: NSG*4*Q*C // results: 2*NSG*Q*PAD2(ne20, 128) - // sparse idx: NSG*C ints (only when use_sparse) + // sparse idx: NSG*C ints (when use_sparse, or when gathering K/V rows) const int64_t dk_pad = GGML_PAD(ne00, 128); const int64_t dv_pad = GGML_PAD(ne20, 128); auto fa_vec_smem = [&](int64_t nsg, int32_t nqptg) -> size_t { const size_t smem_half = (size_t) (dk_pad + 4*ncpsg + 2*dv_pad)*nqptg*nsg; - return GGML_PAD(smem_half*sizeof(ggml_fp16_t) + (use_sparse ? (size_t) nsg*ncpsg*sizeof(int) : 0), 16); + return GGML_PAD(smem_half*sizeof(ggml_fp16_t) + ((use_sparse || has_rows) ? (size_t) nsg*ncpsg*sizeof(int) : 0), 16); }; int64_t nsg = 1; @@ -3972,7 +4233,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { } else { nwg = 32; nsg = 1; - while (2*nwg*nsg*ncpsg < ne11 && nsg < 4) { + while (2*nwg*nsg*ncpsg < ne11_attn && nsg < 4) { nsg *= 2; } } @@ -3993,7 +4254,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, /*.nb03 =*/ nb03, - /*.ne11 =*/ use_sparse ? n_kv_max_padded : ne11, + /*.ne11 =*/ use_sparse ? n_kv_max_padded : (int32_t) ne11_attn, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, /*.ns10 =*/ ns10, @@ -4022,7 +4283,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.n_kv_max_padded =*/ n_kv_max_padded, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, has_rows, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20); GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); @@ -4033,7 +4294,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); - ggml_metal_encoder_set_buffer (enc, use_sparse ? bid_idx : bid_src0, 8); + ggml_metal_encoder_set_buffer (enc, use_sparse ? bid_idx : (has_rows ? ggml_metal_get_buffer_id(kv_rows) : bid_src0), 8); const size_t smem = fa_vec_smem(nsg, nqptg); @@ -4210,7 +4471,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, bid_dst, 3); if (pipeline.cnt) { - ggml_metal_encoder_dispatch_threadgroups(enc, args.ne0, ggml_nrows(op), 1, 1, 1, 1); + const int nth = MIN(256, args.ne0); + + ggml_metal_encoder_dispatch_threadgroups(enc, (args.ne0 + nth - 1)/nth, ggml_nrows(op), 1, nth, 1, 1); } else { const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); @@ -4347,6 +4610,10 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { /*.nb00 =*/ nb00, /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, /*.ngrp =*/ ngrp, /*.eps =*/ eps, }; @@ -4370,7 +4637,7 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - ggml_metal_encoder_dispatch_threadgroups(enc, ngrp, 1, 1, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, ngrp, 1, ne03, nth, 1, 1); return 1; } @@ -5395,6 +5662,25 @@ int ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx) { return 1; } +size_t ggml_metal_op_argmax_extra(const ggml_tensor * op) { + assert(op->op == GGML_OP_ARGMAX); + + const int64_t ne00 = op->src[0]->ne[0]; + const int64_t ne01 = op->src[0]->ne[1]; + + int npart = 1; + if (ne00 > N_ARGMAX_CHUNK) { + npart = (int) std::min<int64_t>(N_ARGMAX_NPART_MAX, (ne00 + N_ARGMAX_CHUNK - 1)/N_ARGMAX_CHUNK); + } + + if (npart == 1) { + return 0; + } + + // partial max + partial argmax for every row + return (sizeof(float) + sizeof(int32_t)) * npart * ne01; +} + int ggml_metal_op_argmax(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -5406,22 +5692,61 @@ int ggml_metal_op_argmax(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + const int64_t nrows = ggml_nrows(op->src[0]); + + int npart = 1; + if (ne00 > N_ARGMAX_CHUNK) { + npart = (int) std::min<int64_t>(N_ARGMAX_NPART_MAX, (ne00 + N_ARGMAX_CHUNK - 1)/N_ARGMAX_CHUNK); + } + ggml_metal_kargs_argmax args = { - /*.ne00 = */ ne00, - /*.nb01 = */ nb01, + /*.ne00 = */ ne00, + /*.nb01 = */ nb01, + /*.npart = */ npart, + /*.ne01 = */ ne01, }; - auto pipeline = ggml_metal_library_get_pipeline_argmax(lib, op); + const size_t smem = 32*(sizeof(float) + sizeof(int32_t)); - const int64_t nrows = ggml_nrows(op->src[0]); + if (npart > 1) { + auto pipeline_part = ggml_metal_library_get_pipeline_argmax_part(lib); + auto pipeline_final = ggml_metal_library_get_pipeline_argmax_final(lib); + + ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); + ggml_metal_buffer_id bid_tmp = bid_dst; + bid_tmp.offs += ggml_nbytes(op); + + ggml_metal_encoder_set_pipeline(enc, pipeline_part); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, bid_tmp, 2); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, npart, nrows, 1, 256, 1, 1); + + ggml_metal_op_concurrency_reset(ctx); + + int nth = 32; // SIMD width + while (nth < npart && nth < 256) { + nth *= 2; + } + + ggml_metal_encoder_set_pipeline(enc, pipeline_final); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_tmp, 1); + ggml_metal_encoder_set_buffer (enc, bid_dst, 2); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, nth, 1, 1); + + return 1; + } + + auto pipeline = ggml_metal_library_get_pipeline_argmax(lib, op); int nth = 32; // SIMD width while (nth < ne00 && nth*ne01*ne02*ne03 < 256) { nth *= 2; } - const size_t smem = pipeline.smem; - ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); @@ -5683,11 +6008,10 @@ static void ggml_metal_op_top_k_radix(ggml_metal_op_t ctx, int idx) { /*.top_k =*/ (int32_t) op->ne[0], }; - // shared memory: 256-entry histogram + bucket/above scalars + output counter + // shared memory: 256-entry histogram + bucket/above scalars const size_t smem_histo = GGML_PAD(256*sizeof(uint32_t), 16); const size_t smem_bucket = GGML_PAD( sizeof(uint32_t), 16); const size_t smem_above = GGML_PAD( sizeof(uint32_t), 16); - const size_t smem_out = GGML_PAD( sizeof(uint32_t), 16); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); @@ -5697,7 +6021,6 @@ static void ggml_metal_op_top_k_radix(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_histo, 0); ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_bucket, 1); ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_above, 2); - ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_out, 3); ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); } @@ -5712,29 +6035,48 @@ int ggml_metal_op_topk_moe(ggml_metal_op_t ctx, int idx) { return 1; } - ggml_tensor * softmax = ctx->node(idx); - ggml_tensor * logits = softmax->src[0]; - ggml_tensor * get_rows = ctx->node(idx + 2); - ggml_tensor * ids = get_rows->src[1]; - ggml_tensor * weights = ctx->node(idx + n_fuse - 1); + ggml_tensor * first = ctx->node(idx); + ggml_tensor * logits = first->src[0]; + + // the unary chains score with sigmoid or sqrt(softplus) and rank biased scores + const bool with_unary = first->op == GGML_OP_UNARY; + const int gating = !with_unary ? 0 : ggml_get_unary_op(first) == GGML_UNARY_OP_SOFTPLUS ? 1 : 2; + + ggml_tensor * get_rows = nullptr; + ggml_tensor * sum_rows = nullptr; + ggml_tensor * prob_bias = nullptr; + int sum_rows_pos = -1; + for (int j = 0; j < n_fuse; ++j) { + ggml_tensor * node = ctx->node(idx + j); + if (node->op == GGML_OP_GET_ROWS) { get_rows = node; } + if (node->op == GGML_OP_ADD) { prob_bias = node->src[1]; } + if (node->op == GGML_OP_SUM_ROWS) { sum_rows = node; sum_rows_pos = j; } + } + + ggml_tensor * ids = get_rows->src[1]; + ggml_tensor * weights = ctx->node(idx + n_fuse - 1); + + const bool with_norm = sum_rows != nullptr; + const bool with_scale = weights->op == GGML_OP_SCALE; const int64_t n_expert = logits->ne[0]; const int64_t n_tokens = logits->ne[1]; const int64_t n_expert_used = ids->ne[0]; - const bool with_norm = n_fuse >= 6; - const bool with_scale = n_fuse == 4 || n_fuse == 7; - float clamp = -INFINITY; + float bias = 0.0f; if (with_norm) { - ggml_tensor * clamp_node = ctx->node(idx + 4); - clamp = ggml_get_op_params_f32(clamp_node, 0); + ggml_tensor * denom_node = ctx->node(idx + sum_rows_pos + 1); + if (denom_node->op == GGML_OP_SCALE) { + bias = ggml_get_op_params_f32(denom_node, 1); + } else { + clamp = ggml_get_op_params_f32(denom_node, 0); + } } float scale = 1.0f; if (with_scale) { - ggml_tensor * scale_node = ctx->node(idx + n_fuse - 1); - scale = ggml_get_op_params_f32(scale_node, 0); + scale = ggml_get_op_params_f32(weights, 0); } ggml_metal_kargs_topk_moe args = { @@ -5742,23 +6084,27 @@ int ggml_metal_op_topk_moe(ggml_metal_op_t ctx, int idx) { /*.nb01 =*/ logits->nb[1], /*.nb1_ids =*/ ids->nb[1], /*.clamp =*/ clamp, + /*.bias =*/ bias, /*.scale =*/ scale, }; - auto pipeline = ggml_metal_library_get_pipeline_topk_moe(lib, (int32_t) n_expert, (int32_t) n_expert_used, with_norm); + auto pipeline = ggml_metal_library_get_pipeline_topk_moe(lib, (int32_t) n_expert, (int32_t) n_expert_used, + with_norm, gating, prob_bias != nullptr); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(logits), 1); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(weights), 2); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(ids), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(prob_bias ? prob_bias : logits), 4); ggml_metal_encoder_dispatch_threadgroups(enc, (uint32_t) n_tokens, 1, 1, 32, 1, 1); ctx->count_fusions(fusion); if (ggml_metal_fusion_info_debug(ctx->finfo) > 1) { - GGML_LOG_DEBUG("%s: fuse: SOFT_MAX + ARGSORT + GET_ROWS\n", __func__); + GGML_LOG_DEBUG("%s: fuse: %s + ARGSORT + GET_ROWS\n", __func__, + gating == 1 ? "SOFTPLUS + SQRT" : gating == 2 ? "SIGMOID" : "SOFT_MAX"); } return n_fuse; diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index 583d1156b..47889fa84 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -36,7 +36,6 @@ size_t ggml_metal_op_mul_mat_id_extra_tpe(const struct ggml_tensor * op); // id map [n_tokens, n_expert] size_t ggml_metal_op_mul_mat_id_extra_ids(const struct ggml_tensor * op); -size_t ggml_metal_op_mul_mat_id_extra_amax(const struct ggml_tensor * op); // return true if we should use the FA vector kernel for this op bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op); @@ -46,6 +45,9 @@ size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_idx(const struct ggml_tensor * op); +size_t ggml_metal_op_flash_attn_ext_extra_rows(const struct ggml_tensor * op); + +size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op); int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx); int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx); @@ -59,6 +61,7 @@ int ggml_metal_op_get_rows (ggml_metal_op_t ctx, int idx); int ggml_metal_op_set_rows (ggml_metal_op_t ctx, int idx); int ggml_metal_op_diag (ggml_metal_op_t ctx, int idx); int ggml_metal_op_lightning_indexer (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_top_k_mask (ggml_metal_op_t ctx, int idx); int ggml_metal_op_dsv4_hc (ggml_metal_op_t ctx, int idx); int ggml_metal_op_soft_max (ggml_metal_op_t ctx, int idx); int ggml_metal_op_ssm_conv (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp index 217ad43b3..0fe50277e 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -40,6 +40,9 @@ int fa_vec_baseline_ne(int dk, int dv) { if (dk == 128 && dv == 128) { return 1; } + if (dk == 128 && dv == 96) { + return 4; + } if (dk == 192 && dv == 192) { return 2; } diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 9e4d970e0..b54db6350 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -68,6 +68,22 @@ static void ggml_backend_metal_buffer_shared_get_tensor(ggml_backend_buffer_t bu ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size); } +static void ggml_backend_metal_buffer_shared_set_tensor_n(ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) { + ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; + + GGML_ASSERT(ggml_metal_buffer_is_shared(ctx)); + + ggml_metal_buffer_set_tensor_n(ctx, tensors, data, offsets, sizes, n_tensors); +} + +static void ggml_backend_metal_buffer_shared_get_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) { + ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; + + GGML_ASSERT(ggml_metal_buffer_is_shared(ctx)); + + ggml_metal_buffer_get_tensor_n(ctx, tensors, data, offsets, sizes, n_tensors); +} + static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; @@ -80,6 +96,20 @@ static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t bu return ggml_metal_buffer_cpy_tensor(ctx, src, dst); } +static bool ggml_backend_metal_buffer_shared_cpy_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) { + ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; + + GGML_ASSERT(ggml_metal_buffer_is_shared(ctx)); + + for (size_t i = 0; i < n_tensors; ++i) { + if (!ggml_backend_buffer_is_metal(srcs[i]->buffer)) { + return false; + } + } + + return ggml_metal_buffer_cpy_tensor_n(ctx, srcs, dsts, n_tensors); +} + static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; @@ -100,6 +130,9 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = { /* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor, /* .clear = */ ggml_backend_metal_buffer_shared_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ ggml_backend_metal_buffer_shared_set_tensor_n, + /* .get_tensor_n = */ ggml_backend_metal_buffer_shared_get_tensor_n, + /* .cpy_tensor_n = */ ggml_backend_metal_buffer_shared_cpy_tensor_n, }; // private buffer @@ -144,6 +177,22 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t b ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size); } +static void ggml_backend_metal_buffer_private_set_tensor_n(ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) { + ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; + + GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx)); + + ggml_metal_buffer_set_tensor_n(ctx, tensors, data, offsets, sizes, n_tensors); +} + +static void ggml_backend_metal_buffer_private_get_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) { + ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; + + GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx)); + + ggml_metal_buffer_get_tensor_n(ctx, tensors, data, offsets, sizes, n_tensors); +} + static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; @@ -156,6 +205,20 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b return ggml_metal_buffer_cpy_tensor(ctx, src, dst); } +static bool ggml_backend_metal_buffer_private_cpy_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) { + ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; + + GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx)); + + for (size_t i = 0; i < n_tensors; ++i) { + if (!ggml_backend_buffer_is_metal(srcs[i]->buffer)) { + return false; + } + } + + return ggml_metal_buffer_cpy_tensor_n(ctx, srcs, dsts, n_tensors); +} + static void ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context; @@ -176,6 +239,9 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = { /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor, /* .clear = */ ggml_backend_metal_buffer_private_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ ggml_backend_metal_buffer_private_set_tensor_n, + /* .get_tensor_n = */ ggml_backend_metal_buffer_private_get_tensor_n, + /* .cpy_tensor_n = */ ggml_backend_metal_buffer_private_cpy_tensor_n, }; static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) { @@ -226,7 +292,6 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ { res += ggml_metal_op_mul_mat_id_extra_tpe(tensor); res += ggml_metal_op_mul_mat_id_extra_ids(tensor); - res += ggml_metal_op_mul_mat_id_extra_amax(tensor); } break; case GGML_OP_FLASH_ATTN_EXT: { @@ -235,6 +300,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor); res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor); res += ggml_metal_op_flash_attn_ext_extra_idx(tensor); + res += ggml_metal_op_flash_attn_ext_extra_rows(tensor); } break; case GGML_OP_CUMSUM: case GGML_OP_ARGSORT: @@ -245,6 +311,10 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ { res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]); } break; + case GGML_OP_ARGMAX: + { + res += ggml_metal_op_argmax_extra(tensor); + } break; default: break; } diff --git a/ggml/src/ggml-metal/kernels/argsort.metal b/ggml/src/ggml-metal/kernels/argsort.metal index 5231b8395..6c6fcb534 100644 --- a/ggml/src/ggml-metal/kernels/argsort.metal +++ b/ggml/src/ggml-metal/kernels/argsort.metal @@ -3,6 +3,8 @@ constant bool FC_topk_moe_with_norm [[function_constant(FC_TOPK_MOE + 0)]]; constant int FC_topk_moe_n_expert [[function_constant(FC_TOPK_MOE + 1)]]; constant int FC_topk_moe_top_k [[function_constant(FC_TOPK_MOE + 2)]]; +constant int FC_topk_moe_gating [[function_constant(FC_TOPK_MOE + 3)]]; +constant bool FC_topk_moe_pbias [[function_constant(FC_TOPK_MOE + 4)]]; constant int FC_moe_reduce_n_expert_used [[function_constant(FC_MOE_REDUCE + 0)]]; @@ -254,7 +256,6 @@ kernel void kernel_top_k_f32_i32( threadgroup atomic_uint * histo [[threadgroup(0)]], threadgroup uint * sh_bucket [[threadgroup(1)]], threadgroup uint * sh_above [[threadgroup(2)]], - threadgroup atomic_uint * out_count [[threadgroup(3)]], uint3 tgpig[[threadgroup_position_in_grid]], ushort3 tpitg[[thread_position_in_threadgroup]], ushort3 ntg[[threads_per_threadgroup]]) { @@ -292,20 +293,32 @@ kernel void kernel_top_k_f32_i32( } threadgroup_barrier(mem_flags::mem_threadgroup); - // top-down scan for the bucket holding the k-th value - if (tid == 0) { - uint acc = 0; - uint b = 0; - for (int bb = 255; bb >= 0; --bb) { - const uint c = atomic_load_explicit(&histo[bb], memory_order_relaxed); - if (acc + c >= desired) { - b = uint(bb); - break; + // top-down scan for the bucket holding the k-th value: each lane of the first + // simdgroup sums 8 bins, the lane whose range reaches desired scans its own bins + if (tid < 32) { + const uint top = 255 - 8*tid; + + uint c[8]; + uint sum = 0; + for (uint j = 0; j < 8; ++j) { + c[j] = atomic_load_explicit(&histo[top - j], memory_order_relaxed); + sum += c[j]; + } + uint acc = simd_prefix_exclusive_sum(sum); + + const simd_vote::vote_t hit = simd_vote::vote_t(simd_ballot(acc + sum >= desired)); + if (tid == (uint) ctz(hit)) { + uint b = top; + for (uint j = 0; j < 8; ++j) { + if (acc + c[j] >= desired) { + b = top - j; + break; + } + acc += c[j]; } - acc += c; + *sh_bucket = b; + *sh_above = acc; } - *sh_bucket = b; - *sh_above = acc; } threadgroup_barrier(mem_flags::mem_threadgroup); @@ -316,39 +329,113 @@ kernel void kernel_top_k_f32_i32( threadgroup_barrier(mem_flags::mem_threadgroup); } - if (tid == 0) { - atomic_store_explicit(out_count, 0u, memory_order_relaxed); + // deterministic compaction: rank the kept elements by column order, so the + // result does not depend on the thread scheduling + const uint threshold = prefix; + + // each warp scans a contiguous chunk of the row, and the lanes of the warp + // interleave within the chunk with stride 32 to keep the accesses coalesced + const uint lane = tid & 31u; + const uint wg = tid >> 5u; + const uint nw = ntg_x >> 5u; + const uint wlo = wg*((ncols + nw - 1) / nw); + const uint whi = MIN(wlo + (ncols + nw - 1) / nw, ncols); + const uint wlen = wlo < whi ? whi - wlo : 0; + const uint nstep = (wlen + 31) >> 5; + + uint ca = 0; // elements above the threshold in this lane's substream + uint ct = 0; // elements tied with the threshold in this lane's substream + for (uint i = wlo + lane; i < whi; i += 32u) { + const uint key = ggml_top_k_f2ui(src0_row[i]); + ca += (key > threshold) ? 1u : 0u; + ct += (key == threshold) ? 1u : 0u; + } + + // per-warp totals into the now unused histogram, so that the first warp can + // compute the exclusive base of each warp with a scan across its lanes + const uint wca = simd_sum(ca); + const uint wct = simd_sum(ct); + if (lane == 0) { + atomic_store_explicit(&histo[wg], wca, memory_order_relaxed); + atomic_store_explicit(&histo[32 + wg], wct, memory_order_relaxed); } threadgroup_barrier(mem_flags::mem_threadgroup); - // emit everything above the threshold, then fill the rest from ties - const uint threshold = prefix; - - for (uint i = tid; i < ncols; i += ntg_x) { - if (ggml_top_k_f2ui(src0_row[i]) > threshold) { - const uint pos = atomic_fetch_add_explicit(out_count, 1u, memory_order_relaxed); - dst_row[pos] = (int32_t) i; + if (tid < 32) { + const bool valid = tid < nw; + const uint sa = valid ? atomic_load_explicit(&histo[tid], memory_order_relaxed) : 0u; + const uint st = valid ? atomic_load_explicit(&histo[32 + tid], memory_order_relaxed) : 0u; + const uint ea = simd_prefix_exclusive_sum(sa); + const uint et = simd_prefix_exclusive_sum(st); + if (valid) { + atomic_store_explicit(&histo[tid], ea, memory_order_relaxed); + atomic_store_explicit(&histo[32 + tid], et, memory_order_relaxed); + } + const uint ta = simd_sum(sa); + const uint tt = simd_sum(st); + if (tid == 0) { + *sh_bucket = ta; // total above the threshold + *sh_above = tt; // total tied with the threshold } } threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint i = tid; i < ncols; i += ntg_x) { - if (ggml_top_k_f2ui(src0_row[i]) == threshold) { - const uint pos = atomic_fetch_add_explicit(out_count, 1u, memory_order_relaxed); - if (pos < top_k) { + const uint above_total = *sh_bucket; + const uint tie_total = *sh_above; + const uint base_wa = atomic_load_explicit(&histo[wg], memory_order_relaxed); + const uint base_wt = above_total + atomic_load_explicit(&histo[32 + wg], memory_order_relaxed); + + // the lanes write their kept elements at lane-major ranks, which does not + // depend on the thread scheduling. with more tied candidates than remaining + // slots, the ties are instead ranked in column order with a per-step + // cross-lane scan, so that the lowest tied indices are the ones kept + const uint excl_a = simd_prefix_exclusive_sum(ca); + const uint excl_t = simd_prefix_exclusive_sum(ct); + + if (tie_total <= top_k - above_total) { + uint na = 0; + uint nt = 0; + for (uint i = wlo + lane; i < whi; i += 32u) { + const uint key = ggml_top_k_f2ui(src0_row[i]); + if (key > threshold) { + dst_row[base_wa + excl_a + na] = (int32_t) i; + ++na; + } else if (key == threshold) { + dst_row[base_wt + excl_t + nt] = (int32_t) i; + ++nt; + } + } + } else { + uint na = 0; + for (uint i = wlo + lane; i < whi; i += 32u) { + if (ggml_top_k_f2ui(src0_row[i]) > threshold) { + dst_row[base_wa + excl_a + na] = (int32_t) i; + ++na; + } + } + + uint tt = 0; // tied elements in the earlier steps of this warp + for (uint s = 0; s < nstep; ++s) { + const uint i = wlo + 32u*s + lane; + const bool ok = i < whi; + const bool t = ok && ggml_top_k_f2ui(src0_row[i]) == threshold; + const uint pos = base_wt + tt + simd_prefix_exclusive_sum((uint) t); + if (t && pos < top_k) { dst_row[pos] = (int32_t) i; } + tt += simd_sum((uint) t); } } } -// fused SOFT_MAX + top-k + GET_ROWS (+ optional norm/scale) for MoE routing. +// fused SOFT_MAX or sqrt(softplus) + top-k + GET_ROWS (+ optional norm/scale) for MoE routing. // One SIMDgroup handles one token row; n_expert is limited to 1024 by the host. kernel void kernel_topk_moe_f32( constant ggml_metal_kargs_topk_moe & args, device const char * src0, device float * weights, device int32_t * ids, + device const float * probs_bias, uint3 tgpig[[threadgroup_position_in_grid]], ushort tiisg[[thread_index_in_simdgroup]]) { const int row = (int) tgpig.x; @@ -377,50 +464,78 @@ kernel void kernel_topk_moe_f32( wt[i / 32] = isnan(v) ? -FLT_MAX : v; } - // softmax over the expert logits - float max_val = -INFINITY; - FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { - max_val = max(max_val, wt[i]); - } - max_val = simd_max(max_val); + if (FC_topk_moe_gating == 1) { + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + const float v = wt[i]; + wt[i] = sqrt(v > 20.0f ? v : log(1.0f + exp(v))); + } + } else if (FC_topk_moe_gating == 2) { + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] = 1.0f / (1.0f + exp(-wt[i])); + } + } else { + // softmax over the expert logits + float max_val = -INFINITY; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + max_val = max(max_val, wt[i]); + } + max_val = simd_max(max_val); - float sum_val = 0.0f; - FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { - wt[i] = exp(wt[i] - max_val); - sum_val += wt[i]; + float sum_val = 0.0f; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] = exp(wt[i] - max_val); + sum_val += wt[i]; + } + sum_val = simd_sum(sum_val); + + const float inv_sum = 1.0f / sum_val; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] *= inv_sum; + } } - sum_val = simd_sum(sum_val); - const float inv_sum = 1.0f / sum_val; - FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { - wt[i] *= inv_sum; + // the ranking keys add the selection bias, the output weights stay unbiased + float sel[32]; + if (FC_topk_moe_pbias) { + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + const int expert = lane + i * 32; + sel[i] = expert < n_expert ? wt[i] + probs_bias[expert] : -INFINITY; + } } float wt_sum = 0.0f; for (int k = 0; k < top_k; ++k) { - float best_val = -INFINITY; + float best_val = -INFINITY; + float best_key = -INFINITY; int best_expert = -1; FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { - const int expert = lane + i * 32; - if (expert < n_expert && (wt[i] > best_val || (wt[i] == best_val && expert < best_expert))) { + const int expert = lane + i * 32; + const float key = FC_topk_moe_pbias ? sel[i] : wt[i]; + if (expert < n_expert && (key > best_key || (key == best_key && expert < best_expert))) { best_val = wt[i]; + best_key = key; best_expert = expert; } } FOR_UNROLL (int mask = 16; mask > 0; mask >>= 1) { const float val = simd_shuffle_xor(best_val, mask); + const float key = simd_shuffle_xor(best_key, mask); const int expert = simd_shuffle_xor(best_expert, mask); - if (val > best_val || (val == best_val && expert < best_expert)) { + if (key > best_key || (key == best_key && expert < best_expert)) { best_val = val; + best_key = key; best_expert = expert; } } if ((best_expert & 31) == lane) { wt[best_expert / 32] = -INFINITY; + if (FC_topk_moe_pbias) { + sel[best_expert / 32] = -INFINITY; + } } if ((k & 31) == lane) { @@ -437,7 +552,7 @@ kernel void kernel_topk_moe_f32( if (FC_topk_moe_with_norm) { wt_sum = simd_sum(wt_sum); - wt_sum = max(wt_sum, args.clamp); + wt_sum = max(wt_sum, args.clamp) + args.bias; const float inv = 1.0f / wt_sum; FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { output_weights[i] *= inv; diff --git a/ggml/src/ggml-metal/kernels/binbcast.metal b/ggml/src/ggml-metal/kernels/binbcast.metal index 7c7ab9b5e..e125b7a03 100644 --- a/ggml/src/ggml-metal/kernels/binbcast.metal +++ b/ggml/src/ggml-metal/kernels/binbcast.metal @@ -22,8 +22,13 @@ kernel void kernel_bin_fuse_impl( if (FC_RB) { // row broadcast - const uint i0 = tgpig.y*args.ne00 + tgpig.x; - const uint i1 = FC_CB ? tgpig.x%args.ne10 : tgpig.x; + const uint i = tgpig.x*ntg.x + tpitg.x; + if (i >= (uint) args.ne0) { + return; + } + + const uint i0 = tgpig.y*args.ne00 + i; + const uint i1 = FC_CB ? i%args.ne10 : i; device const T0 * src0_row = (device const T0 *) (src0); device T * dst_row = (device T *) (dst); @@ -185,6 +190,14 @@ kernel void kernel_add_id( device float * dst_row = (device float *)((device char *)dst + i1*nb1 + i2*nb2); device const float * src0_row = (device const float *)((device char *)src0 + i1*args.nb01 + i2*args.nb02); + + if (i11 == -1) { + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + dst_row[i0] = src0_row[i0]; + } + return; + } + device const float * src1_row = (device const float *)((device char *)src1 + i11*args.nb11); for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { diff --git a/ggml/src/ggml-metal/kernels/fa_aux.metal b/ggml/src/ggml-metal/kernels/fa_aux.metal index ebee1becd..6fe47470f 100644 --- a/ggml/src/ggml-metal/kernels/fa_aux.metal +++ b/ggml/src/ggml-metal/kernels/fa_aux.metal @@ -48,6 +48,80 @@ template [[host_name("kernel_flash_attn_ext_kv_q5_0_f16")]] kernel kernel_flash_ template [[host_name("kernel_flash_attn_ext_kv_q5_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q5_1, 32, dequantize_q5_1>; template [[host_name("kernel_flash_attn_ext_kv_q8_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q8_0, 32, dequantize_q8_0>; +// gather the K/V cache rows referenced by the kv_rows lists into contiguous F16 buffers +// - one threadgroup per (row, stream); the attention is then run as a standard dense op over the +// gathered rows, which avoids the per-tile K/V gather in the main kernels +// - negative rows are padding: filled with zeros, their mask entries are -INF so they contribute nothing +kernel void kernel_flash_attn_ext_rows( + constant ggml_metal_kargs_flash_attn_ext_rows & args, + device const char * k_src, + device const char * v_src, + device const int * rows, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int64_t i1 = tgpig[0]; // row index + const int64_t i3 = tgpig[1]; // stream + + const int64_t row = rows[i1 + i3*args.ne11]; + + // gathered layout: [DK, n_rows, n_head_kv, n_streams] and [DV, n_rows, n_head_kv, n_streams] F16 + device half4 * k_dst = (device half4 *) (dst + i3*args.ne_12_2*args.ne11*args.ne10*sizeof(half)); + device half4 * v_dst = (device half4 *) (dst + args.v_off + i3*args.ne_12_2*args.ne11*args.ne20*sizeof(half)); + + // note: the K/V cache is shared by all streams (ne13 = ne23 = 1), the row list selects the rows + // each thread copies up to 4 half4 values (32 bytes) to keep the memory transactions large + + // K: all heads of this row + { + const int64_t per_head = args.ne10/4; + + for (int64_t i = 4*tiitg; i < args.ne_12_2*per_head; i += 4*ntg.x) { + const int64_t i2 = i/per_head; + const int64_t i0 = i%per_head; + const int n = (int) min((int64_t) 4, per_head - i0); + + device half4 * dst4 = k_dst + (i2*args.ne11 + i1)*per_head + i0; + + if (row >= 0) { + device const half4 * src4 = (device const half4 *) (k_src + row*args.nb11 + i2*args.nb12) + i0; + for (int j = 0; j < n; ++j) { + dst4[j] = src4[j]; + } + } else { + for (int j = 0; j < n; ++j) { + dst4[j] = half4(0.0h); + } + } + } + } + + // V: all heads of this row + { + const int64_t per_head = args.ne20/4; + + for (int64_t i = 4*tiitg; i < args.ne_12_2*per_head; i += 4*ntg.x) { + const int64_t i2 = i/per_head; + const int64_t i0 = i%per_head; + const int n = (int) min((int64_t) 4, per_head - i0); + + device half4 * dst4 = v_dst + (i2*args.ne11 + i1)*per_head + i0; + + if (row >= 0) { + device const half4 * src4 = (device const half4 *) (v_src + row*args.nb21 + i2*args.nb22) + i0; + for (int j = 0; j < n; ++j) { + dst4[j] = src4[j]; + } + } else { + for (int j = 0; j < n; ++j) { + dst4[j] = half4(0.0h); + } + } + } + } +} + constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]]; constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]]; diff --git a/ggml/src/ggml-metal/kernels/fa_f16.metal b/ggml/src/ggml-metal/kernels/fa_f16.metal index 2dfd3799d..6e30b09e3 100644 --- a/ggml/src/ggml-metal/kernels/fa_f16.metal +++ b/ggml/src/ggml-metal/kernels/fa_f16.metal @@ -44,6 +44,7 @@ template [[host_name("kernel_flash_attn_ext_f16_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_f16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 96, 64>; template [[host_name("kernel_flash_attn_ext_f16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 112, 112>; template [[host_name("kernel_flash_attn_ext_f16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 128, 128>; +template [[host_name("kernel_flash_attn_ext_f16_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 128, 96>; template [[host_name("kernel_flash_attn_ext_f16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 192, 192>; template [[host_name("kernel_flash_attn_ext_f16_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 192, 128>; template [[host_name("kernel_flash_attn_ext_f16_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 256, 256>; @@ -62,6 +63,7 @@ template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 96, 64>; template [[host_name("kernel_flash_attn_ext_bf16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 112, 112>; template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 128, 128>; +template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 128, 96>; template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 192, 192>; template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 192, 128>; template [[host_name("kernel_flash_attn_ext_bf16_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 256, 256>; @@ -73,291 +75,3 @@ template [[host_name("kernel_flash_attn_ext_bf16_dk576_dv512")]] kernel flash_at #undef FA_TYPES #undef FA_TYPES_BF #undef FA_TYPES_F32 - -#ifdef GGML_METAL_HAS_TENSOR - -constant bool FC_flash_attn_ext_tensor_has_mask [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 0)]]; -constant bool FC_flash_attn_ext_tensor_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 1)]]; -constant bool FC_flash_attn_ext_tensor_has_bias [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 2)]]; -constant bool FC_flash_attn_ext_tensor_has_scap [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 3)]]; - -// ref: https://arxiv.org/pdf/2307.08691.pdf -template< - short DK, // K head size - short DV, // V head size - short Q = OP_FLASH_ATTN_EXT_TENSOR_NQPSG, // queries per threadgroup - short C = OP_FLASH_ATTN_EXT_TENSOR_NCPSG, // cache items per threadgroup - short NSG = OP_FLASH_ATTN_EXT_TENSOR_NSG> // number of simd groups -kernel void kernel_flash_attn_ext_tensor( - constant ggml_metal_kargs_flash_attn_ext & args, - device const char * q, - device const char * k, - device const char * v, - device const char * mask, - device const char * sinks, - device const char * blk, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig [[threadgroup_position_in_grid]], - ushort tiisg [[thread_index_in_simdgroup]], - ushort sgitg [[simdgroup_index_in_threadgroup]]) { - constexpr short NW = N_SIMDWIDTH; - constexpr short NT = NW*NSG; - constexpr short NQ = Q/NSG; - constexpr short NC = C/NW; // columns per thread - - static_assert(DK % 4 == 0, "DK must be divisible by 4"); - static_assert(Q % NSG == 0, "Q must be divisible by NSG"); - static_assert(C % NW == 0, "C must be divisible by NW"); - - const int iq3 = tgpig[2]; - const int iq2 = tgpig[1]; - const int iq1 = tgpig[0]*Q; - - const short tiitg = sgitg*NW + tiisg; - - threadgroup half * sq = (threadgroup half *) shmem; // [Q, DK] queries - threadgroup float * ss = (threadgroup float *) (sq + Q*DK); // [Q, C] scores - threadgroup half * sp = (threadgroup half *) (ss + Q*C); // [Q, C] probabilities - threadgroup float * sr = (threadgroup float *) (sp + Q*C); // [Q] per-row scale of O - threadgroup int * sf = (threadgroup int *) (sr + Q); // [1] last iteration (ic0 + 1) that rescaled O - - q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03; - - { - const int ikv2 = iq2/(args.ne02/args.ne_12_2); - const int ikv3 = iq3/(args.ne03/args.ne_12_3); - - k += ikv2*args.nb12 + ikv3*args.nb13; - v += ikv2*args.nb22 + ikv3*args.nb23; - } - - // with softcap the scale is small (scale/softcap), so it is applied to the scores to keep the precision of Q - const float qscale = FC_flash_attn_ext_tensor_has_scap ? 1.0f : args.scale; - - // load the queries, with the scale folded in - for (int i = tiitg; i < Q*DK/4; i += NT) { - const int j = i/(DK/4); - - float4 q4 = 0.0f; - if (iq1 + j < args.ne01) { - q4 = ((device const float4 *) (q + j*args.nb01))[i%(DK/4)]; - } - - ((threadgroup half4 *) sq)[i] = (half4) (q4*qscale); - } - - device const half * pm[NQ]; - - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - pm[jj] = (device const half *) (mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); - } - - { - const int nblk1 = (args.ne01 + Q - 1)/Q; - const int nblk0 = (args.ne11 + C - 1)/C; - - blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0; - } - - float M[NQ]; - float S[NQ]; - - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - M[jj] = -FLT_MAX/2; - S[jj] = 0.0f; - } - - float slope = 1.0f; - - // ALiBi - if (FC_flash_attn_ext_tensor_has_bias) { - const short h = iq2; - - const float base = h < args.n_head_log2 ? args.m0 : args.m1; - const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; - - slope = pow(base, exph); - } - - const int sk = args.ns10; - const int sv = args.ns20; - - auto tq = tensor(sq, dextents<int32_t, 2>(DK, Q)); - auto ts = tensor(ss, dextents<int32_t, 2>(C, Q)); - auto tp = tensor(sp, dextents<int32_t, 2>(C, Q)); - - mpp::tensor_ops::matmul2d< - mpp::tensor_ops::matmul2d_descriptor(Q, C, DK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply), - execution_simdgroups<NSG>> mm_qk; - - mpp::tensor_ops::matmul2d< - mpp::tensor_ops::matmul2d_descriptor(Q, DV, C, false, false, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), - execution_simdgroups<NSG>> mm_pv; - - auto tv0 = tensor((device half *) v, dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv})); - - // the O matrix from the paper - auto co = mm_pv.template get_destination_cooperative_tensor<decltype(tp), decltype(tv0), float>(); - - FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { - if (co.is_valid_element(i)) { - co[i] = 0.0f; - } - } - - if (tiitg == 0) { - sf[0] = 0; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - // the host guarantees ne11 % C == 0 - for (int ic0 = 0, ic = 0; ic < args.ne11; ++ic0, ic += C) { - char blk_cur = 1; - - if (FC_flash_attn_ext_tensor_has_mask) { - blk_cur = blk[ic0]; - - if (blk_cur == 0) { - continue; - } - } - - // Q*K^T - { - auto tk = tensor((device half *) (k + (uint64_t) ic*args.nb11), dextents<int32_t, 2>(DK, C), array<int, 2>({1, sk})); - - mm_qk.run(tq, tk, ts); - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - // online softmax - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - float s[NC]; - - FOR_UNROLL (short ii = 0; ii < NC; ++ii) { - s[ii] = ss[j*C + ii*NW + tiisg]; - } - - if (FC_flash_attn_ext_tensor_has_scap) { - FOR_UNROLL (short ii = 0; ii < NC; ++ii) { - s[ii] = args.logit_softcap*precise::tanh(s[ii]*args.scale); - } - } - - if (FC_flash_attn_ext_tensor_has_mask && blk_cur != 2 && iq1 + j < args.ne31) { - FOR_UNROLL (short ii = 0; ii < NC; ++ii) { - s[ii] += slope*(float) pm[jj][ic + ii*NW + tiisg]; - } - } - - float m = M[jj]; - - FOR_UNROLL (short ii = 0; ii < NC; ++ii) { - m = max(m, s[ii]); - } - - m = simd_max(m); - - // lazy rescaling: move the running max only when it grows by more than 8 (e^8 fits in half) - float ms = 1.0f; - - if (m > M[jj] + 8.0f) { - ms = exp(M[jj] - m); - M[jj] = m; - - if (tiisg == 0) { - sf[0] = ic0 + 1; - } - } - - float sum = 0.0f; - - FOR_UNROLL (short ii = 0; ii < NC; ++ii) { - // the sum uses the same rounded values as P*V - const half p = (half) exp(s[ii] - M[jj]); - - sp[j*C + ii*NW + tiisg] = p; - - sum += (float) p; - } - - S[jj] = S[jj]*ms + simd_sum(sum); - - if (tiisg == 0) { - sr[j] = ms; - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - // O = diag(ms)*O + P*V - if (sf[0] == ic0 + 1) { - FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { - if (co.is_valid_element(i)) { - co[i] *= sr[co.get_multidimensional_index(i)[1]]; - } - } - } - - { - auto tv = tensor((device half *) (v + (uint64_t) ic*args.nb21), dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv})); - - mm_pv.run(tp, tv, co); - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - // the sink only adds to the denominator - its rescale of O is folded into the final scale - float ms = 1.0f; - - if (FC_flash_attn_ext_tensor_has_sinks) { - const float s = ((device const float *) sinks)[iq2]; - const float m = max(M[jj], s); - - ms = exp(M[jj] - m); - - S[jj] = S[jj]*ms + exp(s - m); - } - - if (tiisg == 0) { - sr[j] = S[jj] == 0.0f ? 0.0f : ms/S[jj]; - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { - if (co.is_valid_element(i)) { - co[i] *= sr[co.get_multidimensional_index(i)[1]]; - } - } - - // store to global memory - rows past ne01 are clipped by the tensor extents - device float * pdst = (device float *) dst + ((uint64_t) iq3*args.ne2*args.ne1 + iq2 + (uint64_t) iq1*args.ne1)*DV; - - auto td = tensor(pdst, dextents<int32_t, 2>(DV, args.ne01 - iq1), array<int, 2>({1, args.ne1*DV})); - - co.store(td); -} - -typedef decltype(kernel_flash_attn_ext_tensor<64, 64>) flash_attn_ext_tensor_t; - -template [[host_name("kernel_flash_attn_ext_tensor_f16_dk64_dv64" )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<64, 64>; -template [[host_name("kernel_flash_attn_ext_tensor_f16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<128, 128>; -template [[host_name("kernel_flash_attn_ext_tensor_f16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<192, 128>; -template [[host_name("kernel_flash_attn_ext_tensor_f16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<256, 256>; -template [[host_name("kernel_flash_attn_ext_tensor_f16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>; -template [[host_name("kernel_flash_attn_ext_tensor_f16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>; - -#endif // GGML_METAL_HAS_TENSOR diff --git a/ggml/src/ggml-metal/kernels/fa_f32.metal b/ggml/src/ggml-metal/kernels/fa_f32.metal index 8d38d7752..449079e8f 100644 --- a/ggml/src/ggml-metal/kernels/fa_f32.metal +++ b/ggml/src/ggml-metal/kernels/fa_f32.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_f32_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_f32_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 96, 64>; template [[host_name("kernel_flash_attn_ext_f32_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 112, 112>; template [[host_name("kernel_flash_attn_ext_f32_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 128, 128>; +template [[host_name("kernel_flash_attn_ext_f32_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 128, 96>; template [[host_name("kernel_flash_attn_ext_f32_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 192, 192>; template [[host_name("kernel_flash_attn_ext_f32_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 192, 128>; template [[host_name("kernel_flash_attn_ext_f32_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 256, 256>; diff --git a/ggml/src/ggml-metal/kernels/fa_q4_0.metal b/ggml/src/ggml-metal/kernels/fa_q4_0.metal index 49bb4c13c..6b05b5df6 100644 --- a/ggml/src/ggml-metal/kernels/fa_q4_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_q4_0.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 96, 64>; template [[host_name("kernel_flash_attn_ext_q4_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 112, 112>; template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 128, 128>; +template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 128, 96>; template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 192, 192>; template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 192, 128>; template [[host_name("kernel_flash_attn_ext_q4_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 256, 256>; diff --git a/ggml/src/ggml-metal/kernels/fa_q4_1.metal b/ggml/src/ggml-metal/kernels/fa_q4_1.metal index 79797ffce..87fac701e 100644 --- a/ggml/src/ggml-metal/kernels/fa_q4_1.metal +++ b/ggml/src/ggml-metal/kernels/fa_q4_1.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 96, 64>; template [[host_name("kernel_flash_attn_ext_q4_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 112, 112>; template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 128, 128>; +template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 128, 96>; template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 192, 192>; template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 192, 128>; template [[host_name("kernel_flash_attn_ext_q4_1_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 256, 256>; diff --git a/ggml/src/ggml-metal/kernels/fa_q5_0.metal b/ggml/src/ggml-metal/kernels/fa_q5_0.metal index 93de514b4..cd0e31f71 100644 --- a/ggml/src/ggml-metal/kernels/fa_q5_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_q5_0.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 96, 64>; template [[host_name("kernel_flash_attn_ext_q5_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 112, 112>; template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 128, 128>; +template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 128, 96>; template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 192, 192>; template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 192, 128>; template [[host_name("kernel_flash_attn_ext_q5_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 256, 256>; diff --git a/ggml/src/ggml-metal/kernels/fa_q5_1.metal b/ggml/src/ggml-metal/kernels/fa_q5_1.metal index 3918c571e..4d74951ac 100644 --- a/ggml/src/ggml-metal/kernels/fa_q5_1.metal +++ b/ggml/src/ggml-metal/kernels/fa_q5_1.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 96, 64>; template [[host_name("kernel_flash_attn_ext_q5_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 112, 112>; template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 128, 128>; +template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 128, 96>; template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 192, 192>; template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 192, 128>; template [[host_name("kernel_flash_attn_ext_q5_1_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 256, 256>; diff --git a/ggml/src/ggml-metal/kernels/fa_q8_0.metal b/ggml/src/ggml-metal/kernels/fa_q8_0.metal index 146bbb31c..bb29a262c 100644 --- a/ggml/src/ggml-metal/kernels/fa_q8_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_q8_0.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 96, 64>; template [[host_name("kernel_flash_attn_ext_q8_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 112, 112>; template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 128, 128>; +template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 128, 96>; template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 192, 192>; template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 192, 128>; template [[host_name("kernel_flash_attn_ext_q8_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 256, 256>; diff --git a/ggml/src/ggml-metal/kernels/fa_tensor.metal b/ggml/src/ggml-metal/kernels/fa_tensor.metal new file mode 100644 index 000000000..4a9ed30e3 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_tensor.metal @@ -0,0 +1,298 @@ +#include "common.h" + +#ifdef GGML_METAL_HAS_TENSOR + +constant bool FC_flash_attn_ext_tensor_has_mask [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 0)]]; +constant bool FC_flash_attn_ext_tensor_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 1)]]; +constant bool FC_flash_attn_ext_tensor_has_bias [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 2)]]; +constant bool FC_flash_attn_ext_tensor_has_scap [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 3)]]; + +// ref: https://arxiv.org/pdf/2307.08691.pdf +template< + typename T, // Q, K, V and P types + typename T4, + short DK, // K head size + short DV, // V head size + short Q = OP_FLASH_ATTN_EXT_TENSOR_NQPSG, // queries per threadgroup + short C = OP_FLASH_ATTN_EXT_TENSOR_NCPSG, // cache items per threadgroup + short NSG = OP_FLASH_ATTN_EXT_TENSOR_NSG> // number of simd groups +kernel void kernel_flash_attn_ext_tensor( + constant ggml_metal_kargs_flash_attn_ext & args, + device const char * q, + device const char * k, + device const char * v, + device const char * mask, + device const char * sinks, + device const char * blk, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + constexpr short NW = N_SIMDWIDTH; + constexpr short NT = NW*NSG; + constexpr short NQ = Q/NSG; + constexpr short NC = C/NW; // columns per thread + + static_assert(DK % 4 == 0, "DK must be divisible by 4"); + static_assert(Q % NSG == 0, "Q must be divisible by NSG"); + static_assert(C % NW == 0, "C must be divisible by NW"); + + const int iq3 = tgpig[2]; + const int iq2 = tgpig[1]; + const int iq1 = tgpig[0]*Q; + + const short tiitg = sgitg*NW + tiisg; + + threadgroup T * sq = (threadgroup T *) shmem; // [Q, DK] queries + threadgroup float * ss = (threadgroup float *) (sq + Q*DK); // [Q, C] scores + threadgroup T * sp = (threadgroup T *) (ss + Q*C); // [Q, C] probabilities + threadgroup float * sr = (threadgroup float *) (sp + Q*C); // [Q] per-row scale of O + threadgroup int * sf = (threadgroup int *) (sr + Q); // [1] last iteration (ic0 + 1) that rescaled O + + q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03; + + { + const int ikv2 = iq2/(args.ne02/args.ne_12_2); + const int ikv3 = iq3/(args.ne03/args.ne_12_3); + + k += ikv2*args.nb12 + ikv3*args.nb13; + v += ikv2*args.nb22 + ikv3*args.nb23; + } + + // with softcap the scale is small (scale/softcap), so it is applied to the scores to keep the precision of Q + const float qscale = FC_flash_attn_ext_tensor_has_scap ? 1.0f : args.scale; + + // load the queries, with the scale folded in + for (int i = tiitg; i < Q*DK/4; i += NT) { + const int j = i/(DK/4); + + float4 q4 = 0.0f; + if (iq1 + j < args.ne01) { + q4 = ((device const float4 *) (q + j*args.nb01))[i%(DK/4)]; + } + + ((threadgroup T4 *) sq)[i] = (T4) (q4*qscale); + } + + device const half * pm[NQ]; + + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + pm[jj] = (device const half *) (mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); + } + + { + const int nblk1 = (args.ne01 + Q - 1)/Q; + const int nblk0 = (args.ne11 + C - 1)/C; + + blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0; + } + + float M[NQ]; + float S[NQ]; + + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + M[jj] = -FLT_MAX/2; + S[jj] = 0.0f; + } + + float slope = 1.0f; + + // ALiBi + if (FC_flash_attn_ext_tensor_has_bias) { + const short h = iq2; + + const float base = h < args.n_head_log2 ? args.m0 : args.m1; + const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; + + slope = pow(base, exph); + } + + const int sk = args.ns10; + const int sv = args.ns20; + + auto tq = tensor(sq, dextents<int32_t, 2>(DK, Q)); + auto ts = tensor(ss, dextents<int32_t, 2>(C, Q)); + auto tp = tensor(sp, dextents<int32_t, 2>(C, Q)); + + mpp::tensor_ops::matmul2d< + mpp::tensor_ops::matmul2d_descriptor(Q, C, DK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply), + execution_simdgroups<NSG>> mm_qk; + + mpp::tensor_ops::matmul2d< + mpp::tensor_ops::matmul2d_descriptor(Q, DV, C, false, false, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), + execution_simdgroups<NSG>> mm_pv; + + auto tv0 = tensor((device T *) v, dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv})); + + // the O matrix from the paper + auto co = mm_pv.template get_destination_cooperative_tensor<decltype(tp), decltype(tv0), float>(); + + FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { + if (co.is_valid_element(i)) { + co[i] = 0.0f; + } + } + + if (tiitg == 0) { + sf[0] = 0; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // the host guarantees ne11 % C == 0 + for (int ic0 = 0, ic = 0; ic < args.ne11; ++ic0, ic += C) { + char blk_cur = 1; + + if (FC_flash_attn_ext_tensor_has_mask) { + blk_cur = blk[ic0]; + + if (blk_cur == 0) { + continue; + } + } + + // Q*K^T + { + auto tk = tensor((device T *) (k + (uint64_t) ic*args.nb11), dextents<int32_t, 2>(DK, C), array<int, 2>({1, sk})); + + mm_qk.run(tq, tk, ts); + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // online softmax + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + float s[NC]; + + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + s[ii] = ss[j*C + ii*NW + tiisg]; + } + + if (FC_flash_attn_ext_tensor_has_scap) { + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + s[ii] = args.logit_softcap*precise::tanh(s[ii]*args.scale); + } + } + + if (FC_flash_attn_ext_tensor_has_mask && blk_cur != 2 && iq1 + j < args.ne31) { + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + s[ii] += slope*(float) pm[jj][ic + ii*NW + tiisg]; + } + } + + float m = M[jj]; + + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + m = max(m, s[ii]); + } + + m = simd_max(m); + + // lazy rescaling: move the running max only when it grows by more than 8 (e^8 fits in half) + float ms = 1.0f; + + if (m > M[jj] + 8.0f) { + ms = exp(M[jj] - m); + M[jj] = m; + + if (tiisg == 0) { + sf[0] = ic0 + 1; + } + } + + float sum = 0.0f; + + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + // the sum uses the same rounded values as P*V + const T p = (T) exp(s[ii] - M[jj]); + + sp[j*C + ii*NW + tiisg] = p; + + sum += (float) p; + } + + S[jj] = S[jj]*ms + simd_sum(sum); + + if (tiisg == 0) { + sr[j] = ms; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // O = diag(ms)*O + P*V + if (sf[0] == ic0 + 1) { + FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { + if (co.is_valid_element(i)) { + co[i] *= sr[co.get_multidimensional_index(i)[1]]; + } + } + } + + { + auto tv = tensor((device T *) (v + (uint64_t) ic*args.nb21), dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv})); + + mm_pv.run(tp, tv, co); + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + // the sink only adds to the denominator - its rescale of O is folded into the final scale + float ms = 1.0f; + + if (FC_flash_attn_ext_tensor_has_sinks) { + const float s = ((device const float *) sinks)[iq2]; + const float m = max(M[jj], s); + + ms = exp(M[jj] - m); + + S[jj] = S[jj]*ms + exp(s - m); + } + + if (tiisg == 0) { + sr[j] = S[jj] == 0.0f ? 0.0f : ms/S[jj]; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { + if (co.is_valid_element(i)) { + co[i] *= sr[co.get_multidimensional_index(i)[1]]; + } + } + + // store to global memory - rows past ne01 are clipped by the tensor extents + device float * pdst = (device float *) dst + ((uint64_t) iq3*args.ne2*args.ne1 + iq2 + (uint64_t) iq1*args.ne1)*DV; + + auto td = tensor(pdst, dextents<int32_t, 2>(DV, args.ne01 - iq1), array<int, 2>({1, args.ne1*DV})); + + co.store(td); +} + +typedef decltype(kernel_flash_attn_ext_tensor<half, half4, 64, 64>) flash_attn_ext_tensor_t; + +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk64_dv64" )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 64, 64>; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 128, 128>; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 192, 128>; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 256, 256>; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>; + +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk64_dv64" )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 64, 64>; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 128, 128>; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 192, 128>; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 256, 256>; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>; + +#endif // GGML_METAL_HAS_TENSOR diff --git a/ggml/src/ggml-metal/kernels/fa_vec_common.metal b/ggml/src/ggml-metal/kernels/fa_vec_common.metal index 46270ba4b..1a96c6f7e 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_common.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_common.metal @@ -13,6 +13,7 @@ constant int32_t FC_flash_attn_ext_vec_ns20 [[function_constant(FC_FLASH_ATTN_EX constant int32_t FC_flash_attn_ext_vec_nsg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 22)]]; constant int32_t FC_flash_attn_ext_vec_nwg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 23)]]; constant bool FC_flash_attn_ext_vec_has_sparse [[function_constant(FC_FLASH_ATTN_EXT_VEC + 5)]]; +constant bool FC_flash_attn_ext_vec_has_rows [[function_constant(FC_FLASH_ATTN_EXT_VEC + 6)]]; template< typename q4_t, // query types in shared memory typename k4_t, // key types in shared memory @@ -90,7 +91,8 @@ kernel void kernel_flash_attn_ext_vec( threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + SMEM_Q + SMEM_S + 2*sgitg*Q*PV); // scratch buffer for the results // sparse indices for the current block - threadgroup int * spidx = FC_flash_attn_ext_vec_has_sparse + // note: with kv_rows, the buffer holds the row list of the current slice instead + threadgroup int * spidx = (FC_flash_attn_ext_vec_has_sparse || FC_flash_attn_ext_vec_has_rows) ? (threadgroup int *) (shmem_f16 + SMEM) + sgitg*C : nullptr; @@ -129,9 +131,11 @@ kernel void kernel_flash_attn_ext_vec( } // zero out so - for (short qq = 0; qq < Q; ++qq) { - for (short i = 0; i < DV4/NL; ++i) { - so4[qq*DV4 + i*NL] = (o4_t) 0.0f; + if ((DV4/NL % NW == 0) || tiisg < NL) { + for (short qq = 0; qq < Q; ++qq) { + for (short i = 0; i < DV4/NL; ++i) { + so4[qq*DV4 + i*NL] = (o4_t) 0.0f; + } } } @@ -165,6 +169,12 @@ kernel void kernel_flash_attn_ext_vec( ((int64_t)(iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*args.ne31*args.n_kv_max_padded + (iq1%args.ne31)*args.n_kv_max_padded; } + // K/V row list of the current slice (ggml_flash_attn_ext_rows) + // with kv_rows, the loop below runs over the mask columns (args.ne11) and the K/V cache rows + // are read indirectly through the list. negative entries are padding, their mask is -INF + device const int * rows_list = FC_flash_attn_ext_vec_has_rows ? + (device const int *) idx + (int64_t)(iq3%args.ne33)*args.ne11 : nullptr; + float slope = 1.0f; // ALiBi @@ -222,12 +232,12 @@ kernel void kernel_flash_attn_ext_vec( ic = 0; } - // load the sparse KV indices for the current block into shared memory - if (FC_flash_attn_ext_vec_has_sparse) { + // load the KV row indices for the current block into shared memory + if (FC_flash_attn_ext_vec_has_sparse || FC_flash_attn_ext_vec_has_rows) { FOR_UNROLL (short ii = 0; ii < C/NW; ++ii) { const short i = ii*NW + tiisg; - spidx[i] = pidx[ic + i]; + spidx[i] = FC_flash_attn_ext_vec_has_rows ? rows_list[ic + i] : pidx[ic + i]; } simdgroup_barrier(mem_flags::mem_threadgroup); } @@ -242,6 +252,15 @@ kernel void kernel_flash_attn_ext_vec( sm[qq*C + tiisg] = -MAXHALF; } } + } else if (FC_flash_attn_ext_vec_has_rows) { + // the mask is compact: column ic maps to cache row spidx[tiisg] + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if ((iq1*Q + qq) < args.ne01) { + sm[qq*C + tiisg] = pm[qq][ic + tiisg]; + } else { + sm[qq*C + tiisg] = -MAXHALF; + } + } } else { FOR_UNROLL (short qq = 0; qq < Q; ++qq) { if ((iq1*Q + qq) < args.ne01) { @@ -276,7 +295,7 @@ kernel void kernel_flash_attn_ext_vec( { device const k4_t * pk4 = nullptr; - if (!FC_flash_attn_ext_vec_has_sparse) { + if (!FC_flash_attn_ext_vec_has_sparse && !FC_flash_attn_ext_vec_has_rows) { pk4 = (device const k4_t *) (k + ic*args.nb11); pk4 += ty*NS10/4 + tx; @@ -291,7 +310,7 @@ kernel void kernel_flash_attn_ext_vec( // each simdgroup processes Q queries and NE (NW/NL) cache elements FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { - if (FC_flash_attn_ext_vec_has_sparse) { + if (FC_flash_attn_ext_vec_has_sparse || FC_flash_attn_ext_vec_has_rows) { // the KV rows are gathered from the index list; -1 entries are padding const int i11 = spidx[NE*cc + ty]; if (i11 >= 0) { @@ -434,7 +453,7 @@ kernel void kernel_flash_attn_ext_vec( } } - if (FC_flash_attn_ext_vec_has_sparse) { + if (FC_flash_attn_ext_vec_has_sparse || FC_flash_attn_ext_vec_has_rows) { FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { // the KV rows are gathered from the index list; -1 entries are padding const int i11 = spidx[NE*cc + ty]; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_f16.metal b/ggml/src/ggml-metal/kernels/fa_vec_f16.metal index 56b5adb84..bea54f9f7 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_f16.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_f16.metal @@ -56,8 +56,12 @@ template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q2_ne4")]] kerne template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 128, 128, 1, 4>; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 128, 128, 2, 4>; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 128, 128, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 128, 96, 4>; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 128, 96, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 128, 96, 4, 4>; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, bfloat4, 1, dequantize_bf16_t4, bfloat4, 1, dequantize_bf16_t4, 128, 128, 1>; +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, bfloat4, 1, dequantize_bf16_t4, bfloat4, 1, dequantize_bf16_t4, 128, 96, 4>; #endif template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 192, 192, 2>; template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 192, 192, 4, 1>; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_f32.metal b/ggml/src/ggml-metal/kernels/fa_vec_f32.metal index 1c6bed0c6..18f862966 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_f32.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_f32.metal @@ -25,6 +25,7 @@ template [[host_name("kernel_flash_attn_ext_vec_f32_dk64_dv64")]] kernel flas template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 96, 96, 4>; template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 96, 64, 4>; template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 128, 128, 1>; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 128, 96, 4>; template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 192, 192, 2>; template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 192, 128, 2>; template [[host_name("kernel_flash_attn_ext_vec_f32_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 256, 256, 1>; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal index 7bb99ffd7..6bf5fad75 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 128, 128, 1, 4>; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 128, 128, 2, 4>; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 128, 128, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 128, 96, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 128, 96, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 128, 96, 4, 4>; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 192, 192, 2>; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 192, 192, 4, 1>; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 192, 192, 2, 2>; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal b/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal index 8a6b08bca..37277ee10 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 128, 128, 1, 4>; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 128, 128, 2, 4>; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 128, 128, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 128, 96, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 128, 96, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 128, 96, 4, 4>; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 192, 192, 2>; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 192, 192, 4, 1>; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 192, 192, 2, 2>; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal index 1633c4764..26868a228 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 128, 128, 1, 4>; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 128, 128, 2, 4>; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 128, 128, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 128, 96, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 128, 96, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 128, 96, 4, 4>; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 192, 192, 2>; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 192, 192, 4, 1>; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 192, 192, 2, 2>; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal b/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal index 2e7db017e..9d39dc58a 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 128, 128, 1, 4>; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 128, 128, 2, 4>; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 128, 128, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 128, 96, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 128, 96, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 128, 96, 4, 4>; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 192, 192, 2>; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 192, 192, 4, 1>; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 192, 192, 2, 2>; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal index e9c6b1522..7686ccc00 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 128, 128, 1, 4>; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 128, 128, 2, 4>; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 128, 128, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 128, 96, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 128, 96, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 128, 96, 4, 4>; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 192, 192, 2>; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 192, 192, 4, 1>; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 192, 192, 2, 2>; diff --git a/ggml/src/ggml-metal/kernels/misc.metal b/ggml/src/ggml-metal/kernels/misc.metal index fb27983db..5dfd4a60c 100644 --- a/ggml/src/ggml-metal/kernels/misc.metal +++ b/ggml/src/ggml-metal/kernels/misc.metal @@ -60,6 +60,140 @@ kernel void kernel_argmax_f32( dst_i32[tgpig] = arg_val; } +kernel void kernel_argmax_part_f32( + constant ggml_metal_kargs_argmax & args, + device const char * src0, + device char * tmp, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int row = tgpig.y; + const int part = tgpig.x; + const int ne00 = (int) args.ne00; + const int npart = (int) args.npart; + const int chunk = (ne00 + npart - 1) / npart; + const int start = part * chunk; + const int end = min(start + chunk, ne00); + + device const float * x_row = (device const float *) ((device const char *) src0 + row * args.nb01); + + float lmax = -INFINITY; + int32_t larg = -1; + + for (int i00 = start + tpitg.x; i00 < end; i00 += ntg.x) { + if (x_row[i00] > lmax) { + lmax = x_row[i00]; + larg = i00; + } + } + + // find the argmax value in the block + float max_val = simd_max(lmax); + int32_t arg_val = simd_max(select(-1, larg, lmax == max_val)); + + threadgroup float * shared_maxval = (threadgroup float *) shmem; + threadgroup int32_t * shared_argmax = (threadgroup int32_t *) shmem + N_SIMDWIDTH; + + if (ntg.x > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_maxval[tiisg] = -INFINITY; + shared_argmax[tiisg] = -1; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_maxval[sgitg] = max_val; + shared_argmax[sgitg] = arg_val; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = shared_maxval[tiisg]; + arg_val = shared_argmax[tiisg]; + + float max_val_reduced = simd_max(max_val); + int32_t arg_val_reduced = simd_max(select(-1, arg_val, max_val == max_val_reduced)); + + max_val = max_val_reduced; + arg_val = arg_val_reduced; + } + + if (tpitg.x == 0) { + device float * tmp_max = (device float *) tmp; + device int32_t * tmp_arg = (device int32_t *) tmp + npart * (int) args.ne01; + + tmp_max[row*npart + part] = max_val; + tmp_arg[row*npart + part] = arg_val; + } +} + +kernel void kernel_argmax_final_f32( + constant ggml_metal_kargs_argmax & args, + device const char * tmp, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int row = tgpig.x; + const int npart = (int) args.npart; + + device const float * tmp_max = (device const float *) tmp; + device const int32_t * tmp_arg = (device const int32_t *) tmp + npart * (int) args.ne01; + + float lmax = -INFINITY; + int32_t larg = -1; + + for (int i = tpitg.x; i < npart; i += ntg.x) { + if (tmp_max[row*npart + i] > lmax) { + lmax = tmp_max[row*npart + i]; + larg = tmp_arg[row*npart + i]; + } + } + + // find the argmax value in the block + float max_val = simd_max(lmax); + int32_t arg_val = simd_max(select(-1, larg, lmax == max_val)); + + threadgroup float * shared_maxval = (threadgroup float *) shmem; + threadgroup int32_t * shared_argmax = (threadgroup int32_t *) shmem + N_SIMDWIDTH; + + if (ntg.x > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_maxval[tiisg] = -INFINITY; + shared_argmax[tiisg] = -1; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_maxval[sgitg] = max_val; + shared_argmax[sgitg] = arg_val; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = shared_maxval[tiisg]; + arg_val = shared_argmax[tiisg]; + + float max_val_reduced = simd_max(max_val); + int32_t arg_val_reduced = simd_max(select(-1, arg_val, max_val == max_val_reduced)); + + max_val = max_val_reduced; + arg_val = arg_val_reduced; + } + + if (tpitg.x == 0) { + ((device int32_t *) dst)[row] = arg_val; + } +} + kernel void kernel_diag_f32( constant ggml_metal_kargs_diag & args, device const char * src0, @@ -772,3 +906,99 @@ kernel void kernel_dsv4_hc_post_f32( *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = result[idst]; } } + +constant bool FC_top_k_mask_has_score [[function_constant(FC_TOP_K_MASK + 0)]]; +constant bool FC_top_k_mask_has_blocks [[function_constant(FC_TOP_K_MASK + 1)]]; +constant bool FC_top_k_mask_has_extra [[function_constant(FC_TOP_K_MASK + 2)]]; +constant bool FC_top_k_mask_vec [[function_constant(FC_TOP_K_MASK + 3)]]; + +// the fill runs before the scatter, both are fully parallel over the whole dst and over the picks of one row +template <typename T, typename T4> +kernel void kernel_top_k_mask_fill( + constant ggml_metal_kargs_top_k_mask & args, + device char * dst, + uint tid[[thread_position_in_grid]], + uint ntot[[threads_per_grid]]) { + const T ninf = T(-INFINITY); + + if (FC_top_k_mask_vec) { + device T4 * dst_4 = (device T4 *) dst; + for (int64_t i = tid; i < args.n_elems/4; i += ntot) { + dst_4[i] = T4(ninf); + } + } else { + device T * dst_t = (device T *) dst; + for (int64_t i = tid; i < args.n_elems; i += ntot) { + dst_t[i] = ninf; + } + } +} + +typedef decltype(kernel_top_k_mask_fill<half, half4>) kernel_top_k_mask_fill_h_t; +typedef decltype(kernel_top_k_mask_fill<float, float4>) kernel_top_k_mask_fill_f_t; + +template [[host_name("kernel_top_k_mask_fill_f16")]] kernel kernel_top_k_mask_fill_h_t kernel_top_k_mask_fill<half, half4>; +template [[host_name("kernel_top_k_mask_fill_f32")]] kernel kernel_top_k_mask_fill_f_t kernel_top_k_mask_fill<float, float4>; + +// one threadgroup per row: copy the mask cells of the live picks and of the extra cells over the fill +template <typename T, typename T4> +kernel void kernel_top_k_mask( + constant ggml_metal_kargs_top_k_mask & args, + device const char * mask, + device const char * top_k, + device const char * score, + device const char * blocks, + device const char * extra, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg_max[[threads_per_threadgroup]]) { + const int i1 = tgpig.x; + const int i2 = tgpig.y; + const int i3 = tgpig.z; + + const int tiitg = tpitg.x; + const int ntg = ntg_max.x; + + device const char * mask_row = mask + (i1 % args.m1)*args.nbm1 + (i2 % args.m2)*args.nbm2 + (i3 % args.m3)*args.nbm3; + device char * dst_row = dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3; + device const char * top_k_row = top_k + i1*args.nbk1 + i2*args.nbk2 + i3*args.nbk3; + + device const T * mask_row_t = (device const T *) mask_row; + device T * dst_row_t = (device T *) dst_row; + + for (int i = tiitg; i < args.n_top; i += ntg) { + const int32_t p = *(device const int32_t *) (top_k_row + i*args.nbk0); + if (p < 0 || p >= args.n_pick) { + continue; + } + if (FC_top_k_mask_has_score && !(*(device const float *) (score + p*args.nbs0 + i1*args.nbs1 + i2*args.nbs2 + i3*args.nbs3) > -INFINITY)) { + continue; + } + if (FC_top_k_mask_has_blocks) { + FOR_UNROLL (int k = 0; k < args.n_blk; ++k) { + const int32_t c = *(device const int32_t *) (blocks + k*args.nbb0 + p*args.nbb1); + if (c >= 0 && c < args.n_kv) { + dst_row_t[c] = mask_row_t[c]; + } + } + } else if (p < args.n_kv) { + dst_row_t[p] = mask_row_t[p]; + } + } + + if (FC_top_k_mask_has_extra) { + for (int i = tiitg; i < args.n_extra; i += ntg) { + const int32_t c = *(device const int32_t *) (extra + i*args.nbe0 + i1*args.nbe1 + i2*args.nbe2 + i3*args.nbe3); + if (c >= 0 && c < args.n_kv) { + dst_row_t[c] = mask_row_t[c]; + } + } + } +} + +typedef decltype(kernel_top_k_mask<half, half4>) kernel_top_k_mask_h_t; +typedef decltype(kernel_top_k_mask<float, float4>) kernel_top_k_mask_f_t; + +template [[host_name("kernel_top_k_mask_f16")]] kernel kernel_top_k_mask_h_t kernel_top_k_mask<half, half4>; +template [[host_name("kernel_top_k_mask_f32")]] kernel kernel_top_k_mask_f_t kernel_top_k_mask<float, float4>; diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metal index 9d262da1d..fab720097 100644 --- a/ggml/src/ggml-metal/kernels/mul_mm.metal +++ b/ggml/src/ggml-metal/kernels/mul_mm.metal @@ -7,7 +7,7 @@ constant short FC_mul_mm_ne12 [[function_constant(FC_MUL_MM + 2)]]; constant short FC_mul_mm_ne13 [[function_constant(FC_MUL_MM + 3)]]; constant short FC_mul_mm_r2 [[function_constant(FC_MUL_MM + 4)]]; constant short FC_mul_mm_r3 [[function_constant(FC_MUL_MM + 5)]]; -constant bool FC_mul_mm_id_amax [[function_constant(FC_MUL_MM + 6)]]; +constant int32_t FC_mul_mm_id_map0_ne02 [[function_constant(FC_MUL_MM + 7)]]; // each block_q contains 16*nl weights #ifdef GGML_METAL_HAS_TENSOR @@ -134,11 +134,15 @@ kernel void kernel_mul_mm( } // Store result tile to output matrix (with batch offset) - // cT.store handles bounds checking via tD's extents (M, N) - device float * dstBatch = (device float *)dst + im * N * M; + // a slice is not used because its byte offset is int32 and wraps at 2 GiB + // ref: https://github.com/ggml-org/llama.cpp/pull/28748 + const int tileM = min(NRA, M - ra); + const int tileN = min(NRB, N - rb); - auto tD = tensor(dstBatch, dextents<int32_t, 2>(M, N), array<int, 2>({1, M})); - cT.store(tD.slice(ra, rb)); + device float * dstTile = (device float *)dst + (uint64_t) im * N * M + (uint64_t) rb * M + ra; + + auto tD = tensor(dstTile, dextents<int32_t, 2>(tileM, tileN), array<int, 2>({1, M})); + cT.store(tD); } #else @@ -319,10 +323,10 @@ kernel void kernel_mul_mm( // if no bounds checks on the output are needed, we can directly write to device memory device float * C = (device float *) dst + (r0 + 32*(sgitg & 1)) + \ - (r1 + 16*(sgitg >> 1)) * args.ne0 + im*args.ne1*args.ne0; + (uint64_t) (r1 + 16*(sgitg >> 1))*args.ne0 + (uint64_t) im*args.ne1*args.ne0; for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false); + simdgroup_store(mc[i], C + 8*(i%4) + (uint64_t) 8*args.ne0*(i/4), args.ne0, 0, false); } } else { // block is smaller than 64x32, we should avoid writing data outside of the matrix @@ -338,7 +342,7 @@ kernel void kernel_mul_mm( if (sgitg == 0) { for (int j = tiitg; j < nr1; j += NR1) { - device float * D = (device float *) dst + r0 + (r1 + j)*args.ne0 + im*args.ne1*args.ne0; + device float * D = (device float *) dst + r0 + (uint64_t) (r1 + j)*args.ne0 + (uint64_t) im*args.ne1*args.ne0; device float4 * D4 = (device float4 *) D; threadgroup float * C = temp_str + (j*NR0); @@ -371,19 +375,25 @@ kernel void kernel_mul_mm_id_map0( ushort ntg[[threads_per_threadgroup]]) { const short ide = tpitg; // expert id - uint32_t n_all = 0; + // per expert: entry count (pass 1), replaced by the base offset of the slice (pass 2) + threadgroup uint32_t * cnt = (threadgroup uint32_t *) (shmem); + threadgroup uint32_t * bas = cnt + FC_mul_mm_id_map0_ne02; - device int32_t * ids_i32 = (device int32_t *) hids + ide*args.ne21; + // staging of the ids of ntg tokens + threadgroup uint16_t * sids = (threadgroup uint16_t *) (bas + FC_mul_mm_id_map0_ne02); + + // pass 1: count the entries of this expert - every occurrence of this expert within the + // ids is an entry (duplicate ids produce multiple entries for the same token) + uint32_t n_all = 0; for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens if (i21 + tpitg < args.ne21) { device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21); - threadgroup uint16_t * sids = (threadgroup uint16_t *) shmem + tpitg*ne20; - - #pragma unroll(ne20) - for (short i20 = 0; i20 < ne20; i20++) { - sids[i20] = src2_i32[i20]; + // 0xFFFF marks a skipped slot, it never matches an expert since ne02 fits in a threadgroup + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + const int32_t id = src2_i32[i20]; + sids[tpitg*ne20 + i20] = id == -1 ? 0xFFFF : (uint16_t) id; } } @@ -394,102 +404,130 @@ kernel void kernel_mul_mm_id_map0( break; } - threadgroup const uint16_t * sids = (threadgroup const uint16_t *) shmem + t*ne20; + threadgroup const uint16_t * sids_t = sids + t*ne20; - short sel = 0; - #pragma unroll(ne20) - for (short i20 = 0; i20 < ne20; i20++) { - sel += (sids[i20] == ide)*(i20 + 1); + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + n_all += sids_t[i20] == ide; } - - ids_i32[n_all] = (i21 + t)*ne20 + sel - 1; - - n_all += sel > 0; } threadgroup_barrier(mem_flags::mem_threadgroup); } - device uint32_t * tpe_u32 = (device uint32_t *) (htpe); - tpe_u32[ide] = n_all; -} + cnt[ide] = n_all; -kernel void kernel_mul_mm_id_amax_part_f32( - constant ggml_metal_kargs_mul_mm_id_amax & args, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort ntg[[threads_per_threadgroup]]) { - const int nrow = args.ne01*args.ne02; + threadgroup_barrier(mem_flags::mem_threadgroup); + + // exclusive prefix sum over the counts: the entries of this expert start at bas[ide] - + // every slice is padded by one entry for the uniform dummy store of pass 2 - and emit the + // entry tile descriptors of kernel_mul_mm_id: (expert, first entry, valid rows) per tile + if (tpitg == 0) { + constexpr uint32_t NTILE = 32; // must match NR1 of kernel_mul_mm_id + + uint32_t base = 0; + uint32_t tile = 0; - float lmax = 0.0f; + device uint32_t * tpe_u32 = (device uint32_t *) (htpe); - for (int ir = tgpig; ir < nrow; ir += N_MM_NPART_AMAX) { - const int i01 = ir % args.ne01; - const int i02 = ir / args.ne01; + for (int e = 0; e < FC_mul_mm_id_map0_ne02; ++e) { + const uint32_t n = cnt[e]; - device const float * row = (device const float *) (src1 + i02*args.nb02 + i01*args.nb01); + bas[e] = base; - for (int i00 = tiitg; i00 < args.ne00; i00 += ntg) { - lmax = max(lmax, fabs(row[i00])); + for (uint32_t k = 0; k < n; k += NTILE) { + tpe_u32[1 + 3*tile ] = e; + tpe_u32[1 + 3*tile + 1] = base + k; + tpe_u32[1 + 3*tile + 2] = n - k < NTILE ? n - k : NTILE; + + tile++; + } + + base += n + 1; } + + tpe_u32[0] = tile; } - float amax = simd_max(lmax); + threadgroup_barrier(mem_flags::mem_threadgroup); - threadgroup float * shared_amax = (threadgroup float *) shmem; + // pass 2: write the entries of this expert into its slice of the compact list + device int32_t * ids_i32 = (device int32_t *) (hids); - if (ntg > N_SIMDWIDTH) { - if (sgitg == 0) { - shared_amax[tiisg] = 0.0f; - } - threadgroup_barrier(mem_flags::mem_threadgroup); + uint32_t ient = bas[ide]; + + for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens + if (i21 + tpitg < args.ne21) { + device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21); - if (tiisg == 0) { - shared_amax[sgitg] = amax; + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + sids[tpitg*ne20 + i20] = src2_i32[i20]; + } } + threadgroup_barrier(mem_flags::mem_threadgroup); - amax = shared_amax[tiisg]; - amax = simd_max(amax); - } + for (short t = 0; t < ntg; t++) { + if (i21 + t >= args.ne21) { + break; + } - if (tiitg == 0) { - ((device float *) (dst + 8))[tgpig] = amax; - } -} + threadgroup const uint16_t * sids_t = sids + t*ne20; -kernel void kernel_mul_mm_id_amax_f32( - device char * dst, - ushort tiitg[[thread_index_in_threadgroup]]) { - device const float * part = (device const float *) (dst + 8); + short first = -1; + short n_ents = 0; - float amax = 0.0f; + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + const short hit = sids_t[i20] == ide; - for (int i = tiitg; i < N_MM_NPART_AMAX; i += N_SIMDWIDTH) { - amax = max(amax, part[i]); - } + first = hit && first < 0 ? i20 : first; + n_ents += hit; + } + + // one store per token for every thread, regardless of the match - conditional + // stores of the divergent lanes are very slow - the dummy value of the tokens + // without a match lands on the padding slot of this slice and is never read + ids_i32[ient] = first >= 0 ? (i21 + t)*ne20 + first : 0; - amax = simd_max(amax); + if (n_ents > 1) { + // duplicate ids: the remaining matches of this token + short j = 1; - if (tiitg == 0) { - // leave a comfortable margin below the f16 max of 65504 - float scale = 1.0f; + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + if (sids_t[i20] == ide && i20 != first) { + ids_i32[ient + j] = (i21 + t)*ne20 + i20; + j += 1; + } + } + } - // isfinite: src1 already inf/nan is not ours to fix - keep the - // scale at 1.0 instead of turning it into a different failure - if (isfinite(amax) && amax > 32768.0f) { - scale = exp2(ceil(log2(amax)) - 15.0f); + ient += n_ents; } - device float * d = (device float *) dst; + threadgroup_barrier(mem_flags::mem_threadgroup); + } +} - d[0] = 1.0f/scale; // exact: scale is a power of two - d[1] = scale; +// kernel_mul_mm_id only writes the rows that an expert owns, so the skipped rows are zeroed here +kernel void kernel_mul_mm_id_zero( + constant ggml_metal_kargs_mul_mm_id_zero & args, + device const char * src2, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int i20 = tgpig.x; // slot + const int i21 = tgpig.y; // token + + const int32_t id = ((device const int32_t *) (src2 + i21*args.nb21))[i20]; + + if (id != -1) { + return; + } + + device float * dst_row = (device float *) (dst + i20*args.nb1 + i21*args.nb2); + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + dst_row[i0] = 0.0f; } } @@ -501,10 +539,12 @@ template [[host_name("kernel_mul_mm_id_map0_ne20_4" )]] kernel kernel_mul_mm_id_ template [[host_name("kernel_mul_mm_id_map0_ne20_5" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<5>; template [[host_name("kernel_mul_mm_id_map0_ne20_6" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<6>; template [[host_name("kernel_mul_mm_id_map0_ne20_8" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<8>; +template [[host_name("kernel_mul_mm_id_map0_ne20_9" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<9>; template [[host_name("kernel_mul_mm_id_map0_ne20_10")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<10>; template [[host_name("kernel_mul_mm_id_map0_ne20_16")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<16>; template [[host_name("kernel_mul_mm_id_map0_ne20_22")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<22>; +// TODO: refactor into two seperate functions to avoid ifdef branching: with and without tensor support template<typename S0, typename S0_4x4, typename S0_8x8, typename S1, typename S1_2x4, typename S1_8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), typename T0, typename T0_4x4, typename T1, typename T1_2x4> kernel void kernel_mul_mm_id( constant ggml_metal_kargs_mul_mm_id & args, @@ -513,7 +553,6 @@ kernel void kernel_mul_mm_id( device const char * htpe, device const char * hids, device char * dst, - device const char * amax, threadgroup char * shmem [[threadgroup(0)]], uint3 tgpig[[threadgroup_position_in_grid]], ushort tiitg[[thread_index_in_threadgroup]], @@ -533,71 +572,47 @@ kernel void kernel_mul_mm_id( constexpr int NL0 = NK/16; constexpr int NL1 = NK/8; - const int im = tgpig.z; // expert - const int r0 = tgpig.y*NR0; - const int r1 = tgpig.x*NR1; + const int tile = tgpig.x; // entry tile, described by the prepass + const int r0 = tgpig.y*NR0; device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe); device const int32_t * ids_i32 = (device const int32_t *) (hids); - const int32_t neh1 = tpe_u32[im]; - - if (r1 >= neh1) { + // the prepass emits one (expert, first entry, valid rows) descriptor per tile that has + // work - the grid is sized by a static upper bound, so the tail early exits + if (tile >= (int) tpe_u32[0]) { return; } + const int im = tpe_u32[1 + 3*tile]; // expert + const int idb = tpe_u32[1 + 3*tile + 1]; // first entry of this tile in hids + + // number of entries in this tile, always <= NR1 + const short nr1 = (short) ((tpe_u32[1 + 3*tile + 2] < NR1) ? tpe_u32[1 + 3*tile + 2] : NR1); + // if this block is of 64x32 shape or smaller const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; - const short nr1 = ( neh1 - r1 < NR1) ? ( neh1 - r1) : NR1; // a thread shouldn't load data outside of the matrix const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63 - const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31 const short il0 = (tiitg % NL0); - short il = il0; - - const int id = ids_i32[im*args.ne21 + r1 + lr1]; - - const short i11 = (id % args.ne20) % args.ne11; - const short i12 = (id / args.ne20); const short i13 = 0; const uint64_t offset0 = im*args.nb02 + i13*args.nb03; const short offset1 = il0/nl; - device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1; - const short iy = 8*(tiitg % NL1); - device const T1 * y = (device const T1 *)(src1 - + args.nb13*i13 - + args.nb12*i12 - + args.nb11*i11 - + args.nb10*iy); - // skip the upper half of the token tile when the expert did not fill it constexpr short NR1H = NR1/2; - const bool has_hi = nr1 > NR1H; - - // power-of-two rescaling - const float s1_inv = FC_mul_mm_id_amax ? ((device const float *) amax)[0] : 1.0f; - const float s1_scale = FC_mul_mm_id_amax ? ((device const float *) amax)[1] : 1.0f; - #ifndef GGML_METAL_HAS_TENSOR S0_8x8 ma[4]; S1_8x8 mb[2]; simdgroup_float8x8 mc[8]; - - for (short i = 0; i < 8; i++){ - mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f); - } - - // simdgroups 2,3 own rows NR1H..NR1-1 - const bool sg_active = has_hi || sgitg < 2; #else auto tA = tensor<threadgroup S0, dextents<int32_t, 2>, tensor_inline>(sa, dextents<int32_t, 2>(NK, NR0)); @@ -609,6 +624,35 @@ kernel void kernel_mul_mm_id( mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), execution_simdgroups<4>> mm; +#endif + + const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31 + + short il = il0; + + const int id = ids_i32[idb + lr1]; + + const short i11 = (id % args.ne20) % args.ne11; + const int i12 = (id / args.ne20); + + device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1; + + device const T1 * y = (device const T1 *)(src1 + + args.nb13*i13 + + args.nb12*i12 + + args.nb11*i11 + + args.nb10*iy); + + const bool has_hi = nr1 > NR1H; + +#ifndef GGML_METAL_HAS_TENSOR + for (short i = 0; i < 8; i++){ + mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f); + } + + // simdgroups 2,3 own rows NR1H..NR1-1 + const bool sg_active = has_hi || sgitg < 2; +#else auto cT0 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB0), float>(); auto cT1 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB1), float>(); #endif @@ -669,7 +713,7 @@ kernel void kernel_mul_mm_id( const short ib = 4*sx + sy; - *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) (*((device T1 *) y + i) * (T1) s1_inv) : 0; + *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; } } else { const short sx = (tiitg%NL1); @@ -682,7 +726,7 @@ kernel void kernel_mul_mm_id( const short ib = 4*sx + sy; - *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)((*((device T1_2x4 *) y)) * (T1) s1_inv); + *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y)); } #else // load data and store to threadgroup memory @@ -730,7 +774,7 @@ kernel void kernel_mul_mm_id( //const short lx = (tiitg/NL1)%8; //const short ly = i; - *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) (*((device T1 *) y + i) * (T1) s1_inv) : 0; + *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; } } else { const short sx = (tiitg%NL1); @@ -741,7 +785,7 @@ kernel void kernel_mul_mm_id( //const short lx = (tiitg/NL1)%8; //const short ly = i; - *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)((*((device T1_2x4 *) y)) * (T1) s1_inv); + *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y)); } #endif @@ -819,12 +863,12 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); for (short j = sgitg; j < nr1; j += 4) { - const int id = ids_i32[im*args.ne21 + r1 + j]; + const int id = ids_i32[idb + j]; const short ide = id % args.ne20; - const short idt = id / args.ne20; + const int idt = id / args.ne20; - device float * D = (device float *) dst + r0 + ide*args.ne0 + idt*args.ne1*args.ne0; + device float * D = (device float *) dst + r0 + (uint64_t) ide*args.ne0 + (uint64_t) idt*args.ne1*args.ne0; device float4 * D4 = (device float4 *) D; threadgroup float * C = (threadgroup float *) shmem + j*NR0; @@ -832,12 +876,12 @@ kernel void kernel_mul_mm_id( int i = tiisg; for (; i < nr0/4; i += 32) { - *(D4 + i) = *(C4 + i) * s1_scale; + *(D4 + i) = *(C4 + i); } i = (4*(nr0/4)) + tiisg; for (; i < nr0; i += 32) { - *(D + i) = *(C + i) * s1_scale; + *(D + i) = *(C + i); } } } @@ -851,7 +895,8 @@ typedef decltype(kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, template [[host_name("kernel_mul_mm_f32_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, float4x4, 1, dequantize_f32, float, float4x4, float, float2x4>; template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, half4x4, 1, dequantize_f16, half, half4x4, float, float2x4>; #if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm<bfloat, bfloat4x4, simdgroup_bfloat8x8, bfloat, bfloat2x4, simdgroup_bfloat8x8, bfloat4x4, 1, dequantize_bf16, bfloat, bfloat4x4, float, float2x4>; +template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm<bfloat, bfloat4x4, simdgroup_bfloat8x8, bfloat, bfloat2x4, simdgroup_bfloat8x8, bfloat4x4, 1, dequantize_bf16, bfloat, bfloat4x4, float, float2x4>; +template [[host_name("kernel_mul_mm_bf16_bf16")]] kernel mul_mm_t kernel_mul_mm<bfloat, bfloat4x4, simdgroup_bfloat8x8, bfloat, bfloat2x4, simdgroup_bfloat8x8, bfloat4x4, 1, dequantize_bf16, bfloat, bfloat4x4, bfloat, bfloat2x4>; #endif template [[host_name("kernel_mul_mm_q1_0_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q1_0, 8, dequantize_q1_0, float, float4x4, float, float2x4>; template [[host_name("kernel_mul_mm_q2_0_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_0, 4, dequantize_q2_0, float, float4x4, float, float2x4>; diff --git a/ggml/src/ggml-metal/kernels/mul_mv.metal b/ggml/src/ggml-metal/kernels/mul_mv.metal index 0d1069f2c..5d012e484 100644 --- a/ggml/src/ggml-metal/kernels/mul_mv.metal +++ b/ggml/src/ggml-metal/kernels/mul_mv.metal @@ -3030,6 +3030,22 @@ kernel void kernel_mul_mv_iq4_xs_f32( kernel_mul_mv_iq4_xs_f32_impl<N_R0_IQ4_XS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } +// decode the E2M1 value in the low nibble of n, scaled by 2^-14, without a lookup table: the 3 magnitude bits +// land in the fp16 exponent/mantissa (e=0 gives the 0.5 subnormal). same decode as MLX's fp4_e2m1 +static inline half mxfp4_dq1(uint32_t n) { + const half c = as_type<half>(ushort((n & 7) << 9)); + return (n & 8) ? -c : c; +} + +// low / high nibbles of the 4 bytes of w, in byte order, scaled by 2^-14 +static inline float4 mxfp4_lo4(uint32_t w) { + return float4(mxfp4_dq1(w), mxfp4_dq1(w >> 8), mxfp4_dq1(w >> 16), mxfp4_dq1(w >> 24)); +} + +static inline float4 mxfp4_hi4(uint32_t w) { + return float4(mxfp4_dq1(w >> 4), mxfp4_dq1(w >> 12), mxfp4_dq1(w >> 20), mxfp4_dq1(w >> 28)); +} + template<int NR0, typename args_t> void kernel_mul_mv_mxfp4_f32_impl( args_t args, @@ -3042,8 +3058,6 @@ void kernel_mul_mv_mxfp4_f32_impl( ushort sgitg) { const short NSG = FC_mul_mv_nsg; - threadgroup float * shmem_f32 = (threadgroup float *) shmem; - const int r0 = tgpig.x; const int r1 = tgpig.y; const int im = tgpig.z; @@ -3065,9 +3079,6 @@ void kernel_mul_mv_mxfp4_f32_impl( const short ix = tiisg/2; // 0...15 const short it = tiisg%2; // 0 or 1 - shmem_f32[tiisg] = kvalues_mxfp4_f[tiisg%16]; - threadgroup_barrier(mem_flags::mem_threadgroup); - float4 yl[4]; float sumf[NR0]={0.f}; @@ -3087,14 +3098,25 @@ void kernel_mul_mv_mxfp4_f32_impl( device const block_mxfp4 & xb = x[row*ns01 + ib]; device const uint8_t * q2 = (device const uint8_t *)(xb.qs + 8*it); - float4 acc1 = yl[0]*float4(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]); - float4 acc2 = yl[1]*float4(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]); - float4 acc3 = yl[2]*float4(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]); - float4 acc4 = yl[3]*float4(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]); + // blocks are 17 bytes, so the 8 quant bytes are only byte aligned: read the aligned words that cover them + // and funnel-shift. the third word is read only when needed so no word past the block end is touched + device const uint32_t * pw = (device const uint32_t *)((ulong) q2 & ~3ul); + const uint sh = ((uint)(ulong) q2 & 3u)*8; + const uint32_t a = pw[0]; + const uint32_t b = pw[1]; + const uint32_t c = pw[sh ? 2 : 1]; + const uint32_t w0 = (a >> sh) | ((b << 1) << (31 - sh)); + const uint32_t w1 = (b >> sh) | ((c << 1) << (31 - sh)); + + float4 acc1 = yl[0]*mxfp4_lo4(w0); + float4 acc2 = yl[1]*mxfp4_hi4(w0); + float4 acc3 = yl[2]*mxfp4_lo4(w1); + float4 acc4 = yl[3]*mxfp4_hi4(w1); acc1 = (acc1 + acc3) + (acc2 + acc4); - sumf[row] += e8m0_to_fp32(xb.e) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3])); + // 16384 undoes the 2^-14 of the decode + sumf[row] += (e8m0_to_fp32(xb.e)*16384.0f) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3])); } yb += 16 * QK_MXFP4; @@ -3302,7 +3324,8 @@ kernel void kernel_mul_mv_id( uint3 tgpig[[threadgroup_position_in_grid]], ushort tiitg[[thread_index_in_threadgroup]], ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg [[threads_per_threadgroup]]) { const int iid1 = tgpig.z/args.nei0; const int idx = tgpig.z%args.nei0; @@ -3316,11 +3339,20 @@ kernel void kernel_mul_mv_id( const int64_t i1 = idx; const int64_t i2 = i12; + device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float); + + if (i02 == -1) { + device float * dst_f32 = (device float *) dst_cur; + const ushort nth = ntg.x*ntg.y*ntg.z; + for (int i0 = tiitg; i0 < args.ne0; i0 += nth) { + dst_f32[i0] = 0.0f; + } + return; + } + device const char * src0_cur = src0s + i02*args.nb02; device const char * src1_cur = src1 + i11*args.nb11 + i12*args.nb12; - device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float); - ggml_metal_kargs_mul_mv args0 = { /*.ne00 =*/ args.ne00, /*.ne01 =*/ args.ne01, diff --git a/ggml/src/ggml-metal/kernels/mul_mv_mma.metal b/ggml/src/ggml-metal/kernels/mul_mv_mma.metal index fcf981a93..5cf245d52 100644 --- a/ggml/src/ggml-metal/kernels/mul_mv_mma.metal +++ b/ggml/src/ggml-metal/kernels/mul_mv_mma.metal @@ -605,4 +605,23 @@ MUL_MV_MMA_GEN("q5_1", block_q5_1, 2, dequantize_q5_1) MUL_MV_MMA_GEN("q4_K", block_q4_K, QK_NL, dequantize_q4_K) MUL_MV_MMA_GEN("q6_K", block_q6_K, QK_NL, dequantize_q6_K) +#if defined(GGML_METAL_HAS_BF16) +MUL_MV_MMA_GEN("bf16", bfloat4x4, 1, dequantize_bf16) +#endif +MUL_MV_MMA_GEN("q1_0", block_q1_0, 8, dequantize_q1_0) +MUL_MV_MMA_GEN("q2_0", block_q2_0, 4, dequantize_q2_0) +MUL_MV_MMA_GEN("mxfp4", block_mxfp4, 2, dequantize_mxfp4) +MUL_MV_MMA_GEN("q2_K", block_q2_K, QK_NL, dequantize_q2_K) +MUL_MV_MMA_GEN("q3_K", block_q3_K, QK_NL, dequantize_q3_K) +MUL_MV_MMA_GEN("iq2_xxs", block_iq2_xxs, QK_NL, dequantize_iq2_xxs) +MUL_MV_MMA_GEN("iq2_xs", block_iq2_xs, QK_NL, dequantize_iq2_xs) +MUL_MV_MMA_GEN("iq2_s", block_iq2_s, QK_NL, dequantize_iq2_s) +MUL_MV_MMA_GEN("iq3_xxs", block_iq3_xxs, QK_NL, dequantize_iq3_xxs) +MUL_MV_MMA_GEN("iq3_s", block_iq3_s, QK_NL, dequantize_iq3_s) +MUL_MV_MMA_GEN("iq1_s", block_iq1_s, QK_NL, dequantize_iq1_s) +MUL_MV_MMA_GEN("iq1_m", block_iq1_m, QK_NL, dequantize_iq1_m) +MUL_MV_MMA_GEN("iq4_nl", block_iq4_nl, 2, dequantize_iq4_nl) +MUL_MV_MMA_GEN("iq4_xs", block_iq4_xs, QK_NL, dequantize_iq4_xs) +MUL_MV_MMA_GEN("tq2_0", block_tq2_0, QK_NL, dequantize_tq2_0) + #undef MUL_MV_MMA_GEN diff --git a/ggml/src/ggml-metal/kernels/norm.metal b/ggml/src/ggml-metal/kernels/norm.metal index c76d0c7ff..983d9b5cb 100644 --- a/ggml/src/ggml-metal/kernels/norm.metal +++ b/ggml/src/ggml-metal/kernels/norm.metal @@ -236,37 +236,59 @@ typedef decltype(kernel_l2_norm_impl<float, float>) kernel_l2_norm_t; template [[host_name("kernel_l2_norm_f32_f32")]] kernel kernel_l2_norm_t kernel_l2_norm_impl<float, float>; template [[host_name("kernel_l2_norm_f32_f32_4")]] kernel kernel_l2_norm_t kernel_l2_norm_impl<float4, float4>; +constant bool FC_group_norm_flat [[function_constant(FC_GROUP_NORM + 0)]]; + kernel void kernel_group_norm_f32( constant ggml_metal_kargs_group_norm & args, - device const float * src0, - device float * dst, + device const char * src0, + device char * dst, threadgroup float * buf [[threadgroup(0)]], - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint sgitg[[simdgroup_index_in_threadgroup]], - uint tiisg[[thread_index_in_simdgroup]], - uint ntg[[threads_per_threadgroup]]) { - const int64_t ne = args.ne00*args.ne01*args.ne02; - const int64_t gs = args.ne00*args.ne01*((args.ne02 + args.ngrp - 1) / args.ngrp); + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int64_t ne00 = args.ne00; + const int64_t ne01 = args.ne01; + const int64_t ne02 = args.ne02; - int start = tgpig * gs; - int end = start + gs; + const int64_t ngrp = args.ngrp; - start += tpitg; + const int64_t i03 = tgpig.z; // batch index + const int64_t ig = tgpig.x; // group index - if (end >= ne) { - end = ne; - } + const int64_t cpg = (ne02 + ngrp - 1)/ngrp; + + const int64_t c0 = ig*cpg; + const int64_t c1 = (c0 + cpg < ne02) ? c0 + cpg : ne02; + + const int64_t npf = ne00*ne01; // elements per channel + const int64_t gs = npf*(c1 - c0); // elements in this group + + device const char * x03 = src0 + i03*args.nb03 + c0*args.nb02; + device char * y03 = dst + i03*args.nb3 + c0*args.nb2; float tmp = 0.0f; // partial sum for thread in warp - for (int j = start; j < end; j += ntg) { - tmp += src0[j]; + if (FC_group_norm_flat) { + device const float * x = (device const float *) x03; + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + tmp += x[e]; + } + } else { + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + const int64_t i02 = c0 + e/npf; + const int64_t ir = e - (i02 - c0)*npf; + const int64_t i01 = ir/ne00; + const int64_t i00 = ir - i01*ne00; + + tmp += *(device const float *)(x03 + (i02 - c0)*args.nb02 + i01*args.nb01 + i00*sizeof(float)); + } } threadgroup_barrier(mem_flags::mem_threadgroup); tmp = simd_sum(tmp); - if (ntg > N_SIMDWIDTH) { + if (ntg.x > N_SIMDWIDTH) { if (sgitg == 0) { buf[tiisg] = 0.0f; } @@ -286,14 +308,31 @@ kernel void kernel_group_norm_f32( const float mean = tmp / gs; tmp = 0.0f; - for (int j = start; j < end; j += ntg) { - float xi = src0[j] - mean; - dst[j] = xi; - tmp += xi * xi; + if (FC_group_norm_flat) { + device const float * x = (device const float *) x03; + device float * y = (device float *) y03; + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + const float xi = x[e] - mean; + y[e] = xi; + tmp += xi*xi; + } + } else { + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + const int64_t i02 = c0 + e/npf; + const int64_t ir = e - (i02 - c0)*npf; + const int64_t i01 = ir/ne00; + const int64_t i00 = ir - i01*ne00; + + const float xi = *(device const float *)(x03 + (i02 - c0)*args.nb02 + i01*args.nb01 + i00*sizeof(float)) - mean; + + *(device float *)(y03 + (i02 - c0)*args.nb2 + i01*args.nb1 + i00*sizeof(float)) = xi; + + tmp += xi*xi; + } } tmp = simd_sum(tmp); - if (ntg > N_SIMDWIDTH) { + if (ntg.x > N_SIMDWIDTH) { if (sgitg == 0) { buf[tiisg] = 0.0f; } @@ -312,7 +351,20 @@ kernel void kernel_group_norm_f32( const float variance = tmp / gs; const float scale = 1.0f/sqrt(variance + args.eps); - for (int j = start; j < end; j += ntg) { - dst[j] *= scale; + + if (FC_group_norm_flat) { + device float * y = (device float *) y03; + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + y[e] *= scale; + } + } else { + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + const int64_t i02 = c0 + e/npf; + const int64_t ir = e - (i02 - c0)*npf; + const int64_t i01 = ir/ne00; + const int64_t i00 = ir - i01*ne00; + + *(device float *)(y03 + (i02 - c0)*args.nb2 + i01*args.nb1 + i00*sizeof(float)) *= scale; + } } } diff --git a/ggml/src/ggml-metal/kernels/pool.metal b/ggml/src/ggml-metal/kernels/pool.metal index 13d355b9d..3dc0261c3 100644 --- a/ggml/src/ggml-metal/kernels/pool.metal +++ b/ggml/src/ggml-metal/kernels/pool.metal @@ -84,65 +84,117 @@ kernel void kernel_pool_1d_max_f32( constant ggml_metal_kargs_pool_1d & args, device const float * src, device float * dst, - uint gid [[thread_position_in_grid]] -) { - - if (gid >= args.np) { + threadgroup char * shmem [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint ntg[[threads_per_threadgroup]]) { + if (tgpig >= args.np) { return; } - const int ow = (int)gid % args.OW; - const int row = (int)gid / args.OW; + const int out = (int) tgpig; + const int ow = out % (int) args.OW; + const int row = out / (int) args.OW; + + const int base = ow * args.s0 - args.p0; + const int src_off = row * args.IW; + const int dst_off = row * args.OW; + + float lmax = -INFINITY; + + for (int ki = tpitg; ki < args.k0; ki += ntg) { + const int j = base + ki; + if (j >= 0 && j < args.IW) { + lmax = max(lmax, src[src_off + j]); + } + } - const int base = ow * args.s0 - args.p0; + float max_val = simd_max(lmax); - float acc = -INFINITY; + threadgroup float * shared_max = (threadgroup float *) shmem; - const int src_off = row * args.IW; - const int dst_off = row * args.OW; + if (ntg > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_max[tiisg] = -INFINITY; + } + threadgroup_barrier(mem_flags::mem_threadgroup); - for (int ki = 0; ki < args.k0; ++ki) { - int j = base + ki; - if (j < 0 || j >= args.IW){ - continue; + if (tiisg == 0) { + shared_max[sgitg] = max_val; } - float v = src[src_off + j]; - acc = max(acc, v); + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = shared_max[tiisg]; + max_val = simd_max(max_val); } - dst[dst_off + ow] = acc; + if (tpitg == 0) { + dst[dst_off + ow] = max_val; + } } kernel void kernel_pool_1d_avg_f32( constant ggml_metal_kargs_pool_1d & args, device const float * src, device float * dst, - uint gid [[thread_position_in_grid]] -) { - - if (gid >= args.np) { + threadgroup char * shmem [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint ntg[[threads_per_threadgroup]]) { + if (tgpig >= args.np) { return; } - const int ow = (int)gid % args.OW; - const int row = (int)gid / args.OW; - - const int base = ow * args.s0 - args.p0; + const int out = (int) tgpig; + const int ow = out % (int) args.OW; + const int row = out / (int) args.OW; - float acc = 0.0f; - int cnt = 0; + const int base = ow * args.s0 - args.p0; + const int src_off = row * args.IW; + const int dst_off = row * args.OW; - const int src_off = row * args.IW; - const int dst_off = row * args.OW; + float lsum = 0.0f; + int lcnt = 0; - for (int ki = 0; ki < args.k0; ++ki) { + for (int ki = tpitg; ki < args.k0; ki += ntg) { const int j = base + ki; - if (j < 0 || j >= args.IW) { - continue; + if (j >= 0 && j < args.IW) { + lsum += src[src_off + j]; + lcnt += 1; + } + } + + float sum = simd_sum(lsum); + float cnt = simd_sum((float) lcnt); + + threadgroup float * shared_sum = (threadgroup float *) shmem; + threadgroup float * shared_cnt = (threadgroup float *) shmem + N_SIMDWIDTH; + + if (ntg > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_sum[tiisg] = 0.0f; + shared_cnt[tiisg] = 0.0f; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_sum[sgitg] = sum; + shared_cnt[sgitg] = cnt; } - acc += src[src_off + j]; - cnt += 1; + threadgroup_barrier(mem_flags::mem_threadgroup); + + sum = shared_sum[tiisg]; + cnt = shared_cnt[tiisg]; + + sum = simd_sum(sum); + cnt = simd_sum(cnt); } - dst[dst_off + ow] = (cnt > 0) ? (acc / (float)cnt) : 0.0f; + if (tpitg == 0) { + dst[dst_off + ow] = (cnt > 0.0f) ? (sum / cnt) : 0.0f; + } } diff --git a/ggml/src/ggml-metal/kernels/quantize.h b/ggml/src/ggml-metal/kernels/quantize.h index 0741b2225..d3ba8884a 100644 --- a/ggml/src/ggml-metal/kernels/quantize.h +++ b/ggml/src/ggml-metal/kernels/quantize.h @@ -20,6 +20,7 @@ void quantize_q1_0(device const float * src, device block_q1_0 & dst) { } void quantize_q2_0(device const float * src, device block_q2_0 & dst) { +#pragma METAL fp math_mode(safe) float amax = 0.0f; for (int j = 0; j < QK2_0; j++) { float a = fabs(src[j]); diff --git a/ggml/src/ggml-metal/kernels/quantize.metal b/ggml/src/ggml-metal/kernels/quantize.metal index 73679b9ab..663e47d5f 100644 --- a/ggml/src/ggml-metal/kernels/quantize.metal +++ b/ggml/src/ggml-metal/kernels/quantize.metal @@ -26,11 +26,11 @@ kernel void kernel_cpy_t_t( const int32_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0; const int32_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0); - device T1 * dst_data = (device T1 *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + device char * dst_data = dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0; for (int32_t i00 = iw0*ntg[0] + tpitg.x; i00 < args.ne00;) { device const T0 * src = (device T0 *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00); - dst_data[i00] = (T1) src[0]; + *(device T1 *)(dst_data + i00*args.nb0) = (T1) src[0]; break; } } diff --git a/ggml/src/ggml-metal/kernels/solve_tri.metal b/ggml/src/ggml-metal/kernels/solve_tri.metal index 50f16facb..216f605fe 100644 --- a/ggml/src/ggml-metal/kernels/solve_tri.metal +++ b/ggml/src/ggml-metal/kernels/solve_tri.metal @@ -39,7 +39,9 @@ kernel void kernel_solve_tri_f32( for (short t = 0; t*NW < N; ++t) { const short idx = t*NW + tiisg; - sh0_cur[idx] = src0_ptr[idx]; + if (rr + sgitg < N && idx < N) { + sh0_cur[idx] = src0_ptr[idx]; + } } src0_ptr += NSG*N; @@ -60,7 +62,9 @@ kernel void kernel_solve_tri_f32( for (short t = 0; t*NW < r; ++t) { const short idx = t*NW + tiisg; - sum += sh0_cur[idx] * dst_ptr[idx*K] * (idx < r); + if (idx < r) { + sum += sh0_cur[idx] * dst_ptr[idx*K]; + } } sum = simd_sum(sum); diff --git a/ggml/src/ggml-metal/kernels/tri.metal b/ggml/src/ggml-metal/kernels/tri.metal index 862f78678..05c8c4682 100644 --- a/ggml/src/ggml-metal/kernels/tri.metal +++ b/ggml/src/ggml-metal/kernels/tri.metal @@ -46,8 +46,7 @@ kernel void kernel_tri( // threadgroup, so this will loop once for each index that this thread is // responsible for for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) { - // Use the comparison as a mask for branchless - dst_row[i0] = static_cast<T>(_ggml_vec_tri_cmp<ttype>(i0, i1)) * src_row[i0]; + dst_row[i0] = _ggml_vec_tri_cmp<ttype>(i0, i1) ? src_row[i0] : static_cast<T>(0); } } diff --git a/ggml/src/ggml-metal/kernels/unary.metal b/ggml/src/ggml-metal/kernels/unary.metal index e50a64863..b361da996 100644 --- a/ggml/src/ggml-metal/kernels/unary.metal +++ b/ggml/src/ggml-metal/kernels/unary.metal @@ -3,6 +3,24 @@ constant short FC_unary_op [[function_constant(FC_UNARY + 0)]]; constant bool FC_unary_cnt[[function_constant(FC_UNARY + 1)]]; +template <typename T, typename U> +inline constexpr T ggml_metal_unary_cast(U x) { + return (T) x; +} + +#if defined(GGML_METAL_HAS_BF16) +// MSL has no C-style cast from float vectors to bfloat vectors +template <> +inline bfloat4 ggml_metal_unary_cast<bfloat4, float4>(float4 x) { + return bfloat4(bfloat(x[0]), bfloat(x[1]), bfloat(x[2]), bfloat(x[3])); +} + +template <> +inline bfloat4 ggml_metal_unary_cast<bfloat4, float>(float x) { + return bfloat4(bfloat(x), bfloat(x), bfloat(x), bfloat(x)); +} +#endif + template <typename T0, typename T, typename TC> kernel void kernel_unary_impl( constant ggml_metal_kargs_unary & args, @@ -20,7 +38,7 @@ kernel void kernel_unary_impl( int i0; if (FC_CNT) { - i0 = tgpig.x; + i0 = tgpig.x*ntg.x + tpitg.x; src0_ptr = (device const T0 *) (src0); dst_ptr = (device T *) (dst); @@ -43,84 +61,88 @@ kernel void kernel_unary_impl( if (i0 >= args.ne0) { return; } + } else { + if (i0 >= args.ne00*args.ne01*args.ne02*args.ne03) { + return; + } } const TC x = (TC) src0_ptr[i0]; if (FC_OP == OP_UNARY_NUM_SCALE) { - dst_ptr[i0] = (T) (args.scale * x + args.bias); + dst_ptr[i0] = ggml_metal_unary_cast<T>(args.scale * x + args.bias); } if (FC_OP == OP_UNARY_NUM_FILL) { - dst_ptr[i0] = (T) args.val; + dst_ptr[i0] = ggml_metal_unary_cast<T>(args.val); } if (FC_OP == OP_UNARY_NUM_CLAMP) { - dst_ptr[i0] = (T) clamp(x, args.min, args.max); + dst_ptr[i0] = ggml_metal_unary_cast<T>(clamp(x, args.min, args.max)); } if (FC_OP == OP_UNARY_NUM_SQR) { - dst_ptr[i0] = (T) (x * x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(x * x); } if (FC_OP == OP_UNARY_NUM_SQRT) { - dst_ptr[i0] = (T) sqrt(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(sqrt(x)); } if (FC_OP == OP_UNARY_NUM_SIN) { - dst_ptr[i0] = (T) sin(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(sin(x)); } if (FC_OP == OP_UNARY_NUM_COS) { - dst_ptr[i0] = (T) cos(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(cos(x)); } if (FC_OP == OP_UNARY_NUM_LOG) { - dst_ptr[i0] = (T) log(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(log(x)); } if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) { - dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope)); + dst_ptr[i0] = ggml_metal_unary_cast<T>(TC(x > 0)*x + TC(x <= 0)*(x * args.slope)); } if (FC_OP == OP_UNARY_NUM_TANH) { - dst_ptr[i0] = (T) precise::tanh(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(precise::tanh(x)); } if (FC_OP == OP_UNARY_NUM_RELU) { - dst_ptr[i0] = (T) fmax(0, x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, x)); } if (FC_OP == OP_UNARY_NUM_SIGMOID) { - dst_ptr[i0] = (T) (1 / (1 + exp(-x))); + dst_ptr[i0] = ggml_metal_unary_cast<T>(1 / (1 + exp(-x))); } if (FC_OP == OP_UNARY_NUM_GELU) { - dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))); + dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))); } if (FC_OP == OP_UNARY_NUM_GELU_ERF) { - dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x))); + dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + erf_approx(SQRT_2_INV*x))); } if (FC_OP == OP_UNARY_NUM_GELU_QUICK) { - dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x)))); + dst_ptr[i0] = ggml_metal_unary_cast<T>(x * (1/(1 + exp(GELU_QUICK_COEF*x)))); } if (FC_OP == OP_UNARY_NUM_SILU) { - dst_ptr[i0] = (T) (x / (1 + exp(-x))); + dst_ptr[i0] = ggml_metal_unary_cast<T>(x / (1 + exp(-x))); } if (FC_OP == OP_UNARY_NUM_ELU) { - dst_ptr[i0] = (T) elu_approx(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(elu_approx(x)); } if (FC_OP == OP_UNARY_NUM_NEG) { - dst_ptr[i0] = (T) -x; + dst_ptr[i0] = ggml_metal_unary_cast<T>(-x); } if (FC_OP == OP_UNARY_NUM_ABS) { - dst_ptr[i0] = (T) fabs(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(fabs(x)); } if (FC_OP == OP_UNARY_NUM_SGN) { @@ -132,40 +154,40 @@ kernel void kernel_unary_impl( } if (FC_OP == OP_UNARY_NUM_HARDSWISH) { - dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5))); + dst_ptr[i0] = ggml_metal_unary_cast<T>(x * fmax(0, fmin(1, x/6 + 0.5))); } if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) { - dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5)); + dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, fmin(1, x/6 + 0.5))); } if (FC_OP == OP_UNARY_NUM_EXP) { - dst_ptr[i0] = (T) exp(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x)); } if (FC_OP == OP_UNARY_NUM_SOFTPLUS) { - dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20); + dst_ptr[i0] = ggml_metal_unary_cast<T>(select(log(1 + exp(x)), x, x > 20)); } if (FC_OP == OP_UNARY_NUM_EXPM1) { // TODO: precise implementation - dst_ptr[i0] = (T) (exp(x) - 1); + dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x) - 1); } if (FC_OP == OP_UNARY_NUM_FLOOR) { - dst_ptr[i0] = (T) floor(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(floor(x)); } if (FC_OP == OP_UNARY_NUM_CEIL) { - dst_ptr[i0] = (T) ceil(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(ceil(x)); } if (FC_OP == OP_UNARY_NUM_ROUND) { - dst_ptr[i0] = (T) round(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(round(x)); } if (FC_OP == OP_UNARY_NUM_TRUNC) { - dst_ptr[i0] = (T) trunc(x); + dst_ptr[i0] = ggml_metal_unary_cast<T>(trunc(x)); } if (FC_OP == OP_UNARY_NUM_XIELU) { @@ -174,7 +196,7 @@ kernel void kernel_unary_impl( const TC clamped = fmin(xi, TC(args.val)); const TC y_pos = TC(args.scale) * xi * xi + TC(args.bias) * xi; const TC y_neg = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi; - dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg); + dst_ptr[i0] = ggml_metal_unary_cast<T>(gate * y_pos + (TC(1.0f) - gate) * y_neg); } } @@ -188,6 +210,10 @@ template [[host_name("kernel_unary_f32_f32")]] kernel kernel_unary_t kernel_un template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl<float4, float4, float4>; template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl<half, half, float>; template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4, half4, float4>; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_unary_bf16_bf16")]] kernel kernel_unary_t kernel_unary_impl<bfloat, bfloat, float>; +template [[host_name("kernel_unary_bf16_bf16_4")]] kernel kernel_unary_t kernel_unary_impl<bfloat4, bfloat4, float4>; +#endif kernel void kernel_silu_back_f32( constant ggml_metal_kargs_silu_back & args, @@ -209,14 +235,15 @@ kernel void kernel_reglu( device const char * src0, device const char * src1, device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + uint2 tgpig[[threadgroup_position_in_grid]], + uint2 tpitg[[thread_position_in_threadgroup]], + uint2 ntg[[threads_per_threadgroup]], + uint2 ntgs[[threadgroups_per_grid]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1); + + for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) { const float x0 = src0_row[i0]; const float x1 = src1_row[i0]; @@ -235,14 +262,15 @@ kernel void kernel_geglu( device const char * src0, device const char * src1, device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + uint2 tgpig[[threadgroup_position_in_grid]], + uint2 tpitg[[thread_position_in_threadgroup]], + uint2 ntg[[threads_per_threadgroup]], + uint2 ntgs[[threadgroups_per_grid]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1); + + for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) { const float x0 = src0_row[i0]; const float x1 = src1_row[i0]; @@ -263,14 +291,15 @@ kernel void kernel_swiglu( device const char * src0, device const char * src1, device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + uint2 tgpig[[threadgroup_position_in_grid]], + uint2 tpitg[[thread_position_in_threadgroup]], + uint2 ntg[[threads_per_threadgroup]], + uint2 ntgs[[threadgroups_per_grid]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1); + + for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) { const float x0 = src0_row[i0]; const float x1 = src1_row[i0]; @@ -291,14 +320,15 @@ kernel void kernel_swiglu_oai( device const char * src0, device const char * src1, device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + uint2 tgpig[[threadgroup_position_in_grid]], + uint2 tpitg[[thread_position_in_threadgroup]], + uint2 ntg[[threads_per_threadgroup]], + uint2 ntgs[[threadgroups_per_grid]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1); + + for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) { float x0 = src0_row[i0]; float x1 = src1_row[i0]; @@ -323,14 +353,15 @@ kernel void kernel_swiglu_clamp( device const char * src0, device const char * src1, device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + uint2 tgpig[[threadgroup_position_in_grid]], + uint2 tpitg[[thread_position_in_threadgroup]], + uint2 ntg[[threads_per_threadgroup]], + uint2 ntgs[[threadgroups_per_grid]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1); + + for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) { const float gate = min((float) src0_row[i0], args.limit); const float up = clamp((float) src1_row[i0], -args.limit, args.limit); @@ -349,14 +380,15 @@ kernel void kernel_geglu_erf( device const char * src0, device const char * src1, device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + uint2 tgpig[[threadgroup_position_in_grid]], + uint2 tpitg[[thread_position_in_threadgroup]], + uint2 ntg[[threads_per_threadgroup]], + uint2 ntgs[[threadgroups_per_grid]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1); + + for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) { const float x0 = src0_row[i0]; const float x1 = src1_row[i0]; @@ -377,14 +409,15 @@ kernel void kernel_geglu_quick( device const char * src0, device const char * src1, device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + uint2 tgpig[[threadgroup_position_in_grid]], + uint2 tpitg[[thread_position_in_threadgroup]], + uint2 ntg[[threads_per_threadgroup]], + uint2 ntgs[[threadgroups_per_grid]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1); + + for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) { const float x0 = src0_row[i0]; const float x1 = src1_row[i0]; diff --git a/ggml/src/ggml-musa/CMakeLists.txt b/ggml/src/ggml-musa/CMakeLists.txt index 6cb178ebc..f451b0c20 100644 --- a/ggml/src/ggml-musa/CMakeLists.txt +++ b/ggml/src/ggml-musa/CMakeLists.txt @@ -21,7 +21,10 @@ if (MUSAToolkit_FOUND) message(STATUS "MUSA Toolkit found") if (NOT DEFINED MUSA_ARCHITECTURES) - set(MUSA_ARCHITECTURES "21;22;31") + set(MUSA_ARCHITECTURES "22;31") + endif() + if ("21" IN_LIST MUSA_ARCHITECTURES) + message(WARNING "MUSA architecture 21 (MTT S70, MTT S80, MTT S3000) is deprecated and no longer tested") endif() message(STATUS "Using MUSA architectures: ${MUSA_ARCHITECTURES}") diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 97b862aff..cdf4a5d54 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -64,15 +64,21 @@ set(GGML_OPENCL_KERNELS add add_id moe_add_id_glu + argmax argsort + top_k + top_k_mask + lightning_indexer tri fill + arange + fwht clamp cpy cvt - diag_mask_inf diag div + dsv4_hc gelu get_rows glu @@ -80,6 +86,8 @@ set(GGML_OPENCL_KERNELS solve_tri im2col_f32 im2col_f16 + im2col_3d_f32 + im2col_3d_f16 mean mul_mv_f16_f16 mul_mv_f16_f32_1row @@ -96,6 +104,79 @@ set(GGML_OPENCL_KERNELS mul_mv_q4_0_f32_1d_16x_flat mul_mv_q4_1_f32 mul_mv_q4_1_f32_flat + gemm_noshuffle_q2_k_q8_1_dp4a + gemv_noshuffle_q2_k_f32 + gemm_noshuffle_q2_k_f32 + gemm_noshuffle_q3_k_f32 + gemm_noshuffle_iq2_s_f32 + gemm_noshuffle_iq2_xs_f32 + gemm_noshuffle_iq2_xxs_f32 + gemm_noshuffle_iq2_xxs_q8_1_dp4a + gemv_noshuffle_iq2_xxs_f32 + gemm_noshuffle_iq2_xs_q8_1_dp4a + gemv_noshuffle_iq2_xs_f32 + gemm_noshuffle_iq2_s_q8_1_dp4a + gemv_noshuffle_iq2_s_f32 + gemm_noshuffle_q2_0_f32 + gemm_noshuffle_q2_0_q8_1_dp4a + gemv_noshuffle_q2_0_f32 + gemm_noshuffle_iq1_m_f32 + gemm_noshuffle_iq1_m_q8_1_dp4a + gemv_noshuffle_iq1_m_f32 + gemm_noshuffle_iq1_s_f32 + gemm_noshuffle_iq1_s_q8_1_dp4a + gemv_noshuffle_iq1_s_f32 + gemm_noshuffle_iq3_s_f32 + gemm_noshuffle_iq3_s_q8_1_dp4a + gemv_noshuffle_iq3_s_f32 + gemm_noshuffle_iq3_xxs_f32 + gemm_noshuffle_iq3_xxs_q8_1_dp4a + gemv_noshuffle_iq3_xxs_f32 + gemm_noshuffle_iq4_xs_f32 + gemm_noshuffle_iq4_xs_q8_1_dp4a + gemv_noshuffle_iq4_xs_f32 + gemm_noshuffle_q3_k_q8_1_dp4a + gemv_noshuffle_q3_k_f32 + mul_mv_q2_k_f32 + mul_mv_q3_k_f32 + iq2_s_grid + mul_mv_iq2_s_f32 + iq2_xs_grid + mul_mv_iq2_xs_f32 + iq2_xxs_grid + mul_mv_iq2_xxs_f32 + mul_mv_iq2_xxs_f32_flat + mul_mm_iq2_xxs_f32_l4_lm + mul_mv_iq2_xs_f32_flat + mul_mm_iq2_xs_f32_l4_lm + mul_mv_iq2_s_f32_flat + mul_mm_iq2_s_f32_l4_lm + mul_mv_q2_0_f32 + mul_mv_q2_0_f32_flat + mul_mm_q2_0_f32_l4_lm + mul_mv_iq1_m_f32 + mul_mv_iq1_m_f32_flat + mul_mm_iq1_m_f32_l4_lm + iq1_s_grid + mul_mv_iq1_s_f32 + mul_mv_iq1_s_f32_flat + mul_mm_iq1_s_f32_l4_lm + iq3_xxs_grid + mul_mv_iq3_xxs_f32 + mul_mv_iq3_xxs_f32_flat + mul_mm_iq3_xxs_f32_l4_lm + iq3_s_grid + mul_mv_iq3_s_f32 + mul_mv_iq3_s_f32_flat + mul_mm_iq3_s_f32_l4_lm + iq4_xs_grid + mul_mv_iq4_xs_f32 + mul_mv_iq4_xs_f32_flat + mul_mm_iq4_xs_f32_l4_lm + mul_mv_q3_k_f32_flat + mul_mm_q3_k_f32_l4_lm + mul_mv_q2_k_f32_flat + mul_mm_q2_k_f32_l4_lm mul_mv_q4_k_f32 mul_mv_q4_k_f32_flat mul_mv_q5_0_f32 @@ -113,10 +194,13 @@ set(GGML_OPENCL_KERNELS mul_mv_mxfp4_f32 mul_mv_mxfp4_f32_flat mul_mv_id_q4_0_f32_8x_flat + mul_mv_id_f32_f32 + mul_mm_id_f32_f32 mul_mv_id_q8_0_f32 mul_mv_id_q8_0_f32_flat mul_mv_id_mxfp4_f32 mul_mv_id_mxfp4_f32_flat + mul_mat_id_utils gemm_moe_q4_0_f32_ns gemm_moe_q4_0_q8_1_dp4a gemv_moe_q4_0_f32_ns @@ -132,6 +216,7 @@ set(GGML_OPENCL_KERNELS gemm_moe_q6_k_q8_1_dp4a gemm_moe_q8_1_dp4a moe_reorder_quant_a_q8_1 + transpose_activation_q8 gemm_noshuffle_q4_k_q8_1_dp4a gemm_noshuffle_q5_k_q8_1_dp4a gemm_noshuffle_q6_k_q8_1_dp4a @@ -139,6 +224,9 @@ set(GGML_OPENCL_KERNELS gemm_noshuffle_q5_0_q8_1_dp4a gemm_noshuffle_iq4_nl_q8_1_dp4a gemm_noshuffle_q4_0_q8_1_dp4a + gemm_coopk_q4_0_q8_1_dp4a + gemm_coopk_q4_k_q8_1_dp4a + gemm_coopk_q6_k_q8_1_dp4a quant_a_q8_1 gemv_moe_q4_k_f32_ns gemm_moe_q5_k_f32_ns @@ -184,6 +272,7 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q4_k_f32 gemv_noshuffle_q4_k_f32_o4 gemv_noshuffle_q4_k_f32_tiled + gemm_noshuffle_q4_K_f32_tiled_dp4a gemm_noshuffle_q4_k_f32 gemv_noshuffle_q4_k_f32_32b_trans gemv_noshuffle_q6_k_f32 @@ -232,10 +321,14 @@ set(GGML_OPENCL_KERNELS repeat mul_mat_f16_f32 mul_mm_f16_f32_kq_kqv + mul_mm_f16_f32_kq_p8 + mul_mm_q8_f32_kqv conv2d conv2d_f16_f32 + conv2_dw flash_attn_repack flash_attn_pre_f16 + flash_attn_pv_direct flash_attn_f32_f16 flash_attn_f32_q8_0 flash_attn_f32_q4_0 diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 4f1f937bb..7fa1a8e35 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -36,6 +36,7 @@ typedef const void * (*get_adreno_bin_kernel_func_t)( #include <inttypes.h> #include <string.h> +#include <climits> #include <cstddef> #include <cstdint> #include <fstream> @@ -48,6 +49,7 @@ typedef const void * (*get_adreno_bin_kernel_func_t)( #include <mutex> #include <regex> #include <set> +#include <tuple> #include <unordered_set> #undef MIN @@ -77,6 +79,11 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor); static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor); static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +static void ggml_cl_fa_decompose_soft_max(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, cl_mem sums); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool is_kq); +#endif // See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1. // Precompute mp (m' in the paper) and L such that division @@ -280,7 +287,8 @@ static ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) { strstr(device_name, "613") || strstr(device_name, "615") || strstr(device_name, "616") || strstr(device_name, "618") || strstr(device_name, "619") || strstr(device_name, "620") || - strstr(device_name, "630") || strstr(device_name, "640") || + strstr(device_name, "623") || strstr(device_name, "630") || + strstr(device_name, "640") || strstr(device_name, "642") || strstr(device_name, "643") || strstr(device_name, "644") || strstr(device_name, "650") || strstr(device_name, "660") || strstr(device_name, "663") || @@ -493,6 +501,13 @@ struct ggml_opencl_fa_kernels { std::map<std::pair<int, int>, cl_kernel> f32_f16_q1_split; // flash-decoding K-split // vec decode std::map<std::pair<int, int>, cl_kernel> f32_f16_q1_vec; + bool f32_f16_vec_512_attempted = false; + std::map<std::tuple<int, int, int>, cl_kernel> f32_f16_mq_decode; + std::map<std::tuple<int, int, int>, size_t> f32_f16_mq_decode_wg; + std::map<std::tuple<int, int, int>, int> f32_f16_mq_decode_hs; + std::set<std::tuple<int, int, int>> f32_f16_mq_decode_attempted; + ggml_cl_buffer fd_partial; + cl_uint compute_units = 0; // kv-head-coalesced vec decode std::map<std::pair<int, int>, cl_kernel> f32_f16_q1_vec_mq; // kv-head-coalesced + flash-decoding split @@ -561,6 +576,8 @@ struct ggml_opencl_fa_kernels { std::map<std::pair<int, int>, cl_kernel> kv_pad_f16; std::map<std::pair<int, int>, cl_kernel> mask_pad_f16; std::map<std::pair<int, int>, cl_kernel> blk_f16; + // decomposed prefill: V^T, built with the prepass + std::map<std::pair<int, int>, cl_kernel> v_transpose_f16; // generic prefill tile dims (f16 / f32 paths) std::map<std::pair<int, int>, int> bm; std::map<std::pair<int, int>, int> bn; @@ -637,8 +654,8 @@ struct ggml_backend_opencl_context { ggml_cl_version platform_version; ggml_cl_version opencl_c_version; - // argsort is loaded in supports_op because its availability depends on how - // many workgroups are allowed, which requires kernel compilation. + // argmax and argsort are loaded in supports_op; argsort availability depends on the compiled kernel's workgroup limit. + bool kernels_loaded_argmax = false; bool kernels_loaded_argsort = false; // rest of the kernels are currently always loaded in alloc_buffer. bool kernels_loaded = false; @@ -652,8 +669,10 @@ struct ggml_backend_opencl_context { size_t global_mem_size; size_t max_alloc_size; size_t max_workgroup_size; + cl_ulong local_mem_size; bool fp16_support; bool has_vector_subgroup_broadcast; + bool has_khr_subgroup_shuffle = false; bool has_subgroup_shuffle = false; // cl_khr_subgroup_shuffle or cl_qcom_subgroup_shuffle bool has_integer_dot = false; // cl_khr_integer_dot_product or cl_qcom_dot_product8 bool has_qcom_subgroup_shuffle = false; // specifically cl_qcom_subgroup_shuffle @@ -711,6 +730,10 @@ struct ggml_backend_opencl_context { ggml_cl_buffer prealloc_moe_qa; // int8 quants [tok_slots * ne00] ggml_cl_buffer prealloc_moe_da; // per-block d [tok_slots * ne00/32] (half) ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half) + // per-tile transposed qa/da and on-chip scratch for the ldck dp4a MoE GEMM + ggml_cl_buffer prealloc_moe_qa_trans; // int8 quants [tok_slots * ne00] + ggml_cl_buffer prealloc_moe_da_trans; // per-block d [tok_slots * ne00/32] (float) + ggml_cl_buffer prealloc_moe_qda_on_chip; // scratch copy of the router weights to avoid dst aliasing ggml_cl_buffer prealloc_moe_combine_w; ggml_cl_buffer prealloc_splitk_partial; // [ksplit * M] partials for split-K GEMV @@ -742,6 +765,19 @@ struct ggml_backend_opencl_context { // prealloc buffers for src0 and src1 ggml_cl_buffer prealloc_src0; ggml_cl_buffer prealloc_src1; + ggml_cl_buffer prealloc_top_k; + + // scratch for the decomposed flash attention prefill, sized per query chunk + ggml_cl_buffer prealloc_fa_vt; + ggml_cl_buffer prealloc_fa_kq; + ggml_cl_buffer prealloc_fa_kqv; + ggml_cl_buffer prealloc_fa_sums; + ggml_cl_buffer prealloc_fa_vtq; + ggml_cl_buffer prealloc_fa_vtd; + ggml_cl_buffer prealloc_fa_pq; + ggml_cl_buffer prealloc_fa_pd; + ggml_cl_buffer prealloc_fa_bmax; + ggml_cl_buffer prealloc_fa_bsum; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS ggml_cl_buffer prealloc_adreno_xmem_const; @@ -762,7 +798,6 @@ struct ggml_backend_opencl_context { cl_program program_moe_add_id_glu; cl_program program_clamp; cl_program program_cvt; - cl_program program_diag_mask_inf; cl_program program_gelu; cl_program program_gemv_noshuffle_general; cl_program program_gemv_noshuffle; @@ -818,6 +853,7 @@ struct ggml_backend_opencl_context { cl_program program_mul_mv_id_mxfp4_f32_flat; cl_program program_mul_mm_f32_f32_l4_lm; cl_program program_mul_mm_f16_f32_l4_lm; + cl_program program_mul_mm_f16_f32_l4_lm_n8; cl_program program_mul_mm_q8_0_f32_l4_lm; cl_kernel kernel_add, kernel_add_row, kernel_add_f16, kernel_add_row_f16; @@ -836,9 +872,11 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gelu_quick, kernel_gelu_quick_4; cl_kernel kernel_relu; cl_kernel kernel_sigmoid_f32, kernel_sigmoid_f16; + cl_kernel kernel_sigmoid_f32_nc, kernel_sigmoid_f16_nc; cl_kernel kernel_tri; cl_kernel kernel_fill; - cl_kernel kernel_clamp; + cl_kernel kernel_arange; + cl_kernel kernel_clamp, kernel_clamp_nc; cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_swiglu_clamp, kernel_geglu_erf, kernel_geglu_quick, kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_swiglu_clamp_f16, kernel_geglu_erf_f16, kernel_geglu_quick_f16; @@ -847,15 +885,24 @@ struct ggml_backend_opencl_context { cl_kernel kernel_rms_norm_mul_add = nullptr; // fused rms_norm(x)*w + b (residual) cl_kernel kernel_l2_norm_f32; cl_kernel kernel_group_norm, kernel_group_norm_mul_add; - cl_kernel kernel_diag_mask_inf, kernel_diag_mask_inf_8; cl_kernel kernel_diag_f32; cl_kernel kernel_soft_max, kernel_soft_max_4; cl_kernel kernel_soft_max_f16, kernel_soft_max_4_f16; + cl_kernel kernel_fa_decompose_soft_max_4_f16, kernel_fa_decompose_scale_rows_f32; + cl_kernel kernel_mul_mm_f16_f32_kq_p8 = nullptr; + cl_kernel kernel_mul_mm_q8_f32_kqv = nullptr; + cl_kernel kernel_fa_p8_fixup = nullptr; + cl_kernel kernel_fa_kqv_direct_f16 = nullptr; + cl_kernel kernel_fa_kqv_direct_reduce = nullptr; + cl_kernel kernel_fa_v_transpose_q8 = nullptr; ggml_opencl_fa_kernels fa; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS ggml_cl_adreno_xmem_attn_state adreno_xmem_attn; #endif - cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0; + cl_kernel kernel_top_k_f32, kernel_top_k_merge_f32; + cl_kernel kernel_top_k_mask[2], kernel_top_k_mask_fill[2]; + cl_kernel kernel_lightning_indexer; + cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_i32, kernel_get_rows_q4_0; cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32; cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32; cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32; @@ -863,9 +910,13 @@ struct ggml_backend_opencl_context { cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32; cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16; cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16; - cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_f32_f32_pack, kernel_cpy_i32_i32; + cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32, kernel_cpy_i32_f32, kernel_cpy_f32_i32; + cl_kernel kernel_cpy_f32_f32_pack = nullptr; cl_kernel kernel_cpy_f32_f32_flat = nullptr; cl_kernel kernel_mul_mat_f32_f32; + cl_kernel kernel_mul_mv_f32_f32_r2 = nullptr; + cl_kernel kernel_fwht[2] = {}; + size_t fwht_workgroup_size[2] = {}; cl_kernel kernel_mul_mat_f16_f16; cl_kernel kernel_mul_mat_f16_f32_1row; cl_program program_mul_mv_f16_f32_mrow; @@ -944,6 +995,126 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mv_q5_0_f32_flat; cl_kernel kernel_mul_mv_q5_1_f32; cl_kernel kernel_mul_mv_q5_1_f32_flat; + cl_kernel kernel_convert_block_q2_K, kernel_restore_block_q2_K; + cl_kernel kernel_mul_mv_q2_K_f32_flat; + cl_kernel kernel_mul_mm_q2_k_f32_l4_lm; + cl_kernel kernel_convert_block_q2_k_noshuffle; + cl_kernel kernel_restore_block_q2_k_noshuffle; + cl_kernel kernel_gemv_noshuffle_q2_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q2_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q2_k_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow = nullptr; + int q2_k_mv_rows = 4; + int q2_k_mv_nsg = 4; + cl_kernel kernel_mul_mv_q2_K_f32; + cl_kernel kernel_convert_block_q3_K, kernel_restore_block_q3_K; + cl_kernel kernel_mul_mv_q3_K_f32_flat; + cl_kernel kernel_mul_mm_q3_k_f32_l4_lm; + cl_kernel kernel_convert_block_q3_k_noshuffle, kernel_restore_block_q3_k_noshuffle; + cl_kernel kernel_gemv_noshuffle_q3_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = nullptr; + int q3_k_mv_rows = 2; + int q3_k_mv_nsg = 8; + cl_kernel kernel_mul_mv_q3_K_f32; + cl_kernel kernel_convert_block_iq2_s, kernel_restore_block_iq2_s; + cl_kernel kernel_mul_mv_iq2_s_f32_flat; + cl_kernel kernel_mul_mm_iq2_s_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq2_s_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_s_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_s_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_s_q8_1_dp4a_narrow = nullptr; + int iq2_s_mv_nsg = 8; + ggml_cl_buffer iq2_s_grid_buffer; + cl_mem iq2_s_grid_image = nullptr; + cl_kernel kernel_mul_mv_iq2_s_f32; + cl_kernel kernel_convert_block_iq2_xs, kernel_restore_block_iq2_xs; + cl_kernel kernel_mul_mv_iq2_xs_f32_flat; + cl_kernel kernel_mul_mm_iq2_xs_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq2_xs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_xs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a_narrow = nullptr; + int iq2_xs_mv_nsg = 4; + ggml_cl_buffer iq2_xs_grid_buffer; + cl_mem iq2_xs_grid_image = nullptr; + cl_kernel kernel_mul_mv_iq2_xs_f32; + cl_kernel kernel_convert_block_iq2_xxs, kernel_restore_block_iq2_xxs; + cl_kernel kernel_mul_mv_iq2_xxs_f32_flat; + cl_kernel kernel_mul_mm_iq2_xxs_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq2_xxs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_xxs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a_narrow = nullptr; + int iq2_xxs_mv_nsg = 4; + ggml_cl_buffer iq2_xxs_grid_buffer; + cl_mem iq2_xxs_grid_image = nullptr; + cl_kernel kernel_mul_mv_iq2_xxs_f32; + cl_kernel kernel_convert_block_q2_0, kernel_restore_block_q2_0; + cl_kernel kernel_mul_mv_q2_0_f32_flat; + cl_kernel kernel_mul_mm_q2_0_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_q2_0_f32 = nullptr; + cl_kernel kernel_gemv_noshuffle_q2_0_f32_wide = nullptr; + size_t q2_0_mv_workgroup_size = 512; + size_t q2_0_mv_workgroup_size_wide = 512; + cl_kernel kernel_gemm_noshuffle_q2_0_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q2_0_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_q2_0_q8_1_dp4a_narrow = nullptr; + int q2_0_mv_nsg = 32; + cl_kernel kernel_mul_mv_q2_0_f32; + cl_kernel kernel_convert_block_iq1_m, kernel_restore_block_iq1_m; + cl_kernel kernel_mul_mv_iq1_m_f32_flat; + cl_kernel kernel_mul_mm_iq1_m_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq1_m_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq1_m_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq1_m_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq1_m_q8_1_dp4a_narrow = nullptr; + int iq1_m_mv_nsg = 4; + cl_kernel kernel_mul_mv_iq1_m_f32; + cl_kernel kernel_convert_block_iq1_s, kernel_restore_block_iq1_s; + cl_kernel kernel_mul_mv_iq1_s_f32_flat; + cl_kernel kernel_mul_mm_iq1_s_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq1_s_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq1_s_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq1_s_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq1_s_q8_1_dp4a_narrow = nullptr; + int iq1_s_mv_nsg = 4; + ggml_cl_buffer iq1_s_grid_buffer; + cl_mem iq1_s_grid_image = nullptr; + cl_kernel kernel_mul_mv_iq1_s_f32; + cl_kernel kernel_convert_block_iq3_xxs, kernel_restore_block_iq3_xxs; + cl_kernel kernel_mul_mv_iq3_xxs_f32_flat; + cl_kernel kernel_mul_mm_iq3_xxs_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq3_xxs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_xxs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a_narrow = nullptr; + int iq3_xxs_mv_nsg = 4; + ggml_cl_buffer iq3_xxs_grid_buffer; + cl_mem iq3_xxs_grid_image = nullptr; + cl_kernel kernel_mul_mv_iq3_xxs_f32; + cl_kernel kernel_convert_block_iq3_s, kernel_restore_block_iq3_s; + cl_kernel kernel_mul_mv_iq3_s_f32_flat; + cl_kernel kernel_mul_mm_iq3_s_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq3_s_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_s_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_s_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_s_q8_1_dp4a_narrow = nullptr; + int iq3_s_mv_nsg = 4; + ggml_cl_buffer iq3_s_grid_buffer; + cl_mem iq3_s_grid_image = nullptr; + cl_kernel kernel_mul_mv_iq3_s_f32; + cl_kernel kernel_convert_block_iq4_xs, kernel_restore_block_iq4_xs; + cl_kernel kernel_convert_block_iq4_xs_noshuffle = nullptr, kernel_restore_block_iq4_xs_noshuffle = nullptr; + cl_kernel kernel_mul_mv_iq4_xs_f32_flat; + cl_kernel kernel_mul_mm_iq4_xs_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq4_xs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow = nullptr; + int iq4_xs_mv_nsg = 8; + cl_kernel kernel_mul_mv_iq4_xs_f32; cl_kernel kernel_mul_mv_q4_K_f32; cl_kernel kernel_mul_mv_q4_K_f32_flat; cl_kernel kernel_mul_mv_q5_K_f32; @@ -956,10 +1127,16 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mv_iq4_nl_f32_flat; cl_kernel kernel_solve_tri_f32; cl_kernel kernel_im2col_f32, kernel_im2col_f16; + cl_kernel kernel_im2col_3d_f32, kernel_im2col_3d_f16; + cl_kernel kernel_argmax_f32_i32; cl_kernel kernel_argsort_f32_i32; cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4; + cl_kernel kernel_dsv4_hc_pre = nullptr; + cl_kernel kernel_dsv4_hc_post[2] = {}; + size_t local_dsv4_hc_pre = 0; + size_t local_dsv4_hc_post[2] = {}; cl_kernel kernel_cumsum_blk, kernel_cumsum_add; - cl_kernel kernel_repeat_f32; + cl_kernel kernel_repeat_f32, kernel_repeat_f16, kernel_repeat_i32; cl_kernel kernel_pad; cl_kernel kernel_tanh_f32, kernel_tanh_f32_4, kernel_tanh_f32_nc; cl_kernel kernel_tanh_f16, kernel_tanh_f16_4, kernel_tanh_f16_nc; @@ -988,6 +1165,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_conv_2d_f16; cl_kernel kernel_conv_2d_f32; cl_kernel kernel_conv_2d_f16_f32; + cl_kernel kernel_conv2d_dw_f32_f32, kernel_conv2d_dwf16_f32; cl_kernel kernel_ssm_conv_f32_f32, kernel_ssm_conv_f32_f32_4; // [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1. // tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4). @@ -1006,6 +1184,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in) cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) prefill GEMM variant cl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMM + cl_kernel kernel_transpose_activation_q8; // per-tile transpose of q8_1 activations for the ldck dp4a GEMM cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-in cl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16] cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr; // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in @@ -1021,12 +1200,18 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr; // dp4a (int8) mxfp4 MoE prefill GEMM cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMM cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMM + cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMM using ldck cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_0 MoE prefill GEMM + cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_k MoE prefill GEMM + cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q6_k MoE prefill GEMM cl_kernel kernel_moe_reorder_b; cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter; cl_kernel kernel_moe_scatter_stable = nullptr; // deterministic slot assignment + cl_kernel kernel_moe_zero_dst; cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum cl_kernel kernel_moe_combine_bias_f32 = nullptr; // same, with the down-projection bias add folded in + cl_kernel kernel_mul_mv_id_f32_f32; + cl_kernel kernel_mul_mm_id_f32_f32; cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat; cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat; cl_kernel kernel_mul_mv_id_mxfp4_f32; @@ -1034,6 +1219,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mm_f32_f32_l4_lm; cl_kernel kernel_gemv_f32_f32_mc; // multi-column (small-N) f32 GEMV for spec/MTP verify cl_kernel kernel_mul_mm_f16_f32_l4_lm; + cl_kernel kernel_mul_mm_f16_f32_l4_lm_n8; cl_kernel kernel_mul_mm_q1_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_1_f32_l4_lm; @@ -1212,6 +1398,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin; cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; @@ -1223,15 +1410,20 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q4_1_f32; cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin; cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in) + cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in) cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; cl_kernel kernel_gemv_noshuffle_q8_0_f32; cl_kernel kernel_gemv_noshuffle_q8_0_f32_splitk; // split-K across WGs (small-M decode) + cl_kernel kernel_gemv_noshuffle_q8_0_f32_mc; // multi-column (N 2..4) verify GEMV cl_kernel kernel_gemm_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q1_0_f32; + cl_kernel kernel_gemv_noshuffle_q1_0_f32_nsg1 = nullptr; + int min_m_gemv_nsg1_q1_0 = 0; cl_kernel kernel_gemv_noshuffle_q4_k_f32; cl_kernel kernel_gemv_noshuffle_q4_k_f32_o4; // 4-output-per-WI, long-vocab lm_head cl_kernel kernel_gemv_noshuffle_q4_k_f32_tiled; // tiled-wide layout (opt-in) + cl_kernel kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a = nullptr; cl_kernel kernel_gemv_noshuffle_q4_k_f32_splitk; // split-K across WGs (small-M decode) cl_kernel kernel_gemv_splitk_reduce_f32; // sums split-K per-slice partials cl_kernel kernel_gemv_noshuffle_q4_k_f32_glu; // fused gate+up GEMV + GLU (FFN) @@ -1242,9 +1434,17 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow = nullptr; + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg = nullptr; + int tile_n_dp4a_q4_k = 32; + int tile_n_dp4a_narrow_q4_k = 32; + int max_n_dp4a_narrow_q4_k = 16; cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass cl_kernel kernel_gemm_noshuffle_q4_k_f32_r1; cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg; @@ -1253,7 +1453,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4; cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4_global; // weights via __global (opt-in) cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled; // tiled-wide layout (opt-in) - cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled_mc3; // tiled multi-column (N=3) verify lm_head + cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled_mc3; // tiled multi-column (N 2..4) verify lm_head cl_kernel kernel_gemm_noshuffle_q6_K_f32_tiled; // batched (N>1) over the tiled layout cl_kernel kernel_convert_block_q6_k_tiled_ns; // tiled-wide convert (opt-in) cl_kernel kernel_gemv_noshuffle_q6_K_f32_mc3; // multi-column (N=3) verify GEMV @@ -1278,13 +1478,69 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_iq4_nl_f32; cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = nullptr; + cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr; + cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr; + int nsg_coopk_dp4a_q4_0 = 8; + int rows_coopk_dp4a_q4_0 = 4; + cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_c4 = nullptr; + cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_c2 = nullptr; + int nsg_coopk_dp4a_q4_k = 8; + int rows_coopk_dp4a_q4_k = 4; + cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 = nullptr; + cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 = nullptr; + int nsg_coopk_dp4a_q4_k_bin_c4 = 8; + int nsg_coopk_dp4a_q4_k_bin_c2 = 8; + cl_kernel kernel_gemm_coopk_q6_k_q8_1_dp4a_c4 = nullptr; + cl_kernel kernel_gemm_coopk_q6_k_q8_1_dp4a_c2 = nullptr; + int nsg_coopk_dp4a_q6_k = 8; + int rows_coopk_dp4a_q6_k = 4; #endif // GGML_OPENCL_USE_ADRENO_KERNELS + ~ggml_backend_opencl_context() { + for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) { + if (kernel != nullptr) { + CL_CHECK(clReleaseKernel(kernel)); + } + } + if (kernel_dsv4_hc_pre) { + CL_CHECK(clReleaseKernel(kernel_dsv4_hc_pre)); + } + for (int i = 0; i < 2; ++i) { + if (kernel_dsv4_hc_post[i]) { + CL_CHECK(clReleaseKernel(kernel_dsv4_hc_post[i])); + } + } + if (iq1_s_grid_image) { + CL_CHECK(clReleaseMemObject(iq1_s_grid_image)); + } + if (iq3_xxs_grid_image) { + CL_CHECK(clReleaseMemObject(iq3_xxs_grid_image)); + } + if (iq3_s_grid_image) { + CL_CHECK(clReleaseMemObject(iq3_s_grid_image)); + } + if (iq2_xxs_grid_image) { + CL_CHECK(clReleaseMemObject(iq2_xxs_grid_image)); + } + if (iq2_xs_grid_image) { + CL_CHECK(clReleaseMemObject(iq2_xs_grid_image)); + } + if (iq2_s_grid_image) { + CL_CHECK(clReleaseMemObject(iq2_s_grid_image)); + } + } + void free() { clFinish(queue); ref_count--; if (ref_count == 0) { + if (fa.fd_partial.buffer) { + CL_CHECK(clReleaseMemObject(fa.fd_partial.buffer)); + fa.fd_partial.buffer = nullptr; + fa.fd_partial.size = 0; + } #ifdef GGML_OPENCL_PROFILING flush_profiling_batch(); write_profiling_info(); @@ -1431,6 +1687,61 @@ static cl_program build_program_from_binary(cl_context ctx, cl_device_id dev, co return p; } +static void load_cl_kernels_argmax(ggml_backend_opencl_context *backend_ctx) { + auto opencl_c_std = + std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor); + std::string compile_opts = std::string("-cl-std=") + opencl_c_std; + + if (!backend_ctx->kernels_loaded_argmax) { + cl_int err; +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "argmax.cl.h" + }; +#else + const std::string kernel_src = read_file("argmax.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_argmax_f32_i32 = clCreateKernel(prog, "kernel_argmax_f32_i32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + backend_ctx->kernels_loaded_argmax = true; + } +} + +static std::string ggml_cl_q2_k_mm_options(const ggml_backend_opencl_context * backend_ctx) { + const bool x2 = backend_ctx->gpu_family == ADRENO && + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + int bk = x2 ? 16 : 32; + bool half = x2; + if (const char * env = getenv("GGML_OPENCL_LM_BK")) { + const int value = atoi(env); + if (value == 16 || value == 32) { + bk = value; + } + } + if (const char * env = getenv("GGML_OPENCL_LM_HALF")) { + half = atoi(env) != 0; + } + return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0); +} + +static std::string ggml_cl_q3_k_mm_options(const ggml_backend_opencl_context * backend_ctx) { + const bool x2 = backend_ctx->gpu_family == ADRENO && + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + int bk = x2 ? 16 : 32; + bool half = x2; + if (const char * env = getenv("GGML_OPENCL_LM_BK")) { + const int value = atoi(env); + if (value == 16 || value == 32) { + bk = value; + } + } + if (const char * env = getenv("GGML_OPENCL_LM_HALF")) { + half = atoi(env) != 0; + } + return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0); +} + static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) { // compiler options for general kernels auto opencl_c_std = @@ -1469,14 +1780,181 @@ static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) { #endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS } -static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { - if (backend_ctx->kernels_loaded) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_coopk_dp4a_enabled(const ggml_backend_opencl_context * backend_ctx) { + static const char * const env = getenv("GGML_OPENCL_COOPK_DP4A"); + if (env && *env && atoi(env) == 0) { + return false; + } + if (!backend_ctx->has_integer_dot) { + return false; + } + if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X) { + return false; + } + if (backend_ctx->adreno_cl_compiler_version.type == ADRENO_CL_COMPILER_TYPE::E17) { + return false; + } + } + return true; +} + +static bool ggml_cl_coopk_dp4a_narrow_on(const ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * op, int rows) { + if (!ggml_cl_coopk_dp4a_enabled(backend_ctx)) { + return false; + } + // Each lane handles rows output rows; subgroups split K for the same rows. + // Full row groups and 32-value K blocks avoid tail checks in the kernels. + return op->ne[1] >= 2 && op->ne[1] <= 4 + && rows > 0 && (op->src[0]->ne[1] % (64 * rows)) == 0 + && (op->src[0]->ne[0] % 32) == 0; +} + +// COOPK_NSG sizes the local reduction buffer, so a smaller workgroup requires a rebuild. +static cl_program ggml_cl_build_program_coopk(ggml_backend_opencl_context * backend_ctx, + const char * kernel_src, const char * kernel_name, + const std::string & opts_base, int & nsg) { + while (nsg >= 2) { + const std::string opts = opts_base + " -DCOOPK_NSG=" + std::to_string(nsg); + cl_program program = build_program_from_source(backend_ctx, kernel_src, opts); + cl_int err; + cl_kernel kernel = clCreateKernel(program, kernel_name, &err); + CL_CHECK(err); + size_t max_workgroup_size = 0; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, + sizeof(max_workgroup_size), &max_workgroup_size, NULL)); + CL_CHECK(clReleaseKernel(kernel)); + if ((size_t) 64 * nsg <= max_workgroup_size) { + return program; + } + CL_CHECK(clReleaseProgram(program)); + nsg /= 2; + } + return nullptr; +} + +static void ggml_cl_coopk_build_narrow(ggml_backend_opencl_context * backend_ctx, + const std::string & kernel_src, const char * kernel_name, + int rows, int & nsg, cl_kernel & kernel_c4, cl_kernel & kernel_c2) { + const std::string base_opts = backend_ctx->kernel_compile_opts + + " -DCOOPK_ROWS=" + std::to_string(rows); + for (int cols = 4; cols >= 2; cols -= 2) { + int nsg_eff = nsg; + cl_program program = ggml_cl_build_program_coopk( + backend_ctx, kernel_src.c_str(), kernel_name, + base_opts + " -DCOOPK_COLS=" + std::to_string(cols), nsg_eff); + if (program == nullptr) { + return; + } + cl_int err; + cl_kernel kernel = clCreateKernel(program, kernel_name, &err); + if (err != CL_SUCCESS) { kernel = nullptr; } + CL_CHECK(clReleaseProgram(program)); + if (cols == 4) { + kernel_c4 = kernel; + nsg = nsg_eff; + if (kernel == nullptr) { + return; + } + } else { + // Both widths share one stored subgroup count for launch. + if (kernel != nullptr && nsg_eff != nsg) { + CL_CHECK(clReleaseKernel(kernel)); + kernel = nullptr; + } + kernel_c2 = kernel; + } + } +} + +static void load_cl_kernels_gemm_coopk(ggml_backend_opencl_context * backend_ctx) { + if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO || !ggml_cl_coopk_dp4a_enabled(backend_ctx)) { return; } - cl_int err; + // Q4_0 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_coopk_q4_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_coopk_q4_0_q8_1_dp4a.cl"); +#endif + ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q4_0_q8_1_dp4a", + backend_ctx->rows_coopk_dp4a_q4_0, backend_ctx->nsg_coopk_dp4a_q4_0, + backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4, + backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2); + GGML_LOG_CONT("."); + } - // compiler options for general kernels + // Q4_K + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_coopk_q4_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_coopk_q4_k_q8_1_dp4a.cl"); +#endif + ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q4_k_q8_1_dp4a", + backend_ctx->rows_coopk_dp4a_q4_k, backend_ctx->nsg_coopk_dp4a_q4_k, + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4, + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2); + + const char * env = getenv("GGML_OPENCL_COOPK_Q4_K_BIN"); + if (use_adreno_bin_kernels(backend_ctx) && (!env || !*env || atoi(env) != 0)) { + const std::string base_opts = backend_ctx->kernel_compile_opts + + " -DCOOPK_Q_BIN=1 -DCOOPK_ROWS=" + std::to_string(backend_ctx->rows_coopk_dp4a_q4_k); + for (int cols = 4; cols >= 2; cols -= 2) { + int & nsg = (cols == 4) ? backend_ctx->nsg_coopk_dp4a_q4_k_bin_c4 + : backend_ctx->nsg_coopk_dp4a_q4_k_bin_c2; + int nsg_eff = nsg; + cl_program program = ggml_cl_build_program_coopk( + backend_ctx, kernel_src.c_str(), "kernel_gemm_coopk_q4_k_q8_1_dp4a", + base_opts + " -DCOOPK_COLS=" + std::to_string(cols), nsg_eff); + if (program == nullptr) { + continue; + } + cl_int err; + cl_kernel kernel = clCreateKernel(program, "kernel_gemm_coopk_q4_k_q8_1_dp4a", &err); + if (err != CL_SUCCESS) { kernel = nullptr; } + CL_CHECK(clReleaseProgram(program)); + if (cols == 4) { + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 = kernel; + } else { + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 = kernel; + } + nsg = nsg_eff; + } + } + GGML_LOG_CONT("."); + } + + // Q6_K + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_coopk_q6_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_coopk_q6_k_q8_1_dp4a.cl"); +#endif + ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q6_k_q8_1_dp4a", + backend_ctx->rows_coopk_dp4a_q6_k, backend_ctx->nsg_coopk_dp4a_q6_k, + backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4, + backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2); + GGML_LOG_CONT("."); + } +} + +#endif + +static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_context * backend_ctx) { auto opencl_c_std = std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor); std::string compile_opts = std::string("-cl-std=") + opencl_c_std + @@ -1487,7 +1965,18 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { compile_opts += " -qcom-enable-large-buffer "; } - backend_ctx->kernel_compile_opts = compile_opts; + return compile_opts; +} + +static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { + if (backend_ctx->kernels_loaded) { + return; + } + + cl_int err; + const std::string & compile_opts = backend_ctx->kernel_compile_opts; + const std::string opencl_c_std = "CL" + std::to_string(backend_ctx->opencl_c_version.major) + + "." + std::to_string(backend_ctx->opencl_c_version.minor); GGML_LOG_INFO("ggml_opencl: loading OpenCL kernels"); @@ -1526,6 +2015,41 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // mul_mat_id_utils + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mat_id_utils.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mat_id_utils.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // moe_sort_by_expert + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_sort_by_expert.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_sort_by_expert.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // moe_add_id_glu { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1579,6 +2103,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK(clReleaseProgram(prog)); } + // arange + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "arange.cl.h" + }; +#else + const std::string kernel_src = read_file("arange.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_arange = clCreateKernel(prog, "kernel_arange_f32", &err), err)); + GGML_LOG_CONT("."); + + CL_CHECK(clReleaseProgram(prog)); + } + // clamp { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1592,6 +2134,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_clamp = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp", &err), err)); + CL_CHECK((backend_ctx->kernel_clamp_nc = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp_nc", &err), err)); GGML_LOG_CONT("."); } @@ -1604,14 +2147,18 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #else const std::string kernel_src = read_file("cpy.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + const bool no_cpy_pack = backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X; + + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), + no_cpy_pack ? compile_opts + " -DGGML_CL_NO_CPY_PACK" : compile_opts); CL_CHECK((backend_ctx->kernel_cpy_f16_f16 = clCreateKernel(prog, "kernel_cpy_f16_f16", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f16_f32 = clCreateKernel(prog, "kernel_cpy_f16_f32", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f16 = clCreateKernel(prog, "kernel_cpy_f32_f16", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f32 = clCreateKernel(prog, "kernel_cpy_f32_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_cpy_f32_f32_pack = clCreateKernel(prog, "kernel_cpy_f32_f32_pack", &err), err)); + if (!no_cpy_pack) { + CL_CHECK((backend_ctx->kernel_cpy_f32_f32_pack = clCreateKernel(prog, "kernel_cpy_f32_f32_pack", &err), err)); + } { // optional: without it ggml_cl_cpy keeps the row-mapped kernel cl_int err_flat = CL_SUCCESS; cl_kernel k = clCreateKernel(prog, "kernel_cpy_f32_f32_flat", &err_flat); @@ -1620,6 +2167,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { } } CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_cpy_i32_f32 = clCreateKernel(prog, "kernel_cpy_i32_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_cpy_f32_i32 = clCreateKernel(prog, "kernel_cpy_f32_i32", &err), err)); GGML_LOG_CONT("."); } @@ -1694,6 +2243,34 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_restore_block_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq2_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq2_xxs", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq2_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq2_xxs", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq2_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq2_xs", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq2_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq2_xs", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq2_s = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq2_s", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq2_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq2_s", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q2_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_0", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q2_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_0", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq1_s = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq1_s", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq1_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq1_s", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq1_m = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq1_m", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq1_m = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq1_m", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq3_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq3_xxs", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq3_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_xxs", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq3_s", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_s", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq4_xs_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_xs_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq4_xs_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_xs_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq4_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_xs", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq4_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_xs", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K_noshuffle", &err), err)); @@ -1712,23 +2289,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } - // diag_mask_inf - { -#ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "diag_mask_inf.cl.h" - }; -#else - const std::string kernel_src = read_file("diag_mask_inf.cl"); -#endif - backend_ctx->program_diag_mask_inf = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_diag_mask_inf_8 = clCreateKernel(backend_ctx->program_diag_mask_inf, "kernel_diag_mask_inf_8", &err), err)); - CL_CHECK((backend_ctx->kernel_diag_mask_inf = clCreateKernel(backend_ctx->program_diag_mask_inf, "kernel_diag_mask_inf", &err), err)); - GGML_LOG_CONT("."); - } - // diag { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1795,6 +2355,58 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // lightning_indexer + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "lightning_indexer.cl.h" + }; +#else + const std::string kernel_src = read_file("lightning_indexer.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_lightning_indexer = clCreateKernel(prog, "kernel_lightning_indexer", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // top_k + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "top_k.cl.h" + }; +#else + const std::string kernel_src = read_file("top_k.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_top_k_f32 = clCreateKernel(prog, "kernel_top_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_top_k_merge_f32 = clCreateKernel(prog, "kernel_top_k_merge_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // top_k_mask + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "top_k_mask.cl.h" + }; +#else + const std::string kernel_src = read_file("top_k_mask.cl"); +#endif + // Scores can contain NaN and -inf, so do not use the default fast-math options. + const std::string opts = "-cl-std=" + opencl_c_std + + (backend_ctx->adreno_use_large_buffer ? " -qcom-enable-large-buffer" : ""); + for (int i = 0; i < 2; ++i) { + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts + " -DMASK_F16=" + std::to_string(i == 0)); + CL_CHECK((backend_ctx->kernel_top_k_mask[i] = clCreateKernel(prog, "kernel_top_k_mask", &err), err)); + CL_CHECK((backend_ctx->kernel_top_k_mask_fill[i] = clCreateKernel(prog, "kernel_top_k_mask_fill", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + } + GGML_LOG_CONT("."); + } + // get_rows { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1808,6 +2420,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_get_rows_i32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_i32", &err), err)); CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f16", &err), err)); CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_q4_0", &err), err)); GGML_LOG_CONT("."); @@ -1862,6 +2475,40 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // im2col_3d_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "im2col_3d_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("im2col_3d_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_im2col_3d_f32 = clCreateKernel(prog, "kernel_im2col_3d_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // im2col_3d_f16 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "im2col_3d_f16.cl.h" + }; +#else + const std::string kernel_src = read_file("im2col_3d_f16.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_im2col_3d_f16 = clCreateKernel(prog, "kernel_im2col_3d_f16", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q4_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1985,7930 +2632,12147 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } - // mul_mv_q4_k_f32 + // mul_mv_q2_k_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q4_k_f32.cl.h" + #include "mul_mv_q2_k_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q4_k_f32.cl"); + const std::string kernel_src = read_file("mul_mv_q2_k_f32_flat.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q4_K_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_q2_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q2_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q4_k_f32_flat + // mul_mm_q2_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q4_k_f32_flat.cl.h" + #include "mul_mm_q2_k_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q4_k_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mm_q2_k_f32_l4_lm.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q4_K_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + ggml_cl_q2_k_mm_options(backend_ctx)); + CL_CHECK((backend_ctx->kernel_mul_mm_q2_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q2_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q5_0_f32 + // mul_mv_q2_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q5_0_f32.cl.h" + #include "mul_mv_q2_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q5_0_f32.cl"); + const std::string kernel_src = read_file("mul_mv_q2_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mv_q5_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_0_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q2_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q2_K_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q5_0_f32_flat + // mul_mv_q3_k_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q5_0_f32_flat.cl.h" + #include "mul_mv_q3_k_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q5_0_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_q3_k_f32_flat.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q5_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_0_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q5_1_f32 + // mul_mm_q3_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q5_1_f32.cl.h" + #include "mul_mm_q3_k_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q5_1_f32.cl"); + const std::string kernel_src = read_file("mul_mm_q3_k_f32_l4_lm.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q5_1_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_1_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + ggml_cl_q3_k_mm_options(backend_ctx)); + CL_CHECK((backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q3_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q5_1_f32_flat +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq2_s_grid_src { + #include "iq2_s_grid.cl.h" + }; +#else + const std::string iq2_s_grid_src = read_file("iq2_s_grid.cl"); +#endif + + // mul_mv_iq2_s_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q5_1_f32_flat.cl.h" + #include "mul_mv_iq2_s_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q5_1_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_iq2_s_f32_flat.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q5_1_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_1_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq2_s_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq2_s_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q5_k_f32 + // mul_mm_iq2_s_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q5_k_f32.cl.h" + #include "mul_mm_iq2_s_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q5_k_f32.cl"); + const std::string kernel_src = read_file("mul_mm_iq2_s_f32_l4_lm.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q5_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_K_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq2_s_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq2_s_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q5_k_f32_flat +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq2_xs_grid_src { + #include "iq2_xs_grid.cl.h" + }; +#else + const std::string iq2_xs_grid_src = read_file("iq2_xs_grid.cl"); +#endif + + // mul_mv_iq2_xs_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q5_k_f32_flat.cl.h" + #include "mul_mv_iq2_xs_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q5_k_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_iq2_xs_f32_flat.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q5_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_K_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq2_xs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq2_xs_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // mul_mv_q6_k_f32 + // mul_mm_iq2_xs_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q6_k_f32.cl.h" + #include "mul_mm_iq2_xs_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q6_k_f32.cl"); + const std::string kernel_src = read_file("mul_mm_iq2_xs_f32_l4_lm.cl"); #endif - backend_ctx->program_mul_mv_q6_K = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32 = clCreateKernel(backend_ctx->program_mul_mv_q6_K, "kernel_mul_mv_q6_K_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq2_xs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq2_xs_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q6_k_f32_flat +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq2_xxs_grid_src { + #include "iq2_xxs_grid.cl.h" + }; +#else + const std::string iq2_xxs_grid_src = read_file("iq2_xxs_grid.cl"); +#endif + + // mul_mv_iq2_xxs_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q6_k_f32_flat.cl.h" + #include "mul_mv_iq2_xxs_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_iq2_xxs_f32_flat.cl"); #endif - // The codegen workarounds in this kernel are a measured 13-20% loss on - // compilers that do not need them, so only the affected ones build them; - // everyone else gets the original source. - const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler - ? compile_opts + " -DADRENO_OLD_COMPILER=1" - : compile_opts; - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq2_xxs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq2_xxs_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q8_0_f32 + // mul_mm_iq2_xxs_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q8_0_f32.cl.h" + #include "mul_mm_iq2_xxs_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q8_0_f32.cl"); + const std::string kernel_src = read_file("mul_mm_iq2_xxs_f32_l4_lm.cl"); #endif - backend_ctx->program_mul_mv_q8_0_f32 = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32, "kernel_mul_mv_q8_0_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq2_xxs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq2_xxs_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q8_0_f32_flat + // mul_mv_iq2_xxs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q8_0_f32_flat.cl.h" + #include "mul_mv_iq2_xxs_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q8_0_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_iq2_xxs_f32.cl"); #endif - backend_ctx->program_mul_mv_q8_0_f32_flat = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32_flat, "kernel_mul_mv_q8_0_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq2_xxs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq2_xxs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q1_0_f32 + // mul_mv_iq2_xs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q1_0_f32.cl.h" + #include "mul_mv_iq2_xs_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q1_0_f32.cl"); + const std::string kernel_src = read_file("mul_mv_iq2_xs_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq2_xs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq2_xs_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_q1_0_f32_flat + // mul_mv_iq2_s_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_q1_0_f32_flat.cl.h" + #include "mul_mv_iq2_s_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_q1_0_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_iq2_s_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq2_s_f32 = clCreateKernel(prog, "kernel_mul_mv_iq2_s_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_iq4_nl_f32 + // fwht { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_iq4_nl_f32.cl.h" + #include "fwht.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_iq4_nl_f32.cl"); + const std::string kernel_src = read_file("fwht.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_iq4_nl_f32 = clCreateKernel(prog, "kernel_mul_mv_iq4_nl_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + for (int i = 0; i < 2; ++i) { + const std::string opts = compile_opts + (i == 0 ? " -DSRC_TYPE=float" : " -DSRC_TYPE=half"); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_fwht[i] = clCreateKernel(prog, "kernel_fwht", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + size_t nth = 128; + const size_t max_workgroup = backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fwht[i]); + while (nth > max_workgroup) { + nth /= 2; + } + backend_ctx->fwht_workgroup_size[i] = nth; + } GGML_LOG_CONT("."); } - // mul_mv_iq4_nl_f32_flat + // mul_mv_q2_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_iq4_nl_f32_flat.cl.h" + #include "mul_mv_q2_0_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_iq4_nl_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_q2_0_f32_flat.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_iq4_nl_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq4_nl_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_q2_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q2_0_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_mxfp4_f32 + // mul_mm_q2_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_mxfp4_f32.cl.h" + #include "mul_mm_q2_0_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_mxfp4_f32.cl"); + const std::string kernel_src = read_file("mul_mm_q2_0_f32_l4_lm.cl"); #endif - backend_ctx->program_mul_mv_mxfp4_f32 = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32, "kernel_mul_mv_mxfp4_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_q2_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q2_0_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_mxfp4_f32_flat + // mul_mv_q2_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_mxfp4_f32_flat.cl.h" + #include "mul_mv_q2_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_mxfp4_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_q2_0_f32.cl"); #endif - backend_ctx->program_mul_mv_mxfp4_f32_flat = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32_flat, "kernel_mul_mv_mxfp4_f32_flat", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_q2_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q2_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_f16_f16 +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq1_s_grid_src { + #include "iq1_s_grid.cl.h" + }; +#else + const std::string iq1_s_grid_src = read_file("iq1_s_grid.cl"); +#endif + + // mul_mv_iq1_s_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_f16_f16.cl.h" + #include "mul_mv_iq1_s_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_f16_f16.cl"); + const std::string kernel_src = read_file("mul_mv_iq1_s_f32_flat.cl"); #endif - backend_ctx->program_mul_mv_f16_f16 = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f16 = clCreateKernel(backend_ctx->program_mul_mv_f16_f16, "kernel_mul_mat_f16_f16", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq1_s_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq1_s_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_f16_f32_1row + // mul_mm_iq1_s_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_f16_f32_1row.cl.h" + #include "mul_mm_iq1_s_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_f16_f32_1row.cl"); + const std::string kernel_src = read_file("mul_mm_iq1_s_f32_l4_lm.cl"); #endif - backend_ctx->program_mul_mv_f16_f32_1row = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_1row = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_1row, "kernel_mul_mat_f16_f32_1row", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq1_s_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq1_s_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_f16_f32_mrow (multi-row decode GEMV) + // mul_mv_iq1_s_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_f16_f32_mrow.cl.h" + #include "mul_mv_iq1_s_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_f16_f32_mrow.cl"); + const std::string kernel_src = read_file("mul_mv_iq1_s_f32.cl"); #endif - backend_ctx->program_mul_mv_f16_f32_mrow = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r2", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r4", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8r2", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq1_s_f32 = clCreateKernel(prog, "kernel_mul_mv_iq1_s_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_f16_f32_l4 + // mul_mv_iq1_m_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_f16_f32_l4.cl.h" + #include "mul_mv_iq1_m_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_f16_f32_l4.cl"); + const std::string kernel_src = read_file("mul_mv_iq1_m_f32_flat.cl"); #endif - backend_ctx->program_mul_mv_f16_f32_l4 = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr", &err), err)); - if (backend_ctx->gpu_family == ADRENO) { - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_ls", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_lq", &err), err)); - } - - cl_int err_x8 = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8", &err_x8); - if (err_x8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = nullptr; } - - cl_int err_x8p = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_pair", &err_x8p); - if (err_x8p != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = nullptr; } - - cl_int err_x8g = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4", &err_x8g); - if (err_x8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = nullptr; } - - cl_int err_x8gi = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4_img", &err_x8gi); - if (err_x8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = nullptr; } - - cl_int err_x8gi_r4 = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img", &err_x8gi_r4); - if (err_x8gi_r4 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = nullptr; } - - cl_int err_r2dk256 = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img", &err_r2dk256); - if (err_r2dk256 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = nullptr; } - - cl_int err_y8 = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8", &err_y8); - if (err_y8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = nullptr; } - - cl_int err_y8g = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa", &err_y8g); - if (err_y8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = nullptr; } - - cl_int err_y8gi = CL_SUCCESS; - backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = - clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa_img", &err_y8gi); - if (err_y8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = nullptr; } + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq1_m_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq1_m_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_f16_f32 + // mul_mm_iq1_m_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_f16_f32.cl.h" + #include "mul_mm_iq1_m_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_f16_f32.cl"); + const std::string kernel_src = read_file("mul_mm_iq1_m_f32_l4_lm.cl"); #endif - backend_ctx->program_mul_mv_f16_f32 = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32, "kernel_mul_mat_f16_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq1_m_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq1_m_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_f32_f32 + // mul_mv_iq1_m_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_f32_f32.cl.h" + #include "mul_mv_iq1_m_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_f32_f32.cl"); + const std::string kernel_src = read_file("mul_mv_iq1_m_f32.cl"); #endif - backend_ctx->program_mul_mv_f32_f32 = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mat_f32_f32 = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mat_f32_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq1_m_f32 = clCreateKernel(prog, "kernel_mul_mv_iq1_m_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mat_f16_f32_tiled - { #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "mul_mat_f16_f32.cl.h" - }; + const std::string iq3_s_grid_src { + #include "iq3_s_grid.cl.h" + }; #else - const std::string kernel_src = read_file("mul_mat_f16_f32.cl"); + const std::string iq3_s_grid_src = read_file("iq3_s_grid.cl"); #endif - backend_ctx->program_mul_mat_f16_f32_tiled = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_tiled = clCreateKernel(backend_ctx->program_mul_mat_f16_f32_tiled, "mul_mat_f16_f32", &err), err)); - GGML_LOG_CONT("."); - } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // gemm_xmem_f16_f32_os8 + // mul_mv_iq3_s_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_xmem_f16_f32_os8.cl.h" + #include "mul_mv_iq3_s_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("gemm_xmem_f16_f32_os8.cl"); + const std::string kernel_src = read_file("mul_mv_iq3_s_f32_flat.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_adreno_xmem_pack_src_f32 = - clCreateKernel(prog, "adreno_xmem_pack_src_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_adreno_xmem_prepack_weight_f16 = - clCreateKernel(prog, "adreno_xmem_prepack_weight_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_xmem_f16_f32_os8 = - clCreateKernel(prog, "kernel_gemm_xmem_f16_f32_os8", &err), err)); - CL_CHECK((backend_ctx->kernel_adreno_xmem_store_dst_f32 = - clCreateKernel(prog, "adreno_xmem_store_dst_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq3_s_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq3_s_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Adreno xmem SDPA - if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { + // mul_mm_iq3_s_f32_l4_lm + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "sdpa_xmem_f32_f16_os8.cl.h" + #include "mul_mm_iq3_s_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("sdpa_xmem_f32_f16_os8.cl"); + const std::string kernel_src = read_file("mul_mm_iq3_s_f32_l4_lm.cl"); #endif - cl_program program = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - auto & xmem_attn = backend_ctx->adreno_xmem_attn; - CL_CHECK((xmem_attn.kernel_q_f32_to_img_scaled = - clCreateKernel(program, "adreno_xmem_attn_q_f32_to_img_scaled", &err), err)); - CL_CHECK((xmem_attn.kernel_kv_f32_to_img_gqa = - clCreateKernel(program, "adreno_xmem_attn_kv_f32_to_img_gqa", &err), err)); - CL_CHECK((xmem_attn.kernel_kv_f16_to_img_gqa = - clCreateKernel(program, "adreno_xmem_attn_kv_f16_to_img_gqa", &err), err)); - CL_CHECK((xmem_attn.kernel_img_to_f32 = - clCreateKernel(program, "adreno_xmem_attn_img_to_f32", &err), err)); - CL_CHECK((xmem_attn.kernel_k_gather = - clCreateKernel(program, "adreno_xmem_attn_k_gather", &err), err)); - CL_CHECK((xmem_attn.kernel_pack_k = - clCreateKernel(program, "adreno_xmem_attn_pack_k", &err), err)); - CL_CHECK((xmem_attn.kernel_qk_gemm = - clCreateKernel(program, "adreno_xmem_attn_qk_gemm", &err), err)); - CL_CHECK((xmem_attn.kernel_softmax_reduce_basic = - clCreateKernel(program, "adreno_xmem_attn_softmax_reduce_basic", &err), err)); - CL_CHECK((xmem_attn.kernel_softmax_apply_basic = - clCreateKernel(program, "adreno_xmem_attn_softmax_apply_basic", &err), err)); - CL_CHECK((xmem_attn.kernel_mask_scores = - clCreateKernel(program, "adreno_xmem_attn_mask_scores", &err), err)); - CL_CHECK((xmem_attn.kernel_pack_v = - clCreateKernel(program, "adreno_xmem_attn_pack_v", &err), err)); - CL_CHECK((xmem_attn.kernel_pv_gemm = - clCreateKernel(program, "adreno_xmem_attn_pv_gemm", &err), err)); - CL_CHECK(clReleaseProgram(program)); - xmem_attn.compiled = true; + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq3_s_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq3_s_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - // mul_mm_f32_f32_l4_lm + // mul_mv_iq3_s_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_f32_f32_l4_lm.cl.h" + #include "mul_mv_iq3_s_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_f32_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_iq3_s_f32.cl"); #endif - backend_ctx->program_mul_mm_f32_f32_l4_lm = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mm_f32_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_mul_mm_f32_f32_l4_lm", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_f32_f32_mc = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_gemv_f32_f32_mc", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq3_s_f32 = clCreateKernel(prog, "kernel_mul_mv_iq3_s_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_f16_f32_l4_lm +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq3_xxs_grid_src { + #include "iq3_xxs_grid.cl.h" + }; +#else + const std::string iq3_xxs_grid_src = read_file("iq3_xxs_grid.cl"); +#endif + + // mul_mv_iq3_xxs_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_f16_f32_l4_lm.cl.h" + #include "mul_mv_iq3_xxs_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_f16_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_iq3_xxs_f32_flat.cl"); #endif - backend_ctx->program_mul_mm_f16_f32_l4_lm = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_l4_lm, "kernel_mul_mm_f16_f32_l4_lm", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq3_xxs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq3_xxs_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q4_0_f32_l4_lm + // mul_mm_iq3_xxs_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q4_0_f32_l4_lm.cl.h" + #include "mul_mm_iq3_xxs_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q4_0_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mm_iq3_xxs_f32_l4_lm.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_0_f32_l4_lm", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq3_xxs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq3_xxs_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q4_1_f32_l4_lm + // mul_mv_iq3_xxs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q4_1_f32_l4_lm.cl.h" + #include "mul_mv_iq3_xxs_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q4_1_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_iq3_xxs_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_1_f32_l4_lm", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq3_xxs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq3_xxs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q5_0_f32_l4_lm +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq4_xs_grid_src { + #include "iq4_xs_grid.cl.h" + }; +#else + const std::string iq4_xs_grid_src = read_file("iq4_xs_grid.cl"); +#endif + + // mul_mv_iq4_xs_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q5_0_f32_l4_lm.cl.h" + #include "mul_mv_iq4_xs_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q5_0_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_iq4_xs_f32_flat.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_0_f32_l4_lm", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_xs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq4_xs_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q5_1_f32_l4_lm + // mul_mm_iq4_xs_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q5_1_f32_l4_lm.cl.h" + #include "mul_mm_iq4_xs_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q5_1_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mm_iq4_xs_f32_l4_lm.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_1_f32_l4_lm", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq4_xs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq4_xs_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q8_0_f32_l4_lm + // mul_mv_iq4_xs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q8_0_f32_l4_lm.cl.h" + #include "mul_mv_iq4_xs_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q8_0_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_iq4_xs_f32.cl"); #endif - backend_ctx->program_mul_mm_q8_0_f32_l4_lm = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_q8_0_f32_l4_lm, "kernel_mul_mm_q8_0_f32_l4_lm", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_xs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq4_xs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q1_0_f32_l4_lm + // mul_mv_q3_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q1_0_f32_l4_lm.cl.h" + #include "mul_mv_q3_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q1_0_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_q3_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q1_0_f32_l4_lm", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_iq4_nl_f32_l4_lm + // mul_mv_q4_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_iq4_nl_f32_l4_lm.cl.h" + #include "mul_mv_q4_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_iq4_nl_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_q4_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq4_nl_f32_l4_lm", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q4_K_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q4_k_f32_l4_lm + // mul_mv_q4_k_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q4_k_f32_l4_lm.cl.h" + #include "mul_mv_q4_k_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q4_k_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_q4_k_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_k_f32_l4_lm", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q4_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q6_k_f32_l4_lm + // mul_mv_q5_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q6_k_f32_l4_lm.cl.h" + #include "mul_mv_q5_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q6_k_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_q5_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q6_k_f32_l4_lm", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q5_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q5_k_f32_l4_lm + // mul_mv_q5_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_q5_k_f32_l4_lm.cl.h" + #include "mul_mv_q5_0_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_q5_k_f32_l4_lm.cl"); + const std::string kernel_src = read_file("mul_mv_q5_0_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_k_f32_l4_lm", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q5_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_0_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_f16_f32_kq_kqv + // mul_mv_q5_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mm_f16_f32_kq_kqv.cl.h" + #include "mul_mv_q5_1_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mm_f16_f32_kq_kqv.cl"); + const std::string kernel_src = read_file("mul_mv_q5_1_f32.cl"); #endif - backend_ctx->program_mul_mm_f16_f32_kqv = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts+" -DKQV "); - backend_ctx->program_mul_mm_f16_f32_kq = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kqv = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kqv, "mul_mm_f16_f32_kqv", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kq = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kq, "mul_mm_f16_f32_kq", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q5_1_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_1_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul + // mul_mv_q5_1_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul.cl.h" + #include "mul_mv_q5_1_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul.cl"); + const std::string kernel_src = read_file("mul_mv_q5_1_f32_flat.cl"); #endif - backend_ctx->program_mul = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul = clCreateKernel(backend_ctx->program_mul, "kernel_mul", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_row = clCreateKernel(backend_ctx->program_mul, "kernel_mul_row", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_f16 = clCreateKernel(backend_ctx->program_mul, "kernel_mul_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_mul_row_f16 = clCreateKernel(backend_ctx->program_mul, "kernel_mul_row_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q5_1_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_1_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // norm + // mul_mv_q5_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "norm.cl.h" + #include "mul_mv_q5_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("norm.cl"); + const std::string kernel_src = read_file("mul_mv_q5_k_f32.cl"); #endif - backend_ctx->program_norm = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_norm = clCreateKernel(backend_ctx->program_norm, "kernel_norm", &err), err)); - CL_CHECK((backend_ctx->kernel_norm_mul_add = clCreateKernel(backend_ctx->program_norm, "kernel_norm_mul_add", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q5_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_K_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // relu + // mul_mv_q5_k_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "relu.cl.h" + #include "mul_mv_q5_k_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("relu.cl"); + const std::string kernel_src = read_file("mul_mv_q5_k_f32_flat.cl"); #endif - backend_ctx->program_relu = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_relu = clCreateKernel(backend_ctx->program_relu, "kernel_relu", &err), err)); - GGML_LOG_CONT("."); + CL_CHECK((backend_ctx->kernel_mul_mv_q5_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_K_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); } - // rms_norm + // mul_mv_q6_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "rms_norm.cl.h" + #include "mul_mv_q6_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("rms_norm.cl"); + const std::string kernel_src = read_file("mul_mv_q6_k_f32.cl"); #endif - backend_ctx->program_rms_norm = + backend_ctx->program_mul_mv_q6_K = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_rms_norm = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm", &err), err)); - CL_CHECK((backend_ctx->kernel_rms_norm_mul = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul", &err), err)); - CL_CHECK((backend_ctx->kernel_rms_norm_mul_add = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul_add", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32 = clCreateKernel(backend_ctx->program_mul_mv_q6_K, "kernel_mul_mv_q6_K_f32", &err), err)); GGML_LOG_CONT("."); } - // l2_norm + // mul_mv_q6_k_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "l2_norm.cl.h" + #include "mul_mv_q6_k_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("l2_norm.cl"); + const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl"); #endif + // The codegen workarounds in this kernel are a measured 13-20% loss on + // compilers that do not need them, so only the affected ones build them; + // everyone else gets the original source. + const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler + ? compile_opts + " -DADRENO_OLD_COMPILER=1" + : compile_opts; cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts); - CL_CHECK((backend_ctx->kernel_l2_norm_f32 = clCreateKernel(prog, "kernel_l2_norm_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // rope + // mul_mv_q8_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "rope.cl.h" + #include "mul_mv_q8_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("rope.cl"); + const std::string kernel_src = read_file("mul_mv_q8_0_f32.cl"); #endif - backend_ctx->program_rope = + backend_ctx->program_mul_mv_q8_0_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_rope_norm_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_norm_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_rope_norm_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_norm_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_rope_neox_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_neox_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_rope_neox_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_neox_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_rope_multi_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_multi_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_rope_multi_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_multi_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_rope_vision_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_vision_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_rope_vision_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_vision_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32, "kernel_mul_mv_q8_0_f32", &err), err)); GGML_LOG_CONT("."); } - // scale + // mul_mv_q8_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "scale.cl.h" + #include "mul_mv_q8_0_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("scale.cl"); + const std::string kernel_src = read_file("mul_mv_q8_0_f32_flat.cl"); #endif - cl_program prog = + backend_ctx->program_mul_mv_q8_0_f32_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_scale_f32 = clCreateKernel(prog, "kernel_scale_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_scale_f32_4 = clCreateKernel(prog, "kernel_scale_f32_4", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32_flat, "kernel_mul_mv_q8_0_f32_flat", &err), err)); GGML_LOG_CONT("."); } - // silu + // mul_mv_q1_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "silu.cl.h" + #include "mul_mv_q1_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("silu.cl"); + const std::string kernel_src = read_file("mul_mv_q1_0_f32.cl"); #endif - backend_ctx->program_silu = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_silu = clCreateKernel(backend_ctx->program_silu, "kernel_silu", &err), err)); - CL_CHECK((backend_ctx->kernel_silu_4 = clCreateKernel(backend_ctx->program_silu, "kernel_silu_4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // softmax_f32 + // mul_mv_q1_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "softmax_f32.cl.h" + #include "mul_mv_q1_0_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("softmax_f32.cl"); + const std::string kernel_src = read_file("mul_mv_q1_0_f32_flat.cl"); #endif - backend_ctx->program_softmax_f32 = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_soft_max = clCreateKernel(backend_ctx->program_softmax_f32, "kernel_soft_max", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // softmax_f16 + // mul_mv_iq4_nl_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "softmax_f16.cl.h" + #include "mul_mv_iq4_nl_f32.cl.h" }; #else - const std::string kernel_src = read_file("softmax_f16.cl"); + const std::string kernel_src = read_file("mul_mv_iq4_nl_f32.cl"); #endif - backend_ctx->program_softmax_f16 = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_soft_max_f16 = clCreateKernel(backend_ctx->program_softmax_f16, "kernel_soft_max_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_nl_f32 = clCreateKernel(prog, "kernel_mul_mv_iq4_nl_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // softmax_4_f32 + // mul_mv_iq4_nl_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "softmax_4_f32.cl.h" + #include "mul_mv_iq4_nl_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("softmax_4_f32.cl"); + const std::string kernel_src = read_file("mul_mv_iq4_nl_f32_flat.cl"); #endif - backend_ctx->program_softmax_4_f32 = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_soft_max_4 = clCreateKernel(backend_ctx->program_softmax_4_f32, "kernel_soft_max_4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_nl_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq4_nl_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // softmax_4_f16 + // mul_mv_mxfp4_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "softmax_4_f16.cl.h" + #include "mul_mv_mxfp4_f32.cl.h" }; #else - const std::string kernel_src = read_file("softmax_4_f16.cl"); + const std::string kernel_src = read_file("mul_mv_mxfp4_f32.cl"); #endif - backend_ctx->program_softmax_4_f16 = + backend_ctx->program_mul_mv_mxfp4_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_soft_max_4_f16 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_soft_max_4_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32, "kernel_mul_mv_mxfp4_f32", &err), err)); GGML_LOG_CONT("."); } - // div + // mul_mv_mxfp4_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "div.cl.h" + #include "mul_mv_mxfp4_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("div.cl"); + const std::string kernel_src = read_file("mul_mv_mxfp4_f32_flat.cl"); #endif - std::string compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable -cl-finite-math-only "; - - backend_ctx->program_div = + backend_ctx->program_mul_mv_mxfp4_f32_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_div = clCreateKernel(backend_ctx->program_div, "kernel_div", &err), err)); - CL_CHECK((backend_ctx->kernel_div_row = clCreateKernel(backend_ctx->program_div, "kernel_div_row", &err), err)); - CL_CHECK((backend_ctx->kernel_div_f16 = clCreateKernel(backend_ctx->program_div, "kernel_div_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_div_row_f16 = clCreateKernel(backend_ctx->program_div, "kernel_div_row_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32_flat, "kernel_mul_mv_mxfp4_f32_flat", &err), err)); GGML_LOG_CONT("."); } - // sqr + // mul_mv_f16_f16 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "sqr.cl.h" + #include "mul_mv_f16_f16.cl.h" }; #else - const std::string kernel_src = read_file("sqr.cl"); + const std::string kernel_src = read_file("mul_mv_f16_f16.cl"); #endif - cl_program prog = + backend_ctx->program_mul_mv_f16_f16 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_sqr_cont_f32 = clCreateKernel(prog, "kernel_sqr_cont_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_sqr_cont_f32_4 = clCreateKernel(prog, "kernel_sqr_cont_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_sqr_cont_f16 = clCreateKernel(prog, "kernel_sqr_cont_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_sqr_cont_f16_4 = clCreateKernel(prog, "kernel_sqr_cont_f16_4", &err), err)); - - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f16 = clCreateKernel(backend_ctx->program_mul_mv_f16_f16, "kernel_mul_mat_f16_f16", &err), err)); GGML_LOG_CONT("."); } - // sqrt + // mul_mv_f16_f32_1row { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "sqrt.cl.h" + #include "mul_mv_f16_f32_1row.cl.h" }; #else - const std::string kernel_src = read_file("sqrt.cl"); + const std::string kernel_src = read_file("mul_mv_f16_f32_1row.cl"); #endif - cl_program prog = + backend_ctx->program_mul_mv_f16_f32_1row = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_sqrt_cont_f32 = clCreateKernel(prog, "kernel_sqrt_cont_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_sqrt_cont_f32_4 = clCreateKernel(prog, "kernel_sqrt_cont_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_sqrt_cont_f16 = clCreateKernel(prog, "kernel_sqrt_cont_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_sqrt_cont_f16_4 = clCreateKernel(prog, "kernel_sqrt_cont_f16_4", &err), err)); - - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_1row = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_1row, "kernel_mul_mat_f16_f32_1row", &err), err)); GGML_LOG_CONT("."); } - // mean + // mul_mv_f16_f32_mrow (multi-row decode GEMV) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mean.cl.h" + #include "mul_mv_f16_f32_mrow.cl.h" }; #else - const std::string kernel_src = read_file("mean.cl"); + const std::string kernel_src = read_file("mul_mv_f16_f32_mrow.cl"); #endif - cl_program prog = + backend_ctx->program_mul_mv_f16_f32_mrow = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mean_f32 = clCreateKernel(prog, "kernel_mean_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_mean_f32_4 = clCreateKernel(prog, "kernel_mean_f32_4", &err), err)); - - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r2", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8r2", &err), err)); GGML_LOG_CONT("."); } - // sub + // mul_mv_f16_f32_l4 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "sub.cl.h" + #include "mul_mv_f16_f32_l4.cl.h" }; #else - const std::string kernel_src = read_file("sub.cl"); + const std::string kernel_src = read_file("mul_mv_f16_f32_l4.cl"); #endif - backend_ctx->program_sub = + backend_ctx->program_mul_mv_f16_f32_l4 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_sub = clCreateKernel(backend_ctx->program_sub, "kernel_sub", &err), err)); - CL_CHECK((backend_ctx->kernel_sub_row = clCreateKernel(backend_ctx->program_sub, "kernel_sub_row", &err), err)); - CL_CHECK((backend_ctx->kernel_sub_f16 = clCreateKernel(backend_ctx->program_sub, "kernel_sub_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_sub_row_f16 = clCreateKernel(backend_ctx->program_sub, "kernel_sub_row_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr", &err), err)); + if (backend_ctx->gpu_family == ADRENO) { + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_ls", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_lq", &err), err)); + } + + cl_int err_x8 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8", &err_x8); + if (err_x8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = nullptr; } + + cl_int err_x8p = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_pair", &err_x8p); + if (err_x8p != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = nullptr; } + + cl_int err_x8g = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4", &err_x8g); + if (err_x8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = nullptr; } + + cl_int err_x8gi = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4_img", &err_x8gi); + if (err_x8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = nullptr; } + + cl_int err_x8gi_r4 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img", &err_x8gi_r4); + if (err_x8gi_r4 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = nullptr; } + + cl_int err_r2dk256 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img", &err_r2dk256); + if (err_r2dk256 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = nullptr; } + + cl_int err_y8 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8", &err_y8); + if (err_y8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = nullptr; } + + cl_int err_y8g = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa", &err_y8g); + if (err_y8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = nullptr; } + + cl_int err_y8gi = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa_img", &err_y8gi); + if (err_y8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = nullptr; } GGML_LOG_CONT("."); } - // sum_rows + // mul_mv_f16_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "sum_rows.cl.h" + #include "mul_mv_f16_f32.cl.h" }; #else - const std::string kernel_src = read_file("sum_rows.cl"); + const std::string kernel_src = read_file("mul_mv_f16_f32.cl"); #endif - backend_ctx->program_sum_rows_f32 = + backend_ctx->program_mul_mv_f16_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_sum_rows_f32 = clCreateKernel(backend_ctx->program_sum_rows_f32, "kernel_sum_rows_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_sum_rows_f32_4 = clCreateKernel(backend_ctx->program_sum_rows_f32, "kernel_sum_rows_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32, "kernel_mul_mat_f16_f32", &err), err)); GGML_LOG_CONT("."); } - // cumsum + // mul_mv_f32_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "cumsum.cl.h" + #include "mul_mv_f32_f32.cl.h" }; #else - const std::string kernel_src = read_file("cumsum.cl"); + const std::string kernel_src = read_file("mul_mv_f32_f32.cl"); #endif - cl_program prog; - prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + backend_ctx->program_mul_mv_f32_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_cumsum_blk = clCreateKernel(prog, "kernel_cumsum_blk", &err), err)); - CL_CHECK((backend_ctx->kernel_cumsum_add = clCreateKernel(prog, "kernel_cumsum_add", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f32_f32 = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mat_f32_f32", &err), err)); + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X) { + cl_kernel kernel = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mv_f32_f32_r2", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (max_workgroup >= 64) { + backend_ctx->kernel_mul_mv_f32_f32_r2 = kernel; + } else { + CL_CHECK(clReleaseKernel(kernel)); + } + } GGML_LOG_CONT("."); - CL_CHECK(clReleaseProgram(prog)); } - // sigmoid + // mul_mat_f16_f32_tiled { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "sigmoid.cl.h" + #include "mul_mat_f16_f32.cl.h" }; #else - const std::string kernel_src = read_file("sigmoid.cl"); + const std::string kernel_src = read_file("mul_mat_f16_f32.cl"); #endif - backend_ctx->program_sigmoid = + backend_ctx->program_mul_mat_f16_f32_tiled = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_sigmoid_f32 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_sigmoid_f16 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_tiled = clCreateKernel(backend_ctx->program_mul_mat_f16_f32_tiled, "mul_mat_f16_f32", &err), err)); GGML_LOG_CONT("."); } - // group_norm +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // gemm_xmem_f16_f32_os8 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "group_norm.cl.h" + #include "gemm_xmem_f16_f32_os8.cl.h" }; #else - const std::string kernel_src = read_file("group_norm.cl"); + const std::string kernel_src = read_file("gemm_xmem_f16_f32_os8.cl"); #endif - backend_ctx->program_group_norm = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_group_norm = clCreateKernel(backend_ctx->program_group_norm, "kernel_group_norm", &err), err)); - CL_CHECK((backend_ctx->kernel_group_norm_mul_add = clCreateKernel(backend_ctx->program_group_norm, "kernel_group_norm_mul_add", &err), err)); + CL_CHECK((backend_ctx->kernel_adreno_xmem_pack_src_f32 = + clCreateKernel(prog, "adreno_xmem_pack_src_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_adreno_xmem_prepack_weight_f16 = + clCreateKernel(prog, "adreno_xmem_prepack_weight_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_xmem_f16_f32_os8 = + clCreateKernel(prog, "kernel_gemm_xmem_f16_f32_os8", &err), err)); + CL_CHECK((backend_ctx->kernel_adreno_xmem_store_dst_f32 = + clCreateKernel(prog, "adreno_xmem_store_dst_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - // repeat +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno xmem SDPA + if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "sdpa_xmem_f32_f16_os8.cl.h" + }; +#else + const std::string kernel_src = read_file("sdpa_xmem_f32_f16_os8.cl"); +#endif + cl_program program = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + auto & xmem_attn = backend_ctx->adreno_xmem_attn; + CL_CHECK((xmem_attn.kernel_q_f32_to_img_scaled = + clCreateKernel(program, "adreno_xmem_attn_q_f32_to_img_scaled", &err), err)); + CL_CHECK((xmem_attn.kernel_kv_f32_to_img_gqa = + clCreateKernel(program, "adreno_xmem_attn_kv_f32_to_img_gqa", &err), err)); + CL_CHECK((xmem_attn.kernel_kv_f16_to_img_gqa = + clCreateKernel(program, "adreno_xmem_attn_kv_f16_to_img_gqa", &err), err)); + CL_CHECK((xmem_attn.kernel_img_to_f32 = + clCreateKernel(program, "adreno_xmem_attn_img_to_f32", &err), err)); + CL_CHECK((xmem_attn.kernel_k_gather = + clCreateKernel(program, "adreno_xmem_attn_k_gather", &err), err)); + CL_CHECK((xmem_attn.kernel_pack_k = + clCreateKernel(program, "adreno_xmem_attn_pack_k", &err), err)); + CL_CHECK((xmem_attn.kernel_qk_gemm = + clCreateKernel(program, "adreno_xmem_attn_qk_gemm", &err), err)); + CL_CHECK((xmem_attn.kernel_softmax_reduce_basic = + clCreateKernel(program, "adreno_xmem_attn_softmax_reduce_basic", &err), err)); + CL_CHECK((xmem_attn.kernel_softmax_apply_basic = + clCreateKernel(program, "adreno_xmem_attn_softmax_apply_basic", &err), err)); + CL_CHECK((xmem_attn.kernel_mask_scores = + clCreateKernel(program, "adreno_xmem_attn_mask_scores", &err), err)); + CL_CHECK((xmem_attn.kernel_pack_v = + clCreateKernel(program, "adreno_xmem_attn_pack_v", &err), err)); + CL_CHECK((xmem_attn.kernel_pv_gemm = + clCreateKernel(program, "adreno_xmem_attn_pv_gemm", &err), err)); + CL_CHECK(clReleaseProgram(program)); + xmem_attn.compiled = true; + GGML_LOG_CONT("."); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + // mul_mm_f32_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "repeat.cl.h" + #include "mul_mm_f32_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("repeat.cl"); + const std::string kernel_src = read_file("mul_mm_f32_f32_l4_lm.cl"); #endif - cl_program prog = + backend_ctx->program_mul_mm_f32_f32_l4_lm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_repeat_f32 = clCreateKernel(prog, "kernel_repeat_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + + CL_CHECK((backend_ctx->kernel_mul_mm_f32_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_mul_mm_f32_f32_l4_lm", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_f32_f32_mc = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_gemv_f32_f32_mc", &err), err)); GGML_LOG_CONT("."); } - // pad + // mul_mm_f16_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "pad.cl.h" + #include "mul_mm_f16_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("pad.cl"); + const std::string kernel_src = read_file("mul_mm_f16_f32_l4_lm.cl"); #endif - if (!kernel_src.empty()) { - backend_ctx->program_pad = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_pad = clCreateKernel(backend_ctx->program_pad, "kernel_pad", &err), err)); - GGML_LOG_CONT("."); - } else { - GGML_LOG_WARN("ggml_opencl: pad kernel source not found or empty. Pad operations will not be available.\n"); - backend_ctx->program_pad = nullptr; - backend_ctx->kernel_pad = nullptr; - } + backend_ctx->program_mul_mm_f16_f32_l4_lm = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_l4_lm, "kernel_mul_mm_f16_f32_l4_lm", &err), err)); + + const std::string narrow_opts = compile_opts + " -DBN=8 -DTN=1 -DKERNEL_NAME_LM=kernel_mul_mm_f16_f32_l4_lm_n8"; + backend_ctx->program_mul_mm_f16_f32_l4_lm_n8 = + build_program_from_source(backend_ctx, kernel_src.c_str(), narrow_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_l4_lm_n8 = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_l4_lm_n8, "kernel_mul_mm_f16_f32_l4_lm_n8", &err), err)); + GGML_LOG_CONT("."); } - // tanh + // mul_mm_q4_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "tanh.cl.h" + #include "mul_mm_q4_0_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("tanh.cl"); + const std::string kernel_src = read_file("mul_mm_q4_0_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_tanh_f32 = clCreateKernel(prog, "kernel_tanh_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_tanh_f32_4 = clCreateKernel(prog, "kernel_tanh_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_tanh_f32_nc = clCreateKernel(prog, "kernel_tanh_f32_nc", &err), err)); - CL_CHECK((backend_ctx->kernel_tanh_f16 = clCreateKernel(prog, "kernel_tanh_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_tanh_f16_4 = clCreateKernel(prog, "kernel_tanh_f16_4", &err), err)); - CL_CHECK((backend_ctx->kernel_tanh_f16_nc = clCreateKernel(prog, "kernel_tanh_f16_nc", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + + CL_CHECK((backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_0_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } - // neg + // mul_mm_q4_1_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "neg.cl.h" + #include "mul_mm_q4_1_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("neg.cl"); + const std::string kernel_src = read_file("mul_mm_q4_1_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_neg_f32 = clCreateKernel(prog, "kernel_neg_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_neg_f32_4 = clCreateKernel(prog, "kernel_neg_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_neg_f32_nc = clCreateKernel(prog, "kernel_neg_f32_nc", &err), err)); - CL_CHECK((backend_ctx->kernel_neg_f16 = clCreateKernel(prog, "kernel_neg_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_neg_f16_4 = clCreateKernel(prog, "kernel_neg_f16_4", &err), err)); - CL_CHECK((backend_ctx->kernel_neg_f16_nc = clCreateKernel(prog, "kernel_neg_f16_nc", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + + CL_CHECK((backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_1_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } - // exp + // mul_mm_q5_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "exp.cl.h" + #include "mul_mm_q5_0_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("exp.cl"); + const std::string kernel_src = read_file("mul_mm_q5_0_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_exp_f32 = clCreateKernel(prog, "kernel_exp_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_exp_f32_4 = clCreateKernel(prog, "kernel_exp_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_exp_f32_nc = clCreateKernel(prog, "kernel_exp_f32_nc", &err), err)); - CL_CHECK((backend_ctx->kernel_exp_f16 = clCreateKernel(prog, "kernel_exp_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_exp_f16_4 = clCreateKernel(prog, "kernel_exp_f16_4", &err), err)); - CL_CHECK((backend_ctx->kernel_exp_f16_nc = clCreateKernel(prog, "kernel_exp_f16_nc", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + + CL_CHECK((backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_0_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } - // expm1 + // mul_mm_q5_1_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "expm1.cl.h" + #include "mul_mm_q5_1_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("expm1.cl"); + const std::string kernel_src = read_file("mul_mm_q5_1_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_expm1_f32 = clCreateKernel(prog, "kernel_expm1_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_expm1_f32_4 = clCreateKernel(prog, "kernel_expm1_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_expm1_f32_nc = clCreateKernel(prog, "kernel_expm1_f32_nc", &err), err)); - CL_CHECK((backend_ctx->kernel_expm1_f16 = clCreateKernel(prog, "kernel_expm1_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_expm1_f16_4 = clCreateKernel(prog, "kernel_expm1_f16_4", &err), err)); - CL_CHECK((backend_ctx->kernel_expm1_f16_nc = clCreateKernel(prog, "kernel_expm1_f16_nc", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + + CL_CHECK((backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_1_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } - // abs + // mul_mm_q8_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "abs.cl.h" + #include "mul_mm_q8_0_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("abs.cl"); + const std::string kernel_src = read_file("mul_mm_q8_0_f32_l4_lm.cl"); #endif - cl_program prog = + backend_ctx->program_mul_mm_q8_0_f32_l4_lm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_abs_f32 = clCreateKernel(prog, "kernel_abs_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_abs_f32_4 = clCreateKernel(prog, "kernel_abs_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_abs_f32_nc = clCreateKernel(prog, "kernel_abs_f32_nc", &err), err)); - CL_CHECK((backend_ctx->kernel_abs_f16 = clCreateKernel(prog, "kernel_abs_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_abs_f16_4 = clCreateKernel(prog, "kernel_abs_f16_4", &err), err)); - CL_CHECK((backend_ctx->kernel_abs_f16_nc = clCreateKernel(prog, "kernel_abs_f16_nc", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + + CL_CHECK((backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_q8_0_f32_l4_lm, "kernel_mul_mm_q8_0_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } - // unary_ext (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc) + // mul_mm_q1_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "unary_ext.cl.h" + #include "mul_mm_q1_0_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("unary_ext.cl"); + const std::string kernel_src = read_file("mul_mm_q1_0_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); -#define CL_UNARY_EXT_K(op) \ - CL_CHECK((backend_ctx->kernel_##op##_f32 = clCreateKernel(prog, "kernel_" #op "_f32", &err), err)); \ - CL_CHECK((backend_ctx->kernel_##op##_f32_4 = clCreateKernel(prog, "kernel_" #op "_f32_4", &err), err)); \ - CL_CHECK((backend_ctx->kernel_##op##_f32_nc = clCreateKernel(prog, "kernel_" #op "_f32_nc", &err), err)); \ - CL_CHECK((backend_ctx->kernel_##op##_f16 = clCreateKernel(prog, "kernel_" #op "_f16", &err), err)); \ - CL_CHECK((backend_ctx->kernel_##op##_f16_4 = clCreateKernel(prog, "kernel_" #op "_f16_4", &err), err)); \ - CL_CHECK((backend_ctx->kernel_##op##_f16_nc = clCreateKernel(prog, "kernel_" #op "_f16_nc", &err), err)); - CL_UNARY_EXT_K(sgn) - CL_UNARY_EXT_K(step) - CL_UNARY_EXT_K(elu) - CL_UNARY_EXT_K(hardswish) - CL_UNARY_EXT_K(hardsigmoid) - CL_UNARY_EXT_K(floor) - CL_UNARY_EXT_K(ceil) - CL_UNARY_EXT_K(round) - CL_UNARY_EXT_K(trunc) -#undef CL_UNARY_EXT_K + + CL_CHECK((backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q1_0_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // softplus + // mul_mm_iq4_nl_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "softplus.cl.h" + #include "mul_mm_iq4_nl_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("softplus.cl"); + const std::string kernel_src = read_file("mul_mm_iq4_nl_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_softplus_f32 = clCreateKernel(prog, "kernel_softplus_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_softplus_f32_4 = clCreateKernel(prog, "kernel_softplus_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_softplus_f32_nc = clCreateKernel(prog, "kernel_softplus_f32_nc", &err), err)); - CL_CHECK((backend_ctx->kernel_softplus_f16 = clCreateKernel(prog, "kernel_softplus_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_softplus_f16_4 = clCreateKernel(prog, "kernel_softplus_f16_4", &err), err)); - CL_CHECK((backend_ctx->kernel_softplus_f16_nc = clCreateKernel(prog, "kernel_softplus_f16_nc", &err), err)); + + CL_CHECK((backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq4_nl_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // upscale + // mul_mm_q4_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "upscale.cl.h" + #include "mul_mm_q4_k_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("upscale.cl"); + const std::string kernel_src = read_file("mul_mm_q4_k_f32_l4_lm.cl"); #endif - if (!kernel_src.empty()) { - backend_ctx->program_upscale = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_upscale = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale", &err), err)); - if (backend_ctx->program_upscale) { - cl_int err_bilinear; - backend_ctx->kernel_upscale_bilinear = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale_bilinear", &err_bilinear); - if (err_bilinear != CL_SUCCESS) { - GGML_LOG_WARN("ggml_opencl: kernel_upscale_bilinear not found in upscale.cl. Bilinear upscale will not be available. Error: %d\n", err_bilinear); - backend_ctx->kernel_upscale_bilinear = nullptr; - } - } else { - backend_ctx->kernel_upscale_bilinear = nullptr; - } - GGML_LOG_CONT("."); - } else { - GGML_LOG_WARN("ggml_opencl: upscale kernel source not found or empty. Upscale operations will not be available.\n"); - backend_ctx->program_upscale = nullptr; - backend_ctx->kernel_upscale = nullptr; - backend_ctx->kernel_upscale_bilinear = nullptr; - } + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_k_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // concat + // mul_mm_q6_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "concat.cl.h" + #include "mul_mm_q6_k_f32_l4_lm.cl.h" }; #else - const std::string kernel_src = read_file("concat.cl"); + const std::string kernel_src = read_file("mul_mm_q6_k_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_concat_b1 = clCreateKernel(prog, "kernel_concat_b1", &err), err)); - CL_CHECK((backend_ctx->kernel_concat_b2 = clCreateKernel(prog, "kernel_concat_b2", &err), err)); - CL_CHECK((backend_ctx->kernel_concat_b4 = clCreateKernel(prog, "kernel_concat_b4", &err), err)); - CL_CHECK((backend_ctx->kernel_concat_b8 = clCreateKernel(prog, "kernel_concat_b8", &err), err)); - CL_CHECK((backend_ctx->kernel_concat_b4_pack = clCreateKernel(prog, "kernel_concat_b4_pack", &err), err)); + + CL_CHECK((backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q6_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // timestep_embedding + // mul_mm_q5_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "tsembd.cl.h" + #include "mul_mm_q5_k_f32_l4_lm.cl.h" }; #else - - const std::string kernel_src = read_file("tsembd.cl"); + const std::string kernel_src = read_file("mul_mm_q5_k_f32_l4_lm.cl"); #endif - if (!kernel_src.empty()) { - backend_ctx->program_tsembd = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_timestep_embedding = clCreateKernel(backend_ctx->program_tsembd, "kernel_timestep_embedding", &err), err)); - GGML_LOG_CONT("."); - } else { - GGML_LOG_WARN("ggml_opencl: timestep_embedding kernel source not found or empty. This op will not be available.\n"); - backend_ctx->program_tsembd = nullptr; - backend_ctx->kernel_timestep_embedding = nullptr; - } + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_k_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // set_rows + // mul_mm_f16_f32_kq_kqv { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "set_rows.cl.h" + #include "mul_mm_f16_f32_kq_kqv.cl.h" }; #else - const std::string kernel_src = read_file("set_rows.cl"); + const std::string kernel_src = read_file("mul_mm_f16_f32_kq_kqv.cl"); #endif - backend_ctx->program_set_rows = + backend_ctx->program_mul_mm_f16_f32_kqv = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts+" -DKQV "); + backend_ctx->program_mul_mm_f16_f32_kq = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_set_rows_f32_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i64", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_f32_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i32", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_f16_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i64", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_f16_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i32", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i64", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i32", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i64", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i32", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i64", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i32", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i64", &err), err)); - CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kqv = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kqv, "mul_mm_f16_f32_kqv", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kq = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kq, "mul_mm_f16_f32_kq", &err), err)); GGML_LOG_CONT("."); } - // conv2d - { - #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "conv2d.cl.h" - }; - const std::string kernel_src_f16_f32 { - #include "conv2d_f16_f32.cl.h" - }; - #else - const std::string kernel_src = read_file("conv2d.cl"); - const std::string kernel_src_f16_f32 = read_file("conv2d_f16_f32.cl"); - #endif - if (!kernel_src.empty()) { - backend_ctx->program_conv_2d_f16 = - build_program_from_source(backend_ctx, kernel_src.c_str(), (std::string(compile_opts) + " -DUSE_FP16=1").c_str()); - CL_CHECK((backend_ctx->kernel_conv_2d_f16 = clCreateKernel(backend_ctx->program_conv_2d_f16, "kernel_conv_2d", &err), err)); - GGML_LOG_CONT("."); - backend_ctx->program_conv_2d_f32 = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_conv_2d_f32 = clCreateKernel(backend_ctx->program_conv_2d_f32, "kernel_conv_2d", &err), err)); - GGML_LOG_CONT("."); - } else { - GGML_LOG_WARN("ggml_opencl: conv2d kernel source not found or empty. This op will not be available.\n"); - backend_ctx->program_conv_2d_f16 = nullptr; - backend_ctx->kernel_conv_2d_f16 = nullptr; - backend_ctx->program_conv_2d_f32 = nullptr; - backend_ctx->kernel_conv_2d_f32 = nullptr; - } - if (!kernel_src_f16_f32.empty()) { - backend_ctx->program_conv_2d_f16_f32 = - build_program_from_source(backend_ctx, kernel_src_f16_f32.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_conv_2d_f16_f32 = clCreateKernel(backend_ctx->program_conv_2d_f16_f32, "kernel_conv_2d", &err), err)); - GGML_LOG_CONT("."); - } else { - GGML_LOG_WARN("ggml_opencl: conv2d_f16_f32 kernel source not found or empty. This op will not be available.\n"); - backend_ctx->program_conv_2d_f16_f32 = nullptr; - backend_ctx->kernel_conv_2d_f16_f32 = nullptr; - } - } - - // ssm_conv - { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // mul_mm_f16_f32_kq_p8 + if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "ssm_conv.cl.h" + #include "mul_mm_f16_f32_kq_p8.cl.h" }; #else - const std::string kernel_src = read_file("ssm_conv.cl"); + const std::string kernel_src = read_file("mul_mm_f16_f32_kq_p8.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32 = clCreateKernel(prog, "kernel_ssm_conv_f32_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32_4 = clCreateKernel(prog, "kernel_ssm_conv_f32_f32_4", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + cl_program prog = build_program_from_source_ex(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), + compile_opts, /*fatal=*/false); + if (prog != nullptr) { + CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kq_p8 = clCreateKernel(prog, "mul_mm_f16_f32_kq_p8", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + } GGML_LOG_CONT("."); } +#endif - // ssm_scan + // mul { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "ssm_scan.cl.h" + #include "mul.cl.h" }; #else - const std::string kernel_src = read_file("ssm_scan.cl"); + const std::string kernel_src = read_file("mul.cl"); #endif - cl_program prog = + backend_ctx->program_mul = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_ssm_scan_f32 = clCreateKernel(prog, "kernel_ssm_scan_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d128 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d128", &err), err)); - CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d256 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d256", &err), err)); - - cl_kernel * kernels[] = { - &backend_ctx->kernel_ssm_scan_f32_mamba2_d128, - &backend_ctx->kernel_ssm_scan_f32_mamba2_d256 - }; - - // specialized kernels use subgroups and assume subgroup size is 64, - // if device does not support subgroups or subgroup size is not 64, - // release these kernels - for (int i = 0; i < 2; ++i) { - size_t subgroup_size = 0; -#if CL_TARGET_OPENCL_VERSION >= 210 - const size_t local_work_size[] = { 64, 1 }; - const cl_int subgroup_err = clGetKernelSubGroupInfo(*kernels[i], backend_ctx->device, CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, - sizeof(local_work_size), local_work_size, sizeof(subgroup_size), &subgroup_size, nullptr); - if (subgroup_err != CL_SUCCESS) { - subgroup_size = 0; - } -#endif - // The specialized kernels reduce over one 64-lane subgroup. - if (subgroup_size != 64) { - CL_CHECK(clReleaseKernel(*kernels[i])); - *kernels[i] = nullptr; - } - } - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_mul = clCreateKernel(backend_ctx->program_mul, "kernel_mul", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_row = clCreateKernel(backend_ctx->program_mul, "kernel_mul_row", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_f16 = clCreateKernel(backend_ctx->program_mul, "kernel_mul_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_row_f16 = clCreateKernel(backend_ctx->program_mul, "kernel_mul_row_f16", &err), err)); GGML_LOG_CONT("."); } - // gated_delta_net: one kernel per (S_V, KDA, tgpp) triple. + // norm { - #ifdef GGML_OPENCL_EMBED_KERNELS +#ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gated_delta_net.cl.h" + #include "norm.cl.h" }; - #else - const std::string kernel_src = read_file("gated_delta_net.cl"); - #endif - - const int gdn_sizes[4] = { 16, 32, 64, 128 }; - const int sg_size = backend_ctx->gpu_family == GPU_FAMILY::ADRENO ? 64 : backend_ctx->gpu_family == GPU_FAMILY::INTEL ? 32 : -1; - if (sg_size < 0) { - GGML_LOG_ERROR("Unsupported GPU Family: only Adreno and Intel are supported.\n"); - exit(1); - } - - for (int si = 0; si < 4; si++) { - const int S_V = gdn_sizes[si]; - - // MUST match the dispatcher heuristic in ggml_cl_gated_delta_net exactly. - int lanes_per_column; - if (S_V >= 128) { - lanes_per_column = 8; - } else { - lanes_per_column = std::min(S_V, sg_size); - } - - // Round LANES_PER_COLUMN down until it is: - // * power-of-two - // * divides both S_V and sg_size - while (lanes_per_column > 1 && - (((lanes_per_column & (lanes_per_column - 1)) != 0) || - (S_V % lanes_per_column) != 0 || - (sg_size % lanes_per_column) != 0)) { - lanes_per_column >>= 1; - } - - GGML_ASSERT(lanes_per_column >= 1); - GGML_ASSERT(((lanes_per_column & (lanes_per_column - 1)) == 0)); - GGML_ASSERT((S_V % lanes_per_column) == 0); - GGML_ASSERT((sg_size % lanes_per_column) == 0); - - const bool is_partial_reduce = (lanes_per_column != 1) && (lanes_per_column < sg_size); - int use_qcom_shuffle = 0; - if (is_partial_reduce) { - if (backend_ctx->has_qcom_subgroup_shuffle) { - use_qcom_shuffle = 1; - } - } - for (int kda = 0; kda < 2; kda++) { - for (int tgpp = 0; tgpp < 2; tgpp++) { - const int cpl = (tgpp == 0) ? 1 : 4; - const int spw = (tgpp == 0) ? 1 : 1; - - std::string opts = compile_opts; - opts += " -DS_V=" + std::to_string(S_V); - opts += " -DKDA=" + std::to_string(kda); - opts += " -DSUBGROUP_SIZE=" + std::to_string(sg_size); - opts += " -DLANES_PER_COLUMN=" + std::to_string(lanes_per_column); - opts += " -DCOLS_PER_LANE_GROUP=" + std::to_string(cpl); - opts += " -DUSE_QCOM_SUBGROUP_SHUFFLE=" + std::to_string(use_qcom_shuffle); - - // Since spw=1 is found to be optimal, SUBGROUPS_PER_WG > 1 code in - // the kernel is removed. If you want to experiment with spw > 1, - // Please remember to implement code to handle it. - opts += " -DSUBGROUPS_PER_WG=" + std::to_string(spw); - - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); +#else + const std::string kernel_src = read_file("norm.cl"); +#endif + backend_ctx->program_norm = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gated_delta_net_f32[si][kda][tgpp] = - clCreateKernel(prog, "kernel_gated_delta_net", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - } - } - } + CL_CHECK((backend_ctx->kernel_norm = clCreateKernel(backend_ctx->program_norm, "kernel_norm", &err), err)); + CL_CHECK((backend_ctx->kernel_norm_mul_add = clCreateKernel(backend_ctx->program_norm, "kernel_norm_mul_add", &err), err)); GGML_LOG_CONT("."); } - // moe_combine (fused router-weight mul + cross-expert sum) + // relu { - #ifdef GGML_OPENCL_EMBED_KERNELS +#ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "moe_combine.cl.h" + #include "relu.cl.h" }; - #else - const std::string kernel_src = read_file("moe_combine.cl"); - #endif - cl_program prog = build_program_from_source( - backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_moe_combine_f32 = - clCreateKernel(prog, "kernel_moe_combine_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_combine_bias_f32 = - clCreateKernel(prog, "kernel_moe_combine_bias_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); +#else + const std::string kernel_src = read_file("relu.cl"); +#endif + backend_ctx->program_relu = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_relu = clCreateKernel(backend_ctx->program_relu, "kernel_relu", &err), err)); GGML_LOG_CONT("."); } - // mul_mv_id_q4_0_f32_8x_flat + // rms_norm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_id_q4_0_f32_8x_flat.cl.h" + #include "rms_norm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_id_q4_0_f32_8x_flat.cl"); + const std::string kernel_src = read_file("rms_norm.cl"); #endif - backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat = + backend_ctx->program_rms_norm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat, "kernel_mul_mv_id_q4_0_f32_8x_flat", &err), err)); + CL_CHECK((backend_ctx->kernel_rms_norm = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm", &err), err)); + CL_CHECK((backend_ctx->kernel_rms_norm_mul = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul", &err), err)); + CL_CHECK((backend_ctx->kernel_rms_norm_mul_add = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul_add", &err), err)); GGML_LOG_CONT("."); } - // mul_mv_id_q8_0_f32 + // l2_norm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_id_q8_0_f32.cl.h" + #include "l2_norm.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_id_q8_0_f32.cl"); + const std::string kernel_src = read_file("l2_norm.cl"); #endif - backend_ctx->program_mul_mv_id_q8_0_f32 = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32, "kernel_mul_mv_id_q8_0_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_l2_norm_f32 = clCreateKernel(prog, "kernel_l2_norm_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_id_q8_0_f32_flat + // rope { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_id_q8_0_f32_flat.cl.h" + #include "rope.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_id_q8_0_f32_flat.cl"); + const std::string kernel_src = read_file("rope.cl"); #endif - backend_ctx->program_mul_mv_id_q8_0_f32_flat = + backend_ctx->program_rope = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32_flat, "kernel_mul_mv_id_q8_0_f32_flat", &err), err)); + CL_CHECK((backend_ctx->kernel_rope_norm_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_norm_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_rope_norm_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_norm_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_rope_neox_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_neox_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_rope_neox_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_neox_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_rope_multi_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_multi_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_rope_multi_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_multi_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_rope_vision_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_vision_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_rope_vision_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_vision_f16", &err), err)); GGML_LOG_CONT("."); } - // mul_mv_id_mxfp4_f32 + // scale { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_id_mxfp4_f32.cl.h" + #include "scale.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32.cl"); + const std::string kernel_src = read_file("scale.cl"); #endif - backend_ctx->program_mul_mv_id_mxfp4_f32 = + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32, "kernel_mul_mv_id_mxfp4_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_scale_f32 = clCreateKernel(prog, "kernel_scale_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_scale_f32_4 = clCreateKernel(prog, "kernel_scale_f32_4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_id_mxfp4_f32_flat + // silu { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_id_mxfp4_f32_flat.cl.h" + #include "silu.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32_flat.cl"); + const std::string kernel_src = read_file("silu.cl"); #endif - backend_ctx->program_mul_mv_id_mxfp4_f32_flat = + backend_ctx->program_silu = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32_flat, "kernel_mul_mv_id_mxfp4_f32_flat", &err), err)); + CL_CHECK((backend_ctx->kernel_silu = clCreateKernel(backend_ctx->program_silu, "kernel_silu", &err), err)); + CL_CHECK((backend_ctx->kernel_silu_4 = clCreateKernel(backend_ctx->program_silu, "kernel_silu_4", &err), err)); GGML_LOG_CONT("."); } - // Adreno kernels -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // transpose + // softmax_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "transpose.cl.h" + #include "softmax_f32.cl.h" }; #else - const std::string kernel_src = read_file("transpose.cl"); + const std::string kernel_src = read_file("softmax_f32.cl"); #endif - backend_ctx->program_transpose = + backend_ctx->program_softmax_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_transpose_32_16 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32_16", &err), err)); - CL_CHECK((backend_ctx->kernel_transpose_32 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32", &err), err)); - CL_CHECK((backend_ctx->kernel_transpose_16 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16", &err), err)); - CL_CHECK((backend_ctx->kernel_transpose_8_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_8_buf", &err), err)); - CL_CHECK((backend_ctx->kernel_transpose_16_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16_buf", &err), err)); - CL_CHECK((backend_ctx->kernel_transpose_32_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32_buf", &err), err)); - CL_CHECK((backend_ctx->kernel_transpose_16_4x1 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16_4x1", &err), err)); + CL_CHECK((backend_ctx->kernel_soft_max = clCreateKernel(backend_ctx->program_softmax_f32, "kernel_soft_max", &err), err)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q1_0_f32 + // softmax_f16 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q1_0_f32.cl.h" + #include "softmax_f16.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q1_0_f32.cl"); + const std::string kernel_src = read_file("softmax_f16.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q1_0_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + backend_ctx->program_softmax_f16 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_soft_max_f16 = clCreateKernel(backend_ctx->program_softmax_f16, "kernel_soft_max_f16", &err), err)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q1_0_f32 + // softmax_4_f32 { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); - #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_CL_gemv_general { - #include "gemv_noshuffle_q1_0_f32.cl.h" + const std::string kernel_src { + #include "softmax_4_f32.cl.h" }; #else - const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q1_0_f32.cl"); + const std::string kernel_src = read_file("softmax_4_f32.cl"); #endif + backend_ctx->program_softmax_4_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q1_0_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_soft_max_4 = clCreateKernel(backend_ctx->program_softmax_4_f32, "kernel_soft_max_4", &err), err)); GGML_LOG_CONT("."); } - // gemv_noshuffle_general + // softmax_4_f16 { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } - #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_CL_gemv_general { - #include "gemv_noshuffle_q4_0_f32.cl.h" + const std::string kernel_src { + #include "softmax_4_f16.cl.h" }; #else - const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q4_0_f32.cl"); + const std::string kernel_src = read_file("softmax_4_f16.cl"); #endif + backend_ctx->program_softmax_4_f16 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_mc3", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_soft_max_4_f16 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_soft_max_4_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_fa_decompose_soft_max_4_f16 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_fa_decompose_soft_max_4_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_fa_decompose_scale_rows_f32 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_fa_decompose_scale_rows_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_fa_p8_fixup = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_fa_p8_fixup", &err), err)); GGML_LOG_CONT("."); } - // gemv_noshuffle - { - // Gemv 2048, 16384 - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DLINE_STRIDE_A=2048 " - " -DBLOCK_STRIDE_A=16384 " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } - +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) { #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_CL_gemv { - #include "gemv_noshuffle_q4_0_f32_spec.cl.h" + const std::string kernel_src { + #include "flash_attn_pv_direct.cl.h" }; #else - const std::string kernel_src_CL_gemv = read_file("gemv_noshuffle_q4_0_f32_spec.cl"); + const std::string kernel_src = read_file("flash_attn_pv_direct.cl"); #endif - - cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + cl_program prog = build_program_from_source_ex(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), + compile_opts, /*fatal=*/false); + if (prog != nullptr) { + CL_CHECK((backend_ctx->kernel_fa_kqv_direct_f16 = clCreateKernel(prog, "kernel_fa_kqv_direct_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_fa_kqv_direct_reduce = clCreateKernel(prog, "kernel_fa_kqv_direct_reduce", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + } GGML_LOG_CONT("."); + } - // Gemv 2048, 16384 - CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DLINE_STRIDE_A=2048 " - " -DBLOCK_STRIDE_A=16384 " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q8_f32_kqv.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q8_f32_kqv.cl"); +#endif + cl_program prog = build_program_from_source_ex(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), + compile_opts, /*fatal=*/false); + if (prog != nullptr) { + CL_CHECK((backend_ctx->kernel_mul_mm_q8_f32_kqv = clCreateKernel(prog, "kernel_mul_mm_q8_f32_kqv", &err), err)); + CL_CHECK(clReleaseProgram(prog)); } - - prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); + } +#endif - // Gemv 5504, 44032 - CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DLINE_STRIDE_A=5504 " - " -DBLOCK_STRIDE_A=44032 " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } + // div + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "div.cl.h" + }; +#else + const std::string kernel_src = read_file("div.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable -cl-finite-math-only "; - prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + backend_ctx->program_div = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_div = clCreateKernel(backend_ctx->program_div, "kernel_div", &err), err)); + CL_CHECK((backend_ctx->kernel_div_row = clCreateKernel(backend_ctx->program_div, "kernel_div_row", &err), err)); + CL_CHECK((backend_ctx->kernel_div_f16 = clCreateKernel(backend_ctx->program_div, "kernel_div_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_div_row_f16 = clCreateKernel(backend_ctx->program_div, "kernel_div_row_f16", &err), err)); GGML_LOG_CONT("."); + } - // Gemv 16000, 128000 - CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DLINE_STRIDE_A=16000 " - " -DBLOCK_STRIDE_A=128000 " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); + // sqr + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "sqr.cl.h" + }; +#else + const std::string kernel_src = read_file("sqr.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } + CL_CHECK((backend_ctx->kernel_sqr_cont_f32 = clCreateKernel(prog, "kernel_sqr_cont_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_sqr_cont_f32_4 = clCreateKernel(prog, "kernel_sqr_cont_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_sqr_cont_f16 = clCreateKernel(prog, "kernel_sqr_cont_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_sqr_cont_f16_4 = clCreateKernel(prog, "kernel_sqr_cont_f16_4", &err), err)); - prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mat_Ab_Bi_8x4 + // sqrt { #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_CL_gemm { - #include "gemm_noshuffle_q4_0_f32.cl.h" + const std::string kernel_src { + #include "sqrt.cl.h" }; #else - const std::string kernel_src_CL_gemm = read_file("gemm_noshuffle_q4_0_f32.cl"); + const std::string kernel_src = read_file("sqrt.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemm.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_f32", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_sqrt_cont_f32 = clCreateKernel(prog, "kernel_sqrt_cont_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_sqrt_cont_f32_4 = clCreateKernel(prog, "kernel_sqrt_cont_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_sqrt_cont_f16 = clCreateKernel(prog, "kernel_sqrt_cont_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_sqrt_cont_f16_4 = clCreateKernel(prog, "kernel_sqrt_cont_f16_4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; - backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; - backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = nullptr; + // mean { - { - std::string opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "gemv_noshuffle_q4_0_f32_32b_trans.cl.h" - }; + const std::string kernel_src { + #include "mean.cl.h" + }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q4_0_f32_32b_trans.cl"); + const std::string kernel_src = read_file("mean.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = - clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_32b_trans", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - - if (use_adreno_bin_kernels(backend_ctx)) { - size_t bin_size = 0; - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program bin_prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = - clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(bin_prog)); - GGML_LOG_CONT("."); - } + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program bin_prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + CL_CHECK((backend_ctx->kernel_mean_f32 = clCreateKernel(prog, "kernel_mean_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_mean_f32_4 = clCreateKernel(prog, "kernel_mean_f32_4", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = - clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(bin_prog)); - GGML_LOG_CONT("."); - } - } + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // gemm_noshuffle_q4_1_f32 + // sub { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q4_1_f32.cl.h" + #include "sub.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q4_1_f32.cl"); + const std::string kernel_src = read_file("sub.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_1_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + backend_ctx->program_sub = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_sub = clCreateKernel(backend_ctx->program_sub, "kernel_sub", &err), err)); + CL_CHECK((backend_ctx->kernel_sub_row = clCreateKernel(backend_ctx->program_sub, "kernel_sub_row", &err), err)); + CL_CHECK((backend_ctx->kernel_sub_f16 = clCreateKernel(backend_ctx->program_sub, "kernel_sub_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_sub_row_f16 = clCreateKernel(backend_ctx->program_sub, "kernel_sub_row_f16", &err), err)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q4_1_f32 + // dsv4_hc { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "dsv4_hc.cl.h" + }; +#else + const std::string kernel_src = read_file("dsv4_hc.cl"); +#endif + for (int i = 0; i < 2; ++i) { + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + (i == 0 ? " -DHC_VEC=1" : " -DHC_VEC=4")); + if (i == 0) { + CL_CHECK((backend_ctx->kernel_dsv4_hc_pre = clCreateKernel(prog, "kernel_dsv4_hc_pre", &err), err)); + backend_ctx->local_dsv4_hc_pre = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_pre)); + } + CL_CHECK((backend_ctx->kernel_dsv4_hc_post[i] = clCreateKernel(prog, "kernel_dsv4_hc_post", &err), err)); + backend_ctx->local_dsv4_hc_post[i] = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_post[i])); + CL_CHECK(clReleaseProgram(prog)); } + GGML_LOG_CONT("."); + } + // sum_rows + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q4_1_f32.cl.h" + #include "sum_rows.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q4_1_f32.cl"); + const std::string kernel_src = read_file("sum_rows.cl"); #endif + backend_ctx->program_sum_rows_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32_mc3", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_sum_rows_f32 = clCreateKernel(backend_ctx->program_sum_rows_f32, "kernel_sum_rows_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_sum_rows_f32_4 = clCreateKernel(backend_ctx->program_sum_rows_f32, "kernel_sum_rows_f32_4", &err), err)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q5_0_f32 + // cumsum { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q5_0_f32.cl.h" + #include "cumsum.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q5_0_f32.cl"); + const std::string kernel_src = read_file("cumsum.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + cl_program prog; + prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_cumsum_blk = clCreateKernel(prog, "kernel_cumsum_blk", &err), err)); + CL_CHECK((backend_ctx->kernel_cumsum_add = clCreateKernel(prog, "kernel_cumsum_add", &err), err)); GGML_LOG_CONT("."); + CL_CHECK(clReleaseProgram(prog)); } - // gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM) - if (backend_ctx->has_integer_dot) { + // sigmoid + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h" + #include "sigmoid.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q5_0_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("sigmoid.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + backend_ctx->program_sigmoid = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_sigmoid_f32 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_sigmoid_f16 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_sigmoid_f32_nc = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_sigmoid_f16_nc = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f16_nc", &err), err)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q5_0_f32 + // group_norm { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } - #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q5_0_f32.cl.h" + #include "group_norm.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q5_0_f32.cl"); + const std::string kernel_src = read_file("group_norm.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_0_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + backend_ctx->program_group_norm = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_group_norm = clCreateKernel(backend_ctx->program_group_norm, "kernel_group_norm", &err), err)); + CL_CHECK((backend_ctx->kernel_group_norm_mul_add = clCreateKernel(backend_ctx->program_group_norm, "kernel_group_norm_mul_add", &err), err)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q5_1_f32 + // repeat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q5_1_f32.cl.h" + #include "repeat.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q5_1_f32.cl"); + const std::string kernel_src = read_file("repeat.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_1_f32", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_repeat_f32 = clCreateKernel(prog, "kernel_repeat_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_repeat_f16 = clCreateKernel(prog, "kernel_repeat_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_repeat_i32 = clCreateKernel(prog, "kernel_repeat_i32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q5_1_f32 + // pad { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } - #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q5_1_f32.cl.h" + #include "pad.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q5_1_f32.cl"); + const std::string kernel_src = read_file("pad.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_1_f32", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); + if (!kernel_src.empty()) { + backend_ctx->program_pad = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_pad = clCreateKernel(backend_ctx->program_pad, "kernel_pad", &err), err)); + GGML_LOG_CONT("."); + } else { + GGML_LOG_WARN("ggml_opencl: pad kernel source not found or empty. Pad operations will not be available.\n"); + backend_ctx->program_pad = nullptr; + backend_ctx->kernel_pad = nullptr; + } } - // gemm_noshuffle_iq4_nl_f32 + // tanh { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_iq4_nl_f32.cl.h" + #include "tanh.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_f32.cl"); + const std::string kernel_src = read_file("tanh.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_f32", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_tanh_f32 = clCreateKernel(prog, "kernel_tanh_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_tanh_f32_4 = clCreateKernel(prog, "kernel_tanh_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_tanh_f32_nc = clCreateKernel(prog, "kernel_tanh_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_tanh_f16 = clCreateKernel(prog, "kernel_tanh_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_tanh_f16_4 = clCreateKernel(prog, "kernel_tanh_f16_4", &err), err)); + CL_CHECK((backend_ctx->kernel_tanh_f16_nc = clCreateKernel(prog, "kernel_tanh_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM) - if (backend_ctx->has_integer_dot) { + // neg + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h" + #include "neg.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("neg.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_neg_f32 = clCreateKernel(prog, "kernel_neg_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_neg_f32_4 = clCreateKernel(prog, "kernel_neg_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_neg_f32_nc = clCreateKernel(prog, "kernel_neg_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_neg_f16 = clCreateKernel(prog, "kernel_neg_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_neg_f16_4 = clCreateKernel(prog, "kernel_neg_f16_4", &err), err)); + CL_CHECK((backend_ctx->kernel_neg_f16_nc = clCreateKernel(prog, "kernel_neg_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM) - if (backend_ctx->has_integer_dot) { + // exp + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h" + #include "exp.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q4_0_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("exp.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_exp_f32 = clCreateKernel(prog, "kernel_exp_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_exp_f32_4 = clCreateKernel(prog, "kernel_exp_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_exp_f32_nc = clCreateKernel(prog, "kernel_exp_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_exp_f16 = clCreateKernel(prog, "kernel_exp_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_exp_f16_4 = clCreateKernel(prog, "kernel_exp_f16_4", &err), err)); + CL_CHECK((backend_ctx->kernel_exp_f16_nc = clCreateKernel(prog, "kernel_exp_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_iq4_nl_f32 + // expm1 { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } - #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_iq4_nl_f32.cl.h" + #include "expm1.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_iq4_nl_f32.cl"); + const std::string kernel_src = read_file("expm1.cl"); #endif - - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_iq4_nl_f32", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_expm1_f32 = clCreateKernel(prog, "kernel_expm1_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_expm1_f32_4 = clCreateKernel(prog, "kernel_expm1_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_expm1_f32_nc = clCreateKernel(prog, "kernel_expm1_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_expm1_f16 = clCreateKernel(prog, "kernel_expm1_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_expm1_f16_4 = clCreateKernel(prog, "kernel_expm1_f16_4", &err), err)); + CL_CHECK((backend_ctx->kernel_expm1_f16_nc = clCreateKernel(prog, "kernel_expm1_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q8_0_f32_8x4 + // abs { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q8_0_f32.cl.h" + #include "abs.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q8_0_f32.cl"); + const std::string kernel_src = read_file("abs.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_abs_f32 = clCreateKernel(prog, "kernel_abs_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f32_4 = clCreateKernel(prog, "kernel_abs_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f32_nc = clCreateKernel(prog, "kernel_abs_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f16 = clCreateKernel(prog, "kernel_abs_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f16_4 = clCreateKernel(prog, "kernel_abs_f16_4", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f16_nc = clCreateKernel(prog, "kernel_abs_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q8_0_f32_bin - { - size_t bin_size = 0; - backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = nullptr; - - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_f32_ila", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32_ila", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - } - } - - if (backend_ctx->has_integer_dot) { - size_t bin_size = 0; - backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; - - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - } - } - - // gemv_noshuffle_general_q8_0_f32 + // unary_ext (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc) { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; - } - #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_CL_gemv_general { - #include "gemv_noshuffle_q8_0_f32.cl.h" + const std::string kernel_src { + #include "unary_ext.cl.h" }; #else - const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q8_0_f32.cl"); + const std::string kernel_src = read_file("unary_ext.cl"); #endif - - cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_splitk", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); +#define CL_UNARY_EXT_K(op) \ + CL_CHECK((backend_ctx->kernel_##op##_f32 = clCreateKernel(prog, "kernel_" #op "_f32", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f32_4 = clCreateKernel(prog, "kernel_" #op "_f32_4", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f32_nc = clCreateKernel(prog, "kernel_" #op "_f32_nc", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16 = clCreateKernel(prog, "kernel_" #op "_f16", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16_4 = clCreateKernel(prog, "kernel_" #op "_f16_4", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16_nc = clCreateKernel(prog, "kernel_" #op "_f16_nc", &err), err)); + CL_UNARY_EXT_K(sgn) + CL_UNARY_EXT_K(step) + CL_UNARY_EXT_K(elu) + CL_UNARY_EXT_K(hardswish) + CL_UNARY_EXT_K(hardsigmoid) + CL_UNARY_EXT_K(floor) + CL_UNARY_EXT_K(ceil) + CL_UNARY_EXT_K(round) + CL_UNARY_EXT_K(trunc) +#undef CL_UNARY_EXT_K CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q4_k_f32 + // softplus { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q4_k_f32.cl.h" + #include "softplus.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q4_k_f32.cl"); + const std::string kernel_src = read_file("softplus.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_r1", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_kimg", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_cok", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_softplus_f32 = clCreateKernel(prog, "kernel_softplus_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_softplus_f32_4 = clCreateKernel(prog, "kernel_softplus_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_softplus_f32_nc = clCreateKernel(prog, "kernel_softplus_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_softplus_f16 = clCreateKernel(prog, "kernel_softplus_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_softplus_f16_4 = clCreateKernel(prog, "kernel_softplus_f16_4", &err), err)); + CL_CHECK((backend_ctx->kernel_softplus_f16_nc = clCreateKernel(prog, "kernel_softplus_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM) - if (backend_ctx->has_integer_dot) { + // upscale + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h" + #include "upscale.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q4_k_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("upscale.cl"); #endif - // Per-device dp4a dense tile. The X2-tuned TILESIZE_N=32 over-occupies LDS on - // X1 (1152 B/WG -> few resident WGs); TILESIZE_N=8 (288 B) lifts occupancy on - // X1, byte-identical. X2E keeps 32. Env override wins. - int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32; - if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) { q4k_dp4a_ts = atoi(e); } - std::string dp4a_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(q4k_dp4a_ts); - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), dp4a_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); + if (!kernel_src.empty()) { + backend_ctx->program_upscale = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_upscale = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale", &err), err)); + if (backend_ctx->program_upscale) { + cl_int err_bilinear; + backend_ctx->kernel_upscale_bilinear = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale_bilinear", &err_bilinear); + if (err_bilinear != CL_SUCCESS) { + GGML_LOG_WARN("ggml_opencl: kernel_upscale_bilinear not found in upscale.cl. Bilinear upscale will not be available. Error: %d\n", err_bilinear); + backend_ctx->kernel_upscale_bilinear = nullptr; + } + } else { + backend_ctx->kernel_upscale_bilinear = nullptr; + } + GGML_LOG_CONT("."); + } else { + GGML_LOG_WARN("ggml_opencl: upscale kernel source not found or empty. Upscale operations will not be available.\n"); + backend_ctx->program_upscale = nullptr; + backend_ctx->kernel_upscale = nullptr; + backend_ctx->kernel_upscale_bilinear = nullptr; + } } - // gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM) - if (backend_ctx->has_integer_dot) { + // concat + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h" + #include "concat.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q8_0_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("concat.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_concat_b1 = clCreateKernel(prog, "kernel_concat_b1", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b2 = clCreateKernel(prog, "kernel_concat_b2", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b4 = clCreateKernel(prog, "kernel_concat_b4", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b8 = clCreateKernel(prog, "kernel_concat_b8", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b4_pack = clCreateKernel(prog, "kernel_concat_b4_pack", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K) - if (backend_ctx->has_integer_dot) { + // timestep_embedding + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h" + #include "tsembd.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q5_k_q8_1_dp4a.cl"); + + const std::string kernel_src = read_file("tsembd.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); + if (!kernel_src.empty()) { + backend_ctx->program_tsembd = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_timestep_embedding = clCreateKernel(backend_ctx->program_tsembd, "kernel_timestep_embedding", &err), err)); + GGML_LOG_CONT("."); + } else { + GGML_LOG_WARN("ggml_opencl: timestep_embedding kernel source not found or empty. This op will not be available.\n"); + backend_ctx->program_tsembd = nullptr; + backend_ctx->kernel_timestep_embedding = nullptr; + } } - // gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output) - if (backend_ctx->has_integer_dot) { + // set_rows + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h" + #include "set_rows.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q6_k_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("set_rows.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + backend_ctx->program_set_rows = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_set_rows_f32_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_f32_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_f16_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_f16_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i32", &err), err)); GGML_LOG_CONT("."); } - // quant_a_q8_1 (plain activation q8_1 pre-pass for the dense dp4a GEMM) + // conv2d + { + #ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "conv2d.cl.h" + }; + const std::string kernel_src_f16_f32 { + #include "conv2d_f16_f32.cl.h" + }; + #else + const std::string kernel_src = read_file("conv2d.cl"); + const std::string kernel_src_f16_f32 = read_file("conv2d_f16_f32.cl"); + #endif + if (!kernel_src.empty()) { + backend_ctx->program_conv_2d_f16 = + build_program_from_source(backend_ctx, kernel_src.c_str(), (std::string(compile_opts) + " -DUSE_FP16=1").c_str()); + CL_CHECK((backend_ctx->kernel_conv_2d_f16 = clCreateKernel(backend_ctx->program_conv_2d_f16, "kernel_conv_2d", &err), err)); + GGML_LOG_CONT("."); + backend_ctx->program_conv_2d_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_conv_2d_f32 = clCreateKernel(backend_ctx->program_conv_2d_f32, "kernel_conv_2d", &err), err)); + GGML_LOG_CONT("."); + } else { + GGML_LOG_WARN("ggml_opencl: conv2d kernel source not found or empty. This op will not be available.\n"); + backend_ctx->program_conv_2d_f16 = nullptr; + backend_ctx->kernel_conv_2d_f16 = nullptr; + backend_ctx->program_conv_2d_f32 = nullptr; + backend_ctx->kernel_conv_2d_f32 = nullptr; + } + if (!kernel_src_f16_f32.empty()) { + backend_ctx->program_conv_2d_f16_f32 = + build_program_from_source(backend_ctx, kernel_src_f16_f32.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_conv_2d_f16_f32 = clCreateKernel(backend_ctx->program_conv_2d_f16_f32, "kernel_conv_2d", &err), err)); + GGML_LOG_CONT("."); + } else { + GGML_LOG_WARN("ggml_opencl: conv2d_f16_f32 kernel source not found or empty. This op will not be available.\n"); + backend_ctx->program_conv_2d_f16_f32 = nullptr; + backend_ctx->kernel_conv_2d_f16_f32 = nullptr; + } + } + + // conv2_dw { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "quant_a_q8_1.cl.h" + #include "conv2_dw.cl.h" }; #else - const std::string kernel_src = read_file("quant_a_q8_1.cl"); + const std::string kernel_src = read_file("conv2_dw.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_quant_a_q8_1 = clCreateKernel(prog, "kernel_quant_a_q8_1", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_conv2d_dw_f32_f32 = clCreateKernel(prog, "kernel_conv2d_dw_f32_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_conv2d_dwf16_f32 = clCreateKernel(prog, "kernel_conv2d_dwf16_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q4_k_f32 + // ssm_conv { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } - // Opt-in: dequant-once-per-block mc3 verify GEMV (factors q4_K dequant - // out of the 3-column loop; byte-identical, lower spill). A/B vs the - // shipped inline mc3 in the same binary. - if (getenv("GGML_OPENCL_Q4K_MC3_DQ")) { - CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_ONCE "; - } - // Opt-in: LDS-staged dequant mc3 verify GEMV (stages the dequantized - // q4_K weights in __local instead of private regs that spill to slow - // global on Adreno; byte-identical). A/B vs inline + dequant-once. - if (getenv("GGML_OPENCL_Q4K_MC3_LDS")) { - CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_LDS "; - } - #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q4_k_f32.cl.h" + #include "ssm_conv.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32.cl"); + const std::string kernel_src = read_file("ssm_conv.cl"); #endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_mc3", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_splitk", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_splitk_reduce_f32 = clCreateKernel(prog, "kernel_gemv_splitk_reduce_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_glu", &err), err)); + CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32 = clCreateKernel(prog, "kernel_ssm_conv_f32_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32_4 = clCreateKernel(prog, "kernel_ssm_conv_f32_f32_4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q4_k_f32_o4 — 4-output-per-WI variant for the long-vocab - // q4_K lm_head/embed GEMV (shares one activation read across 4 output rows). + // ssm_scan { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q4_k_f32_o4.cl.h" + #include "ssm_scan.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_o4.cl"); + const std::string kernel_src = read_file("ssm_scan.cl"); #endif - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_ssm_scan_f32 = clCreateKernel(prog, "kernel_ssm_scan_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d128 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d128", &err), err)); + CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d256 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d256", &err), err)); + + cl_kernel * kernels[] = { + &backend_ctx->kernel_ssm_scan_f32_mamba2_d128, + &backend_ctx->kernel_ssm_scan_f32_mamba2_d256 + }; + + // specialized kernels use subgroups and assume subgroup size is 64, + // if device does not support subgroups or subgroup size is not 64, + // release these kernels + for (int i = 0; i < 2; ++i) { + size_t subgroup_size = 0; +#if CL_TARGET_OPENCL_VERSION >= 210 + const size_t local_work_size[] = { 64, 1 }; + const cl_int subgroup_err = clGetKernelSubGroupInfo(*kernels[i], backend_ctx->device, CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, + sizeof(local_work_size), local_work_size, sizeof(subgroup_size), &subgroup_size, nullptr); + if (subgroup_err != CL_SUCCESS) { + subgroup_size = 0; + } +#endif + // The specialized kernels reduce over one 64-lane subgroup. + if (subgroup_size != 64) { + CL_CHECK(clReleaseKernel(*kernels[i])); + *kernels[i] = nullptr; + } + } + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gated_delta_net: one kernel per (S_V, KDA, tgpp) triple. + { + #ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gated_delta_net.cl.h" + }; + #else + const std::string kernel_src = read_file("gated_delta_net.cl"); + #endif + + const int gdn_sizes[4] = { 16, 32, 64, 128 }; + const int sg_size = backend_ctx->gpu_family == GPU_FAMILY::ADRENO ? 64 : backend_ctx->gpu_family == GPU_FAMILY::INTEL ? 32 : -1; + if (sg_size < 0) { + GGML_LOG_ERROR("Unsupported GPU Family: only Adreno and Intel are supported.\n"); + exit(1); + } + + for (int si = 0; si < 4; si++) { + const int S_V = gdn_sizes[si]; + + // MUST match the dispatcher heuristic in ggml_cl_gated_delta_net exactly. + int lanes_per_column; + if (S_V >= 128) { + lanes_per_column = 8; + } else { + lanes_per_column = std::min(S_V, sg_size); + } + + // Round LANES_PER_COLUMN down until it is: + // * power-of-two + // * divides both S_V and sg_size + while (lanes_per_column > 1 && + (((lanes_per_column & (lanes_per_column - 1)) != 0) || + (S_V % lanes_per_column) != 0 || + (sg_size % lanes_per_column) != 0)) { + lanes_per_column >>= 1; + } + + GGML_ASSERT(lanes_per_column >= 1); + GGML_ASSERT(((lanes_per_column & (lanes_per_column - 1)) == 0)); + GGML_ASSERT((S_V % lanes_per_column) == 0); + GGML_ASSERT((sg_size % lanes_per_column) == 0); + + const bool is_partial_reduce = (lanes_per_column != 1) && (lanes_per_column < sg_size); + int use_qcom_shuffle = 0; + if (is_partial_reduce) { + if (backend_ctx->has_qcom_subgroup_shuffle) { + use_qcom_shuffle = 1; + } + } + for (int kda = 0; kda < 2; kda++) { + for (int tgpp = 0; tgpp < 2; tgpp++) { + const int cpl = (tgpp == 0) ? 1 : 4; + const int spw = (tgpp == 0) ? 1 : 1; + + std::string opts = compile_opts; + opts += " -DS_V=" + std::to_string(S_V); + opts += " -DKDA=" + std::to_string(kda); + opts += " -DSUBGROUP_SIZE=" + std::to_string(sg_size); + opts += " -DLANES_PER_COLUMN=" + std::to_string(lanes_per_column); + opts += " -DCOLS_PER_LANE_GROUP=" + std::to_string(cpl); + opts += " -DUSE_QCOM_SUBGROUP_SHUFFLE=" + std::to_string(use_qcom_shuffle); + + // Since spw=1 is found to be optimal, SUBGROUPS_PER_WG > 1 code in + // the kernel is removed. If you want to experiment with spw > 1, + // Please remember to implement code to handle it. + opts += " -DSUBGROUPS_PER_WG=" + std::to_string(spw); + + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + + CL_CHECK((backend_ctx->kernel_gated_delta_net_f32[si][kda][tgpp] = + clCreateKernel(prog, "kernel_gated_delta_net", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + } + } } + GGML_LOG_CONT("."); + } + + // moe_combine (fused router-weight mul + cross-expert sum) + { + #ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_combine.cl.h" + }; + #else + const std::string kernel_src = read_file("moe_combine.cl"); + #endif cl_program prog = build_program_from_source( - backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_o4", &err), err)); + backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_moe_combine_f32 = + clCreateKernel(prog, "kernel_moe_combine_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_combine_bias_f32 = + clCreateKernel(prog, "kernel_moe_combine_bias_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q4_k_f32_tiled — tiled-wide canonical layout, default ON - // (opt out: GGML_OPENCL_Q4K_GEMV_TILED=0; separate convert + GEMV; weights via __global). + // mul_mv_id_f32_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q4_k_f32_tiled.cl.h" + #include "mul_mv_id_f32_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_tiled.cl"); + const std::string kernel_src = read_file("mul_mv_id_f32_f32.cl"); #endif - std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled = - clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_tiled", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_id_f32_f32 = clCreateKernel(prog, "kernel_mul_mv_id_f32_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr; - backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr; - backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr; + // mul_mm_id_f32_f32 { - if (backend_ctx->has_vector_subgroup_broadcast) { - std::string opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "gemv_noshuffle_q4_k_f32_32b_trans.cl.h" - }; + const std::string kernel_src { + #include "mul_mm_id_f32_f32.cl.h" + }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_32b_trans.cl"); + const std::string kernel_src = read_file("mul_mm_id_f32_f32.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = - clCreateKernel(prog, "gemv_noshuffle_q4_k_f32_32b_trans", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - - if (use_adreno_bin_kernels(backend_ctx)) { - size_t bin_size = 0; - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program bin_prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = - clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(bin_prog)); - GGML_LOG_CONT("."); - } + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_id_f32_f32 = clCreateKernel(prog, "kernel_mul_mm_id_f32_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program bin_prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + // mul_mv_id_q4_0_f32_8x_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_id_q4_0_f32_8x_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_id_q4_0_f32_8x_flat.cl"); +#endif + backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = - clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(bin_prog)); - GGML_LOG_CONT("."); - } - } + CL_CHECK((backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat, "kernel_mul_mv_id_q4_0_f32_8x_flat", &err), err)); + GGML_LOG_CONT("."); } - backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = nullptr; - backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = nullptr; - backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = nullptr; + // mul_mv_id_q8_0_f32 { - if (backend_ctx->has_vector_subgroup_broadcast) { - std::string opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "gemv_noshuffle_q6_k_f32_32b_trans.cl.h" - }; + const std::string kernel_src { + #include "mul_mv_id_q8_0_f32.cl.h" + }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_32b_trans.cl"); + const std::string kernel_src = read_file("mul_mv_id_q8_0_f32.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = - clCreateKernel(prog, "kernel_gemv_noshuffle_q6_k_f32_32b_trans", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - - if (use_adreno_bin_kernels(backend_ctx)) { - size_t bin_size = 0; - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program bin_prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = - clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(bin_prog)); - GGML_LOG_CONT("."); - } - - kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program bin_prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = - clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(bin_prog)); - GGML_LOG_CONT("."); - } - } - } - - backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = nullptr; - backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = nullptr; - backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = nullptr; - { - if (backend_ctx->has_vector_subgroup_broadcast) { - std::string opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); -#ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "gemv_noshuffle_q5_k_f32_32b_trans.cl.h" - }; -#else - const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32_32b_trans.cl"); -#endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = - clCreateKernel(prog, "gemv_noshuffle_q5_k_f32_32b_trans", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - - if (use_adreno_bin_kernels(backend_ctx)) { - size_t bin_size = 0; - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_f32_32b_trans_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program bin_prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = - clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(bin_prog)); - GGML_LOG_CONT("."); - } - - kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program bin_prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + backend_ctx->program_mul_mv_id_q8_0_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = - clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &err), err)); - CL_CHECK(clReleaseProgram(bin_prog)); - GGML_LOG_CONT("."); - } - } + CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32, "kernel_mul_mv_id_q8_0_f32", &err), err)); + GGML_LOG_CONT("."); } - std::string CL_moe_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -cl-fast-relaxed-math"; - - // gemv_moe_q4_1_f32_ns + // mul_mv_id_q8_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_q4_1_f32_ns.cl.h" + #include "mul_mv_id_q8_0_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_q4_1_f32_ns.cl"); + const std::string kernel_src = read_file("mul_mv_id_q8_0_f32_flat.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + backend_ctx->program_mul_mv_id_q8_0_f32_flat = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_moe_q4_1_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_1_f32_ns", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32_flat, "kernel_mul_mv_id_q8_0_f32_flat", &err), err)); GGML_LOG_CONT("."); } - // gemm_moe_q4_1_f32_ns + // mul_mv_id_mxfp4_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q4_1_f32_ns.cl.h" + #include "mul_mv_id_mxfp4_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q4_1_f32_ns.cl"); + const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + backend_ctx->program_mul_mv_id_mxfp4_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32, "kernel_mul_mv_id_mxfp4_f32", &err), err)); GGML_LOG_CONT("."); } - // gemm_moe_q4_1_f32_ns_bin - { - size_t bin_size = 0; - backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = nullptr; - - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_1_f32_ns_ila", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns_ila", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - } - } - - // gemv_moe_mxfp4_f32 + // mul_mv_id_mxfp4_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_mxfp4_f32.cl.h" + #include "mul_mv_id_mxfp4_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_mxfp4_f32.cl"); + const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32_flat.cl"); #endif - backend_ctx->program_gemv_moe_mxfp4_f32 = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + backend_ctx->program_mul_mv_id_mxfp4_f32_flat = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemv_moe_mxfp4_f32, "kernel_gemv_moe_mxfp4_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32_flat, "kernel_mul_mv_id_mxfp4_f32_flat", &err), err)); GGML_LOG_CONT("."); } - // gemm_moe_mxfp4_f32 + // Adreno kernels +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // transpose { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_mxfp4_f32.cl.h" + #include "transpose.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_mxfp4_f32.cl"); + const std::string kernel_src = read_file("transpose.cl"); #endif - backend_ctx->program_gemm_moe_mxfp4_f32 = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + backend_ctx->program_transpose = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemm_moe_mxfp4_f32, "kernel_gemm_moe_mxfp4_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_transpose_32_16 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32_16", &err), err)); + CL_CHECK((backend_ctx->kernel_transpose_32 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32", &err), err)); + CL_CHECK((backend_ctx->kernel_transpose_16 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16", &err), err)); + CL_CHECK((backend_ctx->kernel_transpose_8_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_8_buf", &err), err)); + CL_CHECK((backend_ctx->kernel_transpose_16_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16_buf", &err), err)); + CL_CHECK((backend_ctx->kernel_transpose_32_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32_buf", &err), err)); + CL_CHECK((backend_ctx->kernel_transpose_16_4x1 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16_4x1", &err), err)); GGML_LOG_CONT("."); } - // gemv_moe_q4_0_f32_ns + // gemm_noshuffle_q1_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_q4_0_f32_ns.cl.h" + #include "gemm_noshuffle_q1_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_q4_0_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q1_0_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_moe_q4_0_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_0_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q1_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_q4_0_f32_ns + // gemv_noshuffle_q1_0_f32 { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); + #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "gemm_moe_q4_0_f32_ns.cl.h" + const std::string kernel_src_CL_gemv_general { + #include "gemv_noshuffle_q1_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q4_0_f32_ns.cl"); + const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q1_0_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - // gemm_moe_q4_0_f32_ns_bin - { - size_t bin_size = 0; - backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = nullptr; + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_f32_ns_ila", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns_ila", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } + const char * nsg1_min_m_env = getenv("GGML_OPENCL_Q1_0_GEMV_NSG1_MIN_M"); + backend_ctx->min_m_gemv_nsg1_q1_0 = nsg1_min_m_env ? atoi(nsg1_min_m_env) : + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? 8192 : 0); + if (backend_ctx->min_m_gemv_nsg1_q1_0 > 0) { + const std::string nsg1_opts = CL_gemv_compile_opts + " -DN_SIMDGROUP=1"; + cl_program prog_nsg1 = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), nsg1_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q1_0_f32_nsg1 = clCreateKernel(prog_nsg1, "kernel_gemv_noshuffle_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog_nsg1)); } + GGML_LOG_CONT("."); } - // gemm_moe_q8_0_f32_ns + // gemm_noshuffle_q3_k_q8_1_dp4a { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q8_0_f32_ns.cl.h" + #include "gemm_noshuffle_q3_k_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q8_0_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q3_k_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q8_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q8_0_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_moe_q5_0_f32_ns + // gemm_noshuffle_q2_0_q8_1_dp4a { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_q5_0_f32_ns.cl.h" + #include "gemm_noshuffle_q2_0_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_q5_0_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q2_0_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_moe_q5_0_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_0_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_0_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_0_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_0_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_q5_0_f32_ns + // gemm_noshuffle_iq4_xs_q8_1_dp4a { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q5_0_f32_ns.cl.h" + #include "gemm_noshuffle_iq4_xs_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q5_0_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq4_xs_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q5_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_0_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_moe_q5_1_f32_ns + // gemm_noshuffle_iq4_xs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_q5_1_f32_ns.cl.h" + #include "gemm_noshuffle_iq4_xs_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_q5_1_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq4_xs_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_moe_q5_1_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_1_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_q5_1_f32_ns + // gemm_noshuffle_q2_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q5_1_f32_ns.cl.h" + #include "gemm_noshuffle_q2_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q5_1_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q2_0_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q5_1_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_1_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_moe_q4_k_f32_ns + // gemv_noshuffle_iq4_xs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_q4_k_f32_ns.cl.h" + #include "gemv_noshuffle_iq4_xs_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_q4_k_f32_ns.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_iq4_xs_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns_wimg", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + if (const char * env = getenv("GGML_OPENCL_IQ4_XS_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq4_xs_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ4_XS_MV_NSG=" + std::to_string(backend_ctx->iq4_xs_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq4_xs_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq4_xs_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq4_xs_f32 = kernel; + CL_CHECK(clReleaseProgram(prog)); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq4_xs_mv_nsg > 1); + backend_ctx->iq4_xs_mv_nsg /= 2; + } GGML_LOG_CONT("."); } - // gemm_moe_q4_k_f32_ns + // gemv_noshuffle_q2_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q4_k_f32_ns.cl.h" + #include "gemv_noshuffle_q2_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q4_k_f32_ns.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_q2_0_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - - // gemm_moe_q4_k_f32_ns_bin - { - size_t bin_size = 0; - backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = nullptr; - - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_f32_ns_ila", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns_ila", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); + if (const char * env = getenv("GGML_OPENCL_Q2_0_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8 || nsg == 16 || nsg == 32) { + backend_ctx->q2_0_mv_nsg = nsg; + } + } + for (int wide = 0; wide < 2; ++wide) { + const std::string opts = compile_opts + " -DQ2_0_MV_WIDE=" + std::to_string(wide); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_0_f32", &err); + CL_CHECK(err); + CL_CHECK(clReleaseProgram(prog)); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + size_t workgroup = 512; + while (workgroup > max_workgroup) { + workgroup /= 2; + } + GGML_ASSERT(workgroup >= 64); + if (wide) { + backend_ctx->kernel_gemv_noshuffle_q2_0_f32_wide = kernel; + backend_ctx->q2_0_mv_workgroup_size_wide = workgroup; + } else { + backend_ctx->kernel_gemv_noshuffle_q2_0_f32 = kernel; + backend_ctx->q2_0_mv_workgroup_size = workgroup; } } + GGML_LOG_CONT("."); } - // gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM) - if (backend_ctx->has_integer_dot) { + // gemm_noshuffle_iq1_s_q8_1_dp4a + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q4_k_q8_1_dp4a.cl.h" + #include "gemm_noshuffle_iq1_s_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q4_k_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq1_s_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq1_s_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq1_s_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq1_s_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq1_s_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM) - if (backend_ctx->has_integer_dot) { + // gemm_noshuffle_iq1_s_f32 + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_mxfp4_q8_1_dp4a.cl.h" + #include "gemm_noshuffle_iq1_s_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_mxfp4_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq1_s_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq1_s_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq1_s_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_mxfp4_q8_1_dp4a_bin (dp4a prefill GEMM) - if (backend_ctx->has_integer_dot) { - size_t bin_size = 0; - backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; - - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila", &err), err)); + // gemv_noshuffle_iq1_s_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_iq1_s_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_iq1_s_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_IQ1_S_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq1_s_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ1_S_MV_NSG=" + std::to_string(backend_ctx->iq1_s_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq1_s_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq1_s_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq1_s_f32 = kernel; + cl_kernel export_grid = clCreateKernel(prog, "kernel_iq1_s_grid_export", &err); + CL_CHECK(err); + backend_ctx->iq1_s_grid_buffer.allocate(backend_ctx->context, 2048*sizeof(cl_uint)); + CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq1_s_grid_buffer.buffer)); + size_t global[] = { 2048 }; + size_t local[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseKernel(export_grid)); CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); + + cl_image_format format = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = 2048; + desc.buffer = backend_ctx->iq1_s_grid_buffer.buffer; + backend_ctx->iq1_s_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); + CL_CHECK(err); + break; } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq1_s_mv_nsg > 1); + backend_ctx->iq1_s_mv_nsg /= 2; } + GGML_LOG_CONT("."); } - // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM) - if (backend_ctx->has_integer_dot) { + // gemm_noshuffle_iq1_m_q8_1_dp4a + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q4_0_q8_1_dp4a.cl.h" + #include "gemm_noshuffle_iq1_m_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q4_0_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq1_m_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq1_m_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq1_m_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq1_m_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq1_m_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_q4_0_q8_1_dp4a_bin (dp4a prefill GEMM) - if (backend_ctx->has_integer_dot) { - size_t bin_size = 0; - backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; - - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_q8_1_dp4a_ila", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a_ila", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - } - } - - // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant) - if (backend_ctx->has_integer_dot) { + // gemm_noshuffle_iq1_m_f32 + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q8_1_dp4a.cl.h" + #include "gemm_noshuffle_iq1_m_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq1_m_f32.cl"); #endif - const std::string opts80 = CL_moe_compile_opts + " -DMOE_QT=80"; - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), opts80.c_str()); - CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 = clCreateKernel(prog, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq1_m_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq1_m_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); - - const std::string opts50 = CL_moe_compile_opts + " -DMOE_QT=50"; - cl_program prog50 = - build_program_from_source(backend_ctx, kernel_src.c_str(), opts50.c_str()); - CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 = clCreateKernel(prog50, "kernel_gemm_moe_q8_1_dp4a", &err), err)); - CL_CHECK(clReleaseProgram(prog50)); - - const std::string opts5 = CL_moe_compile_opts + " -DMOE_QT=5"; - cl_program prog5 = - build_program_from_source(backend_ctx, kernel_src.c_str(), opts5.c_str()); - CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k = clCreateKernel(prog5, "kernel_gemm_moe_q8_1_dp4a", &err), err)); - CL_CHECK(clReleaseProgram(prog5)); GGML_LOG_CONT("."); } - // moe_reorder_quant_a_q8_1 (fused reorder + q8_1 quant) + // gemv_noshuffle_iq1_m_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "moe_reorder_quant_a_q8_1.cl.h" + #include "gemv_noshuffle_iq1_m_f32.cl.h" }; #else - const std::string kernel_src = read_file("moe_reorder_quant_a_q8_1.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_iq1_m_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + if (const char * env = getenv("GGML_OPENCL_IQ1_M_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq1_m_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ1_M_MV_NSG=" + std::to_string(backend_ctx->iq1_m_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq1_s_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq1_m_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq1_m_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq1_m_f32 = kernel; + CL_CHECK(clReleaseProgram(prog)); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq1_m_mv_nsg > 1); + backend_ctx->iq1_m_mv_nsg /= 2; + } + GGML_LOG_CONT("."); + } - CL_CHECK((backend_ctx->kernel_moe_reorder_quant_a_q8_1 = clCreateKernel(prog, "kernel_moe_reorder_quant_a_q8_1", &err), err)); + // gemm_noshuffle_iq3_s_q8_1_dp4a + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq3_s_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq3_s_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_s_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_s_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_s_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_s_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_moe_q5_k_f32_ns + // gemm_noshuffle_iq3_s_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_q5_k_f32_ns.cl.h" + #include "gemm_noshuffle_iq3_s_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_q5_k_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq3_s_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_moe_q5_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_k_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_s_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_s_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_q5_k_f32_ns + // gemv_noshuffle_iq3_s_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q5_k_f32_ns.cl.h" + #include "gemv_noshuffle_iq3_s_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q5_k_f32_ns.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_iq3_s_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + if (const char * env = getenv("GGML_OPENCL_IQ3_S_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq3_s_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ3_S_MV_NSG=" + std::to_string(backend_ctx->iq3_s_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq3_s_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq3_s_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq3_s_f32 = kernel; + cl_kernel export_grid = clCreateKernel(prog, "kernel_iq3_s_grid_export", &err); + CL_CHECK(err); + backend_ctx->iq3_s_grid_buffer.allocate(backend_ctx->context, 512*sizeof(cl_uint)); + CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq3_s_grid_buffer.buffer)); + size_t global[] = { 512 }; + size_t local[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseKernel(export_grid)); + CL_CHECK(clReleaseProgram(prog)); - CL_CHECK((backend_ctx->kernel_gemm_moe_q5_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_k_f32_ns", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + cl_image_format format = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = 512; + desc.buffer = backend_ctx->iq3_s_grid_buffer.buffer; + backend_ctx->iq3_s_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); + CL_CHECK(err); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq3_s_mv_nsg > 1); + backend_ctx->iq3_s_mv_nsg /= 2; + } GGML_LOG_CONT("."); } - // gemv_moe_q6_k_f32_ns + // gemm_noshuffle_iq3_xxs_q8_1_dp4a { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_q6_k_f32_ns.cl.h" + #include "gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_q6_k_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_moe_q6_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q6_k_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_q6_k_f32_ns + // gemm_noshuffle_iq3_xxs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q6_k_f32_ns.cl.h" + #include "gemm_noshuffle_iq3_xxs_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q6_k_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq3_xxs_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_xxs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_xxs_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_q6_k_f32_ns_bin + // gemv_noshuffle_iq3_xxs_f32 { - size_t bin_size = 0; - backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = nullptr; - - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_f32_ns_ila", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns_ila", &err), err)); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_iq3_xxs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_iq3_xxs_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_IQ3_XXS_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq3_xxs_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ3_XXS_MV_NSG=" + std::to_string(backend_ctx->iq3_xxs_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq3_xxs_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq3_xxs_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq3_xxs_f32 = kernel; + cl_kernel export_grid = clCreateKernel(prog, "kernel_iq3_xxs_grid_export", &err); + CL_CHECK(err); + backend_ctx->iq3_xxs_grid_buffer.allocate(backend_ctx->context, 256*sizeof(cl_uint)); + CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq3_xxs_grid_buffer.buffer)); + size_t global[] = { 256 }; + size_t local[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseKernel(export_grid)); CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); + + cl_image_format format = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = 256; + desc.buffer = backend_ctx->iq3_xxs_grid_buffer.buffer; + backend_ctx->iq3_xxs_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); + CL_CHECK(err); + break; } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq3_xxs_mv_nsg > 1); + backend_ctx->iq3_xxs_mv_nsg /= 2; } + GGML_LOG_CONT("."); } - // gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM) - if (backend_ctx->has_integer_dot) { + // gemm_noshuffle_iq2_s_q8_1_dp4a + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_q6_k_q8_1_dp4a.cl.h" + #include "gemm_noshuffle_iq2_s_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_q6_k_q8_1_dp4a.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq2_s_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_s_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_s_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_s_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_s_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_moe_mxfp4_f32_ns + // gemm_noshuffle_iq2_xs_q8_1_dp4a { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_mxfp4_f32_ns.cl.h" + #include "gemm_noshuffle_iq2_xs_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_mxfp4_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq2_xs_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns_wimg", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_mxfp4_f32_ns + // gemm_noshuffle_iq2_xxs_q8_1_dp4a { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_mxfp4_f32_ns.cl.h" + #include "gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_mxfp4_f32_ns.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // gemm_moe_mxfp4_f32_ns_bin + // gemm_noshuffle_iq2_xxs_f32 { - size_t bin_size = 0; - backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = nullptr; - - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_f32_ns_ila", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - - CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns_ila", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - } +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq2_xxs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq2_xxs_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xxs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xxs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // moe_reorder_b + // gemv_noshuffle_iq2_xxs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "moe_reorder_b.cl.h" + #include "gemv_noshuffle_iq2_xxs_f32.cl.h" }; #else - const std::string kernel_src = read_file("moe_reorder_b.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_iq2_xxs_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + if (const char * env = getenv("GGML_OPENCL_IQ2_XXS_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq2_xxs_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ2_XXS_MV_NSG=" + std::to_string(backend_ctx->iq2_xxs_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq2_xxs_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq2_xxs_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq2_xxs_f32 = kernel; + cl_kernel export_grid = clCreateKernel(prog, "kernel_iq2_xxs_grid_export", &err); + CL_CHECK(err); + backend_ctx->iq2_xxs_grid_buffer.allocate(backend_ctx->context, 512*sizeof(cl_uint)); + CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq2_xxs_grid_buffer.buffer)); + size_t global[] = { 512 }; + size_t local[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseKernel(export_grid)); + CL_CHECK(clReleaseProgram(prog)); - CL_CHECK((backend_ctx->kernel_moe_reorder_b = clCreateKernel(prog, "kernel_moe_reorder_b", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + cl_image_format format = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = 512; + desc.buffer = backend_ctx->iq2_xxs_grid_buffer.buffer; + backend_ctx->iq2_xxs_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); + CL_CHECK(err); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq2_xxs_mv_nsg > 1); + backend_ctx->iq2_xxs_mv_nsg /= 2; + } GGML_LOG_CONT("."); } - // moe_sort_by_expert + // gemm_noshuffle_iq2_xs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "moe_sort_by_expert.cl.h" + #include "gemm_noshuffle_iq2_xs_f32.cl.h" }; #else - const std::string kernel_src = read_file("moe_sort_by_expert.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq2_xs_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xs_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q6_k_f32 + // gemv_noshuffle_iq2_xs_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q6_k_f32.cl.h" + #include "gemv_noshuffle_iq2_xs_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_iq2_xs_f32.cl"); #endif - - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + if (const char * env = getenv("GGML_OPENCL_IQ2_XS_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq2_xs_mv_nsg = nsg; + } } + for (;;) { + const std::string opts = compile_opts + " -DIQ2_XS_MV_NSG=" + std::to_string(backend_ctx->iq2_xs_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq2_xs_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq2_xs_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq2_xs_f32 = kernel; + cl_kernel export_grid = clCreateKernel(prog, "kernel_iq2_xs_grid_export", &err); + CL_CHECK(err); + backend_ctx->iq2_xs_grid_buffer.allocate(backend_ctx->context, 1024*sizeof(cl_uint)); + CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq2_xs_grid_buffer.buffer)); + size_t global[] = { 1024 }; + size_t local[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseKernel(export_grid)); + CL_CHECK(clReleaseProgram(prog)); - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_mc3", &err), err)); - if (getenv("GGML_OPENCL_MC3_PROBE")) { - cl_ulong pm6 = 0, pm4 = 0; size_t wg6 = 0, wg4 = 0, mult = 0; - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm6), &pm6, NULL); - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg6), &wg6, NULL); - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm4), &pm4, NULL); - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg4), &wg4, NULL); - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE, sizeof(mult), &mult, NULL); - fprintf(stderr, "[MC3-PROBE] q4K_mc3 private=%llu wg_cap=%zu | q6K_mc3 private=%llu wg_cap=%zu | pref_mult=%zu\n", - (unsigned long long)pm4, wg4, (unsigned long long)pm6, wg6, mult); - fflush(stderr); + cl_image_format format = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = 1024; + desc.buffer = backend_ctx->iq2_xs_grid_buffer.buffer; + backend_ctx->iq2_xs_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); + CL_CHECK(err); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq2_xs_mv_nsg > 1); + backend_ctx->iq2_xs_mv_nsg /= 2; } GGML_LOG_CONT("."); } - // gemv_noshuffle_q6_k_f32_o4 — 4-output-per-WI variant, opt-in via - // GGML_OPENCL_Q6K_GEMV_O4=1 (~3x fewer dispatches on long-vocab lm_head). + // gemm_noshuffle_iq2_s_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q6_k_f32_o4.cl.h" + #include "gemm_noshuffle_iq2_s_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_o4.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq2_s_f32.cl"); #endif - - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; - } - - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_o4", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_s_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_s_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); - - // Global-read variant: weights read from __global coalesced instead of - // image1d_buffer (the texture cache caps the streaming lm_head read - // bandwidth). Opt-in via GGML_OPENCL_Q6K_GEMV_O4_GLOBAL. - cl_program prog_g = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts + " -DQ6K_O4_GLOBAL"); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global = - clCreateKernel(prog_g, "kernel_gemv_noshuffle_q6_K_f32_o4_global", &err), err)); - CL_CHECK(clReleaseProgram(prog_g)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q6_k_f32_tiled — tiled-wide canonical layout, default ON - // (opt out: GGML_OPENCL_Q6K_GEMV_TILED=0; separate convert + GEMV; weights via __global). + // gemv_noshuffle_iq2_s_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q6_k_f32_tiled.cl.h" + #include "gemv_noshuffle_iq2_s_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_tiled.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_iq2_s_f32.cl"); #endif - std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled = - clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 = - clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled_mc3", &err), err)); - CL_CHECK(clReleaseProgram(prog)); + if (const char * env = getenv("GGML_OPENCL_IQ2_S_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq2_s_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ2_S_MV_NSG=" + std::to_string(backend_ctx->iq2_s_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq2_s_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq2_s_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq2_s_f32 = kernel; + cl_kernel export_grid = clCreateKernel(prog, "kernel_iq2_s_grid_export", &err); + CL_CHECK(err); + backend_ctx->iq2_s_grid_buffer.allocate(backend_ctx->context, 2048*sizeof(cl_uint)); + CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq2_s_grid_buffer.buffer)); + size_t global[] = { 2048 }; + size_t local[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseKernel(export_grid)); + CL_CHECK(clReleaseProgram(prog)); + + cl_image_format format = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = 2048; + desc.buffer = backend_ctx->iq2_s_grid_buffer.buffer; + backend_ctx->iq2_s_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); + CL_CHECK(err); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq2_s_mv_nsg > 1); + backend_ctx->iq2_s_mv_nsg /= 2; + } GGML_LOG_CONT("."); } - // gemm_noshuffle_q6_k_f32_tiled — batched (N>1) GEMM over the same tiled-wide - // canonical layout, so batched lm_head/embed stays correct + on GPU. + // gemm_noshuffle_q3_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q6_k_f32_tiled.cl.h" + #include "gemm_noshuffle_q3_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32_tiled.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q3_k_f32.cl"); #endif - std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled = - clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_tiled", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q6_k_f32 + // gemv_noshuffle_q3_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q6_k_f32.cl.h" + #include "gemv_noshuffle_q3_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_q3_k_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_cok", &err), err)); + if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_R")) { + const int rows = atoi(env); + if (rows == 1 || rows == 2 || rows == 4) { + backend_ctx->q3_k_mv_rows = rows; + } + } + if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->q3_k_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DQ3_K_MV_R=" + std::to_string(backend_ctx->q3_k_mv_rows) + + " -DQ3_K_MV_NSG=" + std::to_string(backend_ctx->q3_k_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q3_k_f32", &err); + CL_CHECK(err); + CL_CHECK(clReleaseProgram(prog)); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->q3_k_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_q3_k_f32 = kernel; + break; + } + CL_CHECK(clReleaseKernel(kernel)); + GGML_ASSERT(backend_ctx->q3_k_mv_nsg > 1); + backend_ctx->q3_k_mv_nsg /= 2; + } GGML_LOG_CONT("."); } - // gemv_noshuffle_q5_k_f32 + // gemm_noshuffle_q2_k_f32 { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; - if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; - } - #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q5_k_f32.cl.h" + #include "gemm_noshuffle_q2_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q2_k_f32.cl"); #endif - - cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32_mc3", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q5_k_f32 - { + // gemm_noshuffle_q2_k_q8_1_dp4a + if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q5_k_f32.cl.h" + #include "gemm_noshuffle_q2_k_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q5_k_f32.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q2_k_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // repack + // gemv_noshuffle_q2_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "flash_attn_repack.cl.h" + #include "gemv_noshuffle_q2_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("flash_attn_repack.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_q2_k_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->fa.kernel_repack_q_for_wmm = clCreateKernel(prog, "kernel_repack_q_for_wmm", &err), err)); - CL_CHECK((backend_ctx->fa.kernel_repack_k_for_wmm = clCreateKernel(prog, "kernel_repack_k_for_wmm", &err), err)); - CL_CHECK((backend_ctx->fa.kernel_repack_v_for_wmm = clCreateKernel(prog, "kernel_repack_v_for_wmm", &err), err)); - CL_CHECK((backend_ctx->fa.kernel_repack_mask_for_wmm = clCreateKernel(prog, "kernel_repack_mask_for_wmm", &err), err)); + if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_R")) { + const int rows = atoi(env); + if (rows == 1 || rows == 2 || rows == 4) { + backend_ctx->q2_k_mv_rows = rows; + } + } + if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->q2_k_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DQ2_K_MV_R=" + std::to_string(backend_ctx->q2_k_mv_rows) + + " -DQ2_K_MV_NSG=" + std::to_string(backend_ctx->q2_k_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_k_f32", &err); + CL_CHECK(err); + CL_CHECK(clReleaseProgram(prog)); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->q2_k_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_q2_k_f32 = kernel; + break; + } + CL_CHECK(clReleaseKernel(kernel)); + GGML_ASSERT(backend_ctx->q2_k_mv_nsg > 1); + backend_ctx->q2_k_mv_nsg /= 2; + } GGML_LOG_CONT("."); } - // kernel_flash_attn_f32_f16_bin + // gemv_noshuffle_general { - size_t bin_size = 0; - backend_ctx->fa.kernel_flash_attn_f32_f16_bin = nullptr; + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X) { + CL_gemv_compile_opts += " -DGGML_CL_A6X_CONSTFOLD_FIX"; + } - if (use_adreno_bin_kernels(backend_ctx)) { - const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("flash_attn_f32_f16_wmm", &bin_size); - if (kernel_bin && bin_size > 0) { - cl_program prog = - build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src_CL_gemv_general { + #include "gemv_noshuffle_q4_0_f32.cl.h" + }; +#else + const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q4_0_f32.cl"); +#endif - CL_CHECK((backend_ctx->fa.kernel_flash_attn_f32_f16_bin = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - } + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_mc3", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - GGML_LOG_CONT("\n"); - backend_ctx->kernels_loaded = true; -} -static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev); -static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev); + // gemv_noshuffle + { + // Gemv 2048, 16384 + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DLINE_STRIDE_A=2048 " + " -DBLOCK_STRIDE_A=16384 " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } -// FA per-(dk,dv) tile tuning table + GGML_OPENCL_FA_TUNE override parsing. -#include "fa_tune.h" - -// FA variant key for the per-(dk,dv,variant) lazy compile cache. -// Kernel built on first dispatch to reduce kernel loading time. -// NB - a warmup run is recommended to get all necessary FA variants compiled -// before actual runs. -enum ggml_opencl_fa_variant { - FA_VARIANT_PRE = 0, // prepass kernels (kv_pad, mask_pad, blk) - FA_VARIANT_F16 = 1, - FA_VARIANT_F32 = 2, - FA_VARIANT_F32_F16 = 3, - FA_VARIANT_Q8_0 = 4, - FA_VARIANT_Q4_0 = 5, - FA_VARIANT_F32_F16_SPLIT = 6, - FA_VARIANT_Q8_0_SPLIT = 7, - FA_VARIANT_Q4_0_SPLIT = 8, -}; - -static std::string ggml_opencl_fa_kernel_src(ggml_opencl_fa_variant v) { #ifdef GGML_OPENCL_EMBED_KERNELS - switch (v) { - case FA_VARIANT_F16: - return std::string{ - #include "flash_attn_f16.cl.h" - }; - case FA_VARIANT_F32: - return std::string{ - #include "flash_attn_f32.cl.h" - }; - case FA_VARIANT_F32_F16: - case FA_VARIANT_F32_F16_SPLIT: - return std::string{ - #include "flash_attn_f32_f16.cl.h" - }; - case FA_VARIANT_PRE: - return std::string{ - #include "flash_attn_pre_f16.cl.h" - }; - case FA_VARIANT_Q8_0: - case FA_VARIANT_Q8_0_SPLIT: - return std::string{ - #include "flash_attn_f32_q8_0.cl.h" - }; - case FA_VARIANT_Q4_0: - case FA_VARIANT_Q4_0_SPLIT: - return std::string{ - #include "flash_attn_f32_q4_0.cl.h" - }; - } - return {}; + const std::string kernel_src_CL_gemv { + #include "gemv_noshuffle_q4_0_f32_spec.cl.h" + }; #else - switch (v) { - case FA_VARIANT_F16: return read_file("flash_attn_f16.cl"); - case FA_VARIANT_F32: return read_file("flash_attn_f32.cl"); - case FA_VARIANT_F32_F16: - case FA_VARIANT_F32_F16_SPLIT: return read_file("flash_attn_f32_f16.cl"); - case FA_VARIANT_PRE: return read_file("flash_attn_pre_f16.cl"); - case FA_VARIANT_Q8_0: - case FA_VARIANT_Q8_0_SPLIT: return read_file("flash_attn_f32_q8_0.cl"); - case FA_VARIANT_Q4_0: - case FA_VARIANT_Q4_0_SPLIT: return read_file("flash_attn_f32_q4_0.cl"); - } - return {}; + const std::string kernel_src_CL_gemv = read_file("gemv_noshuffle_q4_0_f32_spec.cl"); #endif -} - -static std::string ggml_opencl_fa_compile_opts(ggml_backend_opencl_context * backend_ctx, - const ggml_opencl_fa_dim * cfg, - ggml_opencl_fa_variant variant) { - std::string opts = backend_ctx->kernel_compile_opts + - " -D DK=" + std::to_string(cfg->dk) + - " -D DV=" + std::to_string(cfg->dv) + - " -D BLOCK_M=" + std::to_string(cfg->bm) + - " -D BLOCK_N=" + std::to_string(cfg->bn); - if (backend_ctx->gpu_family == INTEL) { - opts += " -D FA_SG=32"; - } + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); - const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || - variant == FA_VARIANT_Q8_0_SPLIT || - variant == FA_VARIANT_Q4_0_SPLIT; - if (is_split) { - opts += " -D N_SPLIT=" + std::to_string(cfg->n_split); - } - // Shuffle define for the split tile paths AND the cluster-parallel decode - // kernel (q1_vec_mq_split_c8) in the plain F32_F16 program. Without it the - // c8 kernel is compiled out (HAS_SUBGROUP_SHUFFLE guard) and dispatch - // falls back to the baseline mq_split. - if ((is_split || variant == FA_VARIANT_F32_F16) && backend_ctx->has_subgroup_shuffle) { - opts += backend_ctx->has_qcom_subgroup_shuffle - ? " -D cl_qcom_subgroup_shuffle=1" - : " -D cl_khr_subgroup_shuffle=1"; - } - // X1E drops the explicit sub-group size pin on the c8 kernels, compiler - // routes the fp16-heavy kernel to a slow variant with explicit subgroup size - if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) { - opts += " -D FA_C8_NO_SG_PIN"; - } - // Transposed K tile in local memory: the KV rows the QK loop walks together become - // adjacent, so a group of them is ONE 128-bit local read instead of several narrow - // ones. The QK loop is LDS-read-issue-bound (a wrong-math probe that kept every FMA/dp4a - // but removed the LDS reads ran the kernel ~40% faster), so this is worth up to +26% on - // fa=1 prefill. Output is bit-identical -- only the layout moves. - // - // DK <= 128 only. At DK=256 (gemma-3-4b) it measures 1-2% NEGATIVE and reproduces across - // rounds; padding the row stride does not recover it, so the cause is not a simple bank - // conflict and the wider tile does not want this layout. - // - // Default on within that gate; GGML_OPENCL_FA_K_LDS_T=0 restores the row-major tile. - { - const char * e = getenv("GGML_OPENCL_FA_K_LDS_T"); - if ((e == nullptr || e[0] != '0') && cfg->dk <= 128) { - opts += " -D FA_K_LDS_T"; + // Gemv 2048, 16384 + CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DLINE_STRIDE_A=2048 " + " -DBLOCK_STRIDE_A=16384 " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - } - return opts; -} - -// only register when the kernel's required dispatch workgroup size is within -// the limit of the device's maximum workgroup size -static bool ggml_opencl_fa_kernel_fits_wg(ggml_backend_opencl_context * backend_ctx, - cl_kernel kernel, size_t required_wg, - const char * name, int dk, int dv) { - if (kernel == NULL) { return false; } - const size_t dev_max = backend_ctx->max_workgroup_size; - if (dev_max < required_wg) { - GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d requires WG %zu > device max %zu; skipping registration (will fall back)\n", - name, dk, dv, required_wg, dev_max); - return false; - } - size_t kwg = 0; - cl_int err = clGetKernelWorkGroupInfo(kernel, backend_ctx->device, - CL_KERNEL_WORK_GROUP_SIZE, - sizeof(kwg), &kwg, NULL); - if (err != CL_SUCCESS) { - GGML_LOG_INFO("ggml_opencl: clGetKernelWorkGroupInfo failed for %s DK=%d DV=%d (err=%d); skipping registration\n", - name, dk, dv, err); - return false; - } - if (kwg < required_wg) { - GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d per-kernel max %zu < required %zu; skipping registration (will fall back)\n", - name, dk, dv, kwg, required_wg); - return false; - } - return true; -} -// Log private memory for an FA kernel. Enable via `GGML_OPENCL_FA_LOG_SPILL=1`. -// On Adreno non-zero private_mem means spilling to global memory due to resource -// constraint and usually causes performance degradation. -// (per-work-item, no cache locality) — a strong signal to pick a config -// with smaller per-thread state (e.g. larger N_SPLIT). -static void ggml_opencl_log_fa_kernel_spill(ggml_backend_opencl_context * backend_ctx, - cl_kernel kernel, const char * name, int dk, int dv) { - static const bool enabled = []{ - const char * e = std::getenv("GGML_OPENCL_FA_LOG_SPILL"); - return e && e[0] && e[0] != '0'; - }(); + prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); - if (!enabled || kernel == nullptr) { - return; - } + // Gemv 5504, 44032 + CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DLINE_STRIDE_A=5504 " + " -DBLOCK_STRIDE_A=44032 " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } - cl_ulong priv_mem = 0; - if (clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, - sizeof(priv_mem), &priv_mem, NULL) == CL_SUCCESS) { - const char * tag = priv_mem > 0 ? "SPILL" : "ok"; - GGML_LOG_INFO("ggml_opencl: [%s] %s DK=%d DV=%d private_mem=%llu bytes\n", - tag, name, dk, dv, (unsigned long long) priv_mem); - } -} + prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); -static void ggml_opencl_ensure_fa_pre_kernels(ggml_backend_opencl_context * backend_ctx, int dk, int dv) { - const std::pair<int, int> dk_dv = {dk, dv}; + // Gemv 16000, 128000 + CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DLINE_STRIDE_A=16000 " + " -DBLOCK_STRIDE_A=128000 " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); - const ggml_opencl_fa_dim * cfg = nullptr; - for (const auto & d : g_opencl_fa_dims) { - if (d.dk == dk && d.dv == dv) { - cfg = &d; break; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - } - if (cfg == nullptr) { - GGML_ABORT("ggml_opencl: no flash_attn config for DK=%d DV=%d", dk, dv); + prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // BM-tile metadata is consumed by the prefill dispatch (n_q_blocks / wg - // sizing) regardless of whether the prepass kernels are needed for this - // n_kv — set it unconditionally - backend_ctx->fa.f32_f16_bm[{dk, dv}] = cfg->bm; - backend_ctx->fa.f32_f16_bn[{dk, dv}] = cfg->bn; - backend_ctx->fa.f32_f16_wg_size[{dk, dv}] = cfg->bm; - backend_ctx->fa.bm[{dk, dv}] = cfg->bm; - backend_ctx->fa.bn[{dk, dv}] = cfg->bn; - - if (backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0) { return; } + // mul_mat_Ab_Bi_8x4 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src_CL_gemm { + #include "gemm_noshuffle_q4_0_f32.cl.h" + }; +#else + const std::string kernel_src_CL_gemm = read_file("gemm_noshuffle_q4_0_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemm.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - GGML_LOG_INFO("ggml_opencl: lazy-compiling flash_attn prepass for DK=%d DV=%d\n", dk, dv); - cl_int err; - const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_PRE); - const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_PRE); - // retry when kernel compile fails - cl_program prog_pre_f16 = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts, - /*fatal=*/false, "fa prepass f16", backend_ctx->queue); - if (!prog_pre_f16) { return; } - cl_kernel k_kv_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_kv_pad_f16", &err); - if (err != CL_SUCCESS) { clReleaseProgram(prog_pre_f16); return; } - cl_kernel k_mask_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_mask_pad_f16", &err); - if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseProgram(prog_pre_f16); return; } - cl_kernel k_blk_f16 = clCreateKernel(prog_pre_f16, "flash_attn_blk_f16", &err); - if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseKernel(k_mask_pad_f16); clReleaseProgram(prog_pre_f16); return; } - backend_ctx->fa.kv_pad_f16[{dk, dv}] = k_kv_pad_f16; - backend_ctx->fa.mask_pad_f16[{dk, dv}] = k_mask_pad_f16; - backend_ctx->fa.blk_f16[{dk, dv}] = k_blk_f16; - clReleaseProgram(prog_pre_f16); -} + backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = nullptr; + { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_0_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_0_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } -// DK=512 prefill BM-tile -static bool ggml_opencl_ensure_fa_f32_f16_prefill_512(ggml_backend_opencl_context * backend_ctx, bool split) { - const int dk = 512, dv = 512; - const std::pair<int, int> dk_dv = {dk, dv}; - auto & target = split ? backend_ctx->fa.f32_f16_split : backend_ctx->fa.f32_f16; - if (target.count(dk_dv) > 0) { return true; } + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - static bool failed[2] = { false, false }; - if (failed[split ? 1 : 0]) { return false; } + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } - const ggml_opencl_fa_dim * cfg = nullptr; - for (const auto & d : g_opencl_fa_dims) { - if (d.dk == dk && d.dv == dv) { cfg = &d; break; } - } - if (cfg == nullptr) { failed[split ? 1 : 0] = true; return false; } - if (split && cfg->n_split <= 1) { failed[1] = true; return false; } + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - const ggml_opencl_fa_variant variant = split ? FA_VARIANT_F32_F16_SPLIT : FA_VARIANT_F32_F16; - std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + " -D FA_PREFILL_ONLY"; - cl_program prog = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, - ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts, - /*fatal=*/false, split ? "fa f32_f16 prefill512 split" : "fa f32_f16 prefill512", - backend_ctx->queue); - if (!prog) { failed[split ? 1 : 0] = true; return false; } + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } - cl_int err; - cl_kernel k = clCreateKernel(prog, "flash_attn_f32_f16", &err); - if (err != CL_SUCCESS) { clReleaseProgram(prog); failed[split ? 1 : 0] = true; return false; } - target[dk_dv] = k; - if (split) { - backend_ctx->fa.f32_f16_split_wg_size[dk_dv] = cfg->bm * cfg->n_split; - backend_ctx->fa.f32_f16_split_nkv_threshold[dk_dv] = cfg->nkv_split_threshold; - } - ggml_opencl_log_fa_kernel_spill(backend_ctx, k, - split ? "flash_attn_f32_f16 (prefill512 split)" : "flash_attn_f32_f16 (prefill512)", dk, dv); - clReleaseProgram(prog); + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a7", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - // determine whether to use the K-image variant of the split tile - static const char * pkimg_build_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); - const bool pkimg_build = (pkimg_build_env != NULL) && (pkimg_build_env[0] != '0'); - if (split && pkimg_build && backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) == 0) { - std::string opts_img = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + - " -D FA_PREFILL_ONLY -D FA_K_IMG -D FA_TILE_NAME=flash_attn_f32_f16_k_img"; - cl_program prog_img = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, - ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts_img, - /*fatal=*/false, "fa f32_f16 prefill512 split k_img", backend_ctx->queue); - if (prog_img) { - cl_int err_img; - cl_kernel k_img = clCreateKernel(prog_img, "flash_attn_f32_f16_k_img", &err_img); - if (err_img == CL_SUCCESS) { - backend_ctx->fa.f32_f16_split_k_img[dk_dv] = k_img; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_img, - "flash_attn_f32_f16 (prefill512 split k_img)", dk, dv); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } - clReleaseProgram(prog_img); } } - return true; -} - -// Compile one (variant, dk, dv); memoised. false = compiler rejected. -static bool ggml_opencl_ensure_fa_variant(ggml_backend_opencl_context * backend_ctx, int dk, int dv, ggml_opencl_fa_variant variant) { - const std::pair<int, int> dk_dv = {dk, dv}; - const ggml_opencl_fa_dim * cfg = nullptr; - for (const auto & d : g_opencl_fa_dims) { - if (d.dk == dk && d.dv == dv) { - cfg = &d; break; - } - } - if (cfg == nullptr) { - return false; + // gemm_noshuffle_q4_1_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_1_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_1_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_1_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // if a variant has already been compiled - switch (variant) { - case FA_VARIANT_F16: { - if (backend_ctx->fa.f16.count(dk_dv)) { - return true; - } - break; - } - case FA_VARIANT_F32: { - if (backend_ctx->fa.f32.count(dk_dv)) { - return true; - } - break; - } - case FA_VARIANT_F32_F16: { - // The DK=512 decode-only program does not create the f32_f16 - // prefill kernel; check the q1 kernel instead so that repeated - // calls return a consistent result. - const bool decode_only = (dk == 512); - if (decode_only ? (backend_ctx->fa.f32_f16_q1.count(dk_dv) > 0) - : (backend_ctx->fa.f32_f16.count(dk_dv) > 0)) { - return true; - } - break; - } - case FA_VARIANT_Q8_0: { - if (backend_ctx->fa.f32_q8_0.count(dk_dv)) { - return true; - } - break; - } - case FA_VARIANT_Q4_0: { - if (backend_ctx->fa.f32_q4_0.count(dk_dv)) { - return true; - } - break; - } - case FA_VARIANT_F32_F16_SPLIT: { - if (backend_ctx->fa.f32_f16_split.count(dk_dv)) { - return true; - } - break; - } - case FA_VARIANT_Q8_0_SPLIT: { - if (backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { - return true; - } - break; - } - case FA_VARIANT_Q4_0_SPLIT: { - if (backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { - return true; - } - break; - } - case FA_VARIANT_PRE: { - ggml_opencl_ensure_fa_pre_kernels(backend_ctx, dk, dv); - return true; + // gemv_noshuffle_q4_1_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } - } - // not registered but attempted - meaning these kernels failed to compile - const auto attempt_key = std::make_pair(variant, dk_dv); - if (backend_ctx->fa.variant_attempted.count(attempt_key)) { - return false; - } - backend_ctx->fa.variant_attempted.insert(attempt_key); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_1_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_1_f32.cl"); +#endif - const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || - variant == FA_VARIANT_Q8_0_SPLIT || - variant == FA_VARIANT_Q4_0_SPLIT; - const bool is_quant = variant == FA_VARIANT_Q8_0 || variant == FA_VARIANT_Q8_0_SPLIT || - variant == FA_VARIANT_Q4_0 || variant == FA_VARIANT_Q4_0_SPLIT; - if (is_quant && (dk % 32 != 0 || dv % 32 != 0)) { - return false; - } - if (is_split && cfg->n_split <= 1) { - return false; - } - if ((variant == FA_VARIANT_Q8_0_SPLIT || variant == FA_VARIANT_Q4_0_SPLIT) && - ((dk / 32) % cfg->n_split != 0 || (dv / 4) % cfg->n_split != 0)) { - return false; - } + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - const std::string src = ggml_opencl_fa_kernel_src(variant); - if (src.empty()) { return false; } - std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32_mc3", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - // bypass kernels for DK=512 - const bool fa_decode_only = (variant == FA_VARIANT_F32_F16 && dk == 512); - if (fa_decode_only) { - opts += " -D FA_DECODE_ONLY -D FA_DECODE_MINIMAL"; + // gemm_noshuffle_q5_0_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_0_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_0_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // c8 cluster width (GGML_OPENCL_FA_CL_C overrides): value = GQA4 cluster - // width (kernel default 8); the g8 programs use 2x the value (default 16). - // Wider clusters halve per-lane o_acc at the cost of position streams per - // subgroup - static const int fa_cl_c_env = []{ - const char * e = std::getenv("GGML_OPENCL_FA_CL_C"); - const int x = (e && e[0]) ? atoi(e) : 0; - return (x == 8 || x == 16 || x == 32) ? x : 0; // 0 = per-gen default - }(); - // X2E needs 16 to keep per-lane o_acc at 128B (the compiler spills the - // kernel-default width); X1E does not spill, but C=16 is still a measured - // +28-30% DK128-GQA4 decode win there (X1-85, kv 4096/8192), neutral on - // DK64 / GQA1 / quant-KV. - const int fa_cl_c_gqa4 = fa_cl_c_env ? fa_cl_c_env - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || - backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E ? 16 : 0); - const std::string opts_cl_c_gqa4 = fa_cl_c_gqa4 - ? " -D FA_CL_C=" + std::to_string(fa_cl_c_gqa4) : std::string(); - const std::string fa_cl_c_g8_val = std::to_string(fa_cl_c_gqa4 ? fa_cl_c_gqa4 * 2 : 16); + // gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_0_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - const char * tag = nullptr; - switch (variant) { - case FA_VARIANT_F16: tag = "fa f16"; break; - case FA_VARIANT_F32: tag = "fa f32"; break; - case FA_VARIANT_F32_F16: tag = "fa f32_f16"; break; - case FA_VARIANT_Q8_0: tag = "fa q8_0"; break; - case FA_VARIANT_Q4_0: tag = "fa q4_0"; break; - case FA_VARIANT_F32_F16_SPLIT: tag = "fa f32_f16 split"; break; - case FA_VARIANT_Q8_0_SPLIT: tag = "fa q8_0 split"; break; - case FA_VARIANT_Q4_0_SPLIT: tag = "fa q4_0 split"; break; - default: break; + // gemv_noshuffle_q5_0_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } + +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q5_0_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q5_0_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - cl_program prog = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts + opts_cl_c_gqa4, - /*fatal=*/false, tag, backend_ctx->queue); - if (!prog) { return false; } - cl_int err; - switch (variant) { - case FA_VARIANT_F16: { - cl_kernel k, kq1; - CL_CHECK((k = clCreateKernel(prog, "flash_attn_f16", &err), err)); - CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f16_q1", &err), err)); - backend_ctx->fa.f16[{dk, dv}] = k; - backend_ctx->fa.f16_q1[{dk, dv}] = kq1; - break; + // gemm_noshuffle_q5_1_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_1_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_1_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_1_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q5_1_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - case FA_VARIANT_F32: { - cl_kernel k, kq1; - CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32", &err), err)); - CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_q1", &err), err)); - backend_ctx->fa.f32[{dk, dv}] = k; - backend_ctx->fa.f32_q1[{dk, dv}] = kq1; - break; + +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q5_1_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q5_1_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_1_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_iq4_nl_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq4_nl_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_0_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_iq4_nl_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - case FA_VARIANT_F32_F16: { - cl_kernel kq1; - // BM-tile prefill kernel is excluded from the decode-only (DK=512) - if (!fa_decode_only) { - cl_kernel k; - CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); - backend_ctx->fa.f32_f16[{dk, dv}] = k; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k, "flash_attn_f32_f16", dk, dv); - } - CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_f16_q1", &err), err)); - backend_ctx->fa.f32_f16_q1[{dk, dv}] = kq1; - ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, "flash_attn_f32_f16_q1", dk, dv); - cl_kernel k_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_split", &err); - if (err == CL_SUCCESS) { - backend_ctx->fa.f32_f16_q1_split[{dk, dv}] = k_split; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, "flash_attn_f32_f16_q1_split", dk, dv); + +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_iq4_nl_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_iq4_nl_f32.cl"); +#endif + + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_iq4_nl_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_q8_0_f32_8x4 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q8_0_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q8_0_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q8_0_f32_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_f32_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // q1_vec decode kernel (DV-split + subgroup reduce) - cl_kernel k_q1_vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, - "flash_attn_f32_f16_q1_vec", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec[{dk, dv}] = k_q1_vec; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, "flash_attn_f32_f16_q1_vec", dk, dv); - } else { - clReleaseKernel(k_q1_vec); - } - } - // KV-head-coalesced vec for high-GQA small models - cl_kernel k_q1_vec_mq = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq, 256, - "flash_attn_f32_f16_q1_vec_mq", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq[{dk, dv}] = k_q1_vec_mq; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq, "flash_attn_f32_f16_q1_vec_mq", dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq); - } - } - // KV-head-coalesced + flash-decoding split, reuses merge kernel - cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, - "flash_attn_f32_f16_q1_vec_mq_split", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split[{dk, dv}] = k_q1_vec_mq_split; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, "flash_attn_f32_f16_q1_vec_mq_split", dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq_split); - } - } - // K-image variant of MQ_GQA=4 split - cl_kernel k_q1_vec_mq_split_k_img = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_k_img, 256, - "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img[{dk, dv}] = k_q1_vec_mq_split_k_img; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_k_img, "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq_split_k_img); - } - } - // Cluster-parallel decode variant - cl_kernel k_q1_vec_mq_split_c8 = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_c8, 256, - "flash_attn_f32_f16_q1_vec_mq_split_c8", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split_c8[{dk, dv}] = k_q1_vec_mq_split_c8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq_split_c8); - } - } - cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); - if (err == CL_SUCCESS) { - backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; - } - // local-tile decode variant - if (dk == 128 && dv == 128) { - cl_kernel k_lt = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_tile", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lt, 128, - "flash_attn_f32_f16_q1_local_tile", dk, dv)) { - backend_ctx->fa.f32_f16_q1_local_tile[{dk, dv}] = k_lt; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lt, "flash_attn_f32_f16_q1_local_tile", dk, dv); - } else { - clReleaseKernel(k_lt); - } - } - // hybrid local-tile + MQ + FD-split - cl_kernel k_lmq = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_mq_split", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq, 64, - "flash_attn_f32_f16_q1_local_mq_split", dk, dv)) { - backend_ctx->fa.f32_f16_q1_local_mq_split[{dk, dv}] = k_lmq; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq, "flash_attn_f32_f16_q1_local_mq_split", dk, dv); - } else { - clReleaseKernel(k_lmq); - } - } + } + } + + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } + } + } - // second compile of the same source with -DMQ_GQA=8. - // FA_MQ_ONLY keeps only the vec_mq kernels so that the program - // compiles within the Adreno compiler's memory budget at DK>=256. - // FA_CL_C for the g8 program: MQ_GQA=8 doubles the c8 kernel's - // per-lane o_acc, so widen the cluster to keep the register - // footprint inside the 192-thread WG cap (see fa_cl_c_gqa4 above - // for the per-gen default). - const std::string opts_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3 -D FA_MQ_ONLY -D FA_CL_C=" + fa_cl_c_g8_val; - cl_program prog_g8 = fa_decode_only ? nullptr : build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8, - /*fatal=*/false, "fa f32_f16 MQ_GQA=8", backend_ctx->queue); - if (prog_g8) { - const size_t mq_g8_required_wg = 192; // Q1_WG_SIZE(64) * MQ_NSG_SPLIT(3) - cl_kernel k_q1_vec_mq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_g8, mq_g8_required_wg, - "flash_attn_f32_f16_q1_vec_mq (g8)", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_g8[{dk, dv}] = k_q1_vec_mq_g8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_g8, "flash_attn_f32_f16_q1_vec_mq_g8", dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq_g8); - } - } - cl_kernel k_q1_vec_mq_split_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8, mq_g8_required_wg, - "flash_attn_f32_f16_q1_vec_mq_split (g8)", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8[{dk, dv}] = k_q1_vec_mq_split_g8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8, "flash_attn_f32_f16_q1_vec_mq_split_g8", dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq_split_g8); - } - } - // K-image variant - cl_kernel k_q1_vec_mq_split_g8_k_img = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_k_img, mq_g8_required_wg, - "flash_attn_f32_f16_q1_vec_mq_split_k_img (g8)", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img[{dk, dv}] = k_q1_vec_mq_split_g8_k_img; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_k_img, "flash_attn_f32_f16_q1_vec_mq_split_g8_k_img", dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq_split_g8_k_img); - } - } - // Cluster-parallel decode, MQ_GQA=8 / FA_CL_C=16 specialization - cl_kernel k_q1_vec_mq_split_g8_c8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_c8, mq_g8_required_wg, - "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8)", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8[{dk, dv}] = k_q1_vec_mq_split_g8_c8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_g8_c8", dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq_split_g8_c8); - } - } - // hybrid local-tile + MQ_GQA=8 - if (dk == 128 && dv == 128) { - cl_kernel k_lmq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_local_mq_split", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq_g8, 64, - "flash_attn_f32_f16_q1_local_mq_split (g8)", dk, dv)) { - backend_ctx->fa.f32_f16_q1_local_mq_split_g8[{dk, dv}] = k_lmq_g8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq_g8, "flash_attn_f32_f16_q1_local_mq_split_g8", dk, dv); - } else { - clReleaseKernel(k_lmq_g8); - } - } - } - clReleaseProgram(prog_g8); - } - // NSG_SPLIT=2 programs for the cluster-parallel kernel: its register - // footprint caps the per-kernel WG at 128 on X2 (< the stock 256/192 - // requirement), so it can never register from the stock programs. - // With FA_CL_NCL position streams per subgroup, 2 subgroups still - // carry 16 in-flight rows per WG (baseline WG has 4). FA_MQ_ONLY - // keeps these compiles minimal; skipped when the stock program c8 - // registered (some other device) or shuffles are absent. - if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && - backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count({dk, dv}) == 0) { - const std::string opts_c8_ns2 = opts + " -D FA_MQ_ONLY -D MQ_GQA=4 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2" + opts_cl_c_gqa4; - cl_program prog_c8 = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8_ns2, - /*fatal=*/false, "fa f32_f16 c8 NSG2", backend_ctx->queue); - if (prog_c8) { - cl_kernel k_c8 = clCreateKernel(prog_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); - if (err == CL_SUCCESS) { - // WG = MQ_NSG(2) × Q1_WG_SIZE(=FA_SG): 128 Adreno (64), 64 Intel (32). - const size_t c8_ns2_wg = backend_ctx->gpu_family == INTEL ? 64 : 128; - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8, c8_ns2_wg, - "flash_attn_f32_f16_q1_vec_mq_split_c8 (ns2)", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2[{dk, dv}] = k_c8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8_ns2", dk, dv); - } else { - clReleaseKernel(k_c8); - } - } - clReleaseProgram(prog_c8); - } - } - // FA_CL_C=32 g8 program for the DK=DV=256 GQA=8 - if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && - dk == 256 && dv == 256) { - const std::string opts_g8_c32 = opts + " -D FA_MQ_ONLY -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2 -D FA_CL_C=32"; - cl_program prog_g8_c32 = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8_c32, - /*fatal=*/false, "fa f32_f16 c32 g8 d256 NSG2", backend_ctx->queue); - if (prog_g8_c32) { - cl_kernel k_g8_c32 = clCreateKernel(prog_g8_c32, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8_c32, 128, - "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8 c32 d256)", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32[{dk, dv}] = k_g8_c32; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8_c32, "flash_attn_f32_f16_q1_vec_mq_split_g8_c32", dk, dv); - } else { - clReleaseKernel(k_g8_c32); - } - } - clReleaseProgram(prog_g8_c32); - } - } - if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count({dk, dv}) == 0) { - const std::string opts_g8_c8_ns2 = opts + " -D FA_MQ_ONLY -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2 -D FA_CL_C=" + fa_cl_c_g8_val; - cl_program prog_g8_c8 = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8_c8_ns2, - /*fatal=*/false, "fa f32_f16 c8 g8 NSG2", backend_ctx->queue); - if (prog_g8_c8) { - cl_kernel k_g8_c8 = clCreateKernel(prog_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8_c8, 128, - "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8 ns2)", dk, dv)) { - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2[{dk, dv}] = k_g8_c8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_g8_c8_ns2", dk, dv); - } else { - clReleaseKernel(k_g8_c8); - } - } - clReleaseProgram(prog_g8_c8); - } - } - break; + // gemv_noshuffle_general_q8_0_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } - case FA_VARIANT_Q8_0: - case FA_VARIANT_Q4_0: { - const bool is_q8 = variant == FA_VARIANT_Q8_0; - const std::string base = is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0"; - const std::string name_q1 = base + "_q1"; - const std::string name_q1_split = base + "_q1_split"; - auto & m_q1 = is_q8 ? backend_ctx->fa.f32_q8_0_q1 : backend_ctx->fa.f32_q4_0_q1; - auto & m_prefill = is_q8 ? backend_ctx->fa.f32_q8_0 : backend_ctx->fa.f32_q4_0; - auto & m_q1_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_split : backend_ctx->fa.f32_q4_0_q1_split; - cl_kernel k, kq1; - CL_CHECK((kq1 = clCreateKernel(prog, name_q1.c_str(), &err), err)); - CL_CHECK((k = clCreateKernel(prog, base.c_str(), &err), err)); - m_q1[{dk, dv}] = kq1; - m_prefill[{dk, dv}] = k; - ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, name_q1.c_str(), dk, dv); - ggml_opencl_log_fa_kernel_spill(backend_ctx, k, base.c_str(), dk, dv); - cl_kernel k_split = clCreateKernel(prog, name_q1_split.c_str(), &err); - if (err == CL_SUCCESS) { - m_q1_split[{dk, dv}] = k_split; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, name_q1_split.c_str(), dk, dv); - } +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src_CL_gemv_general { + #include "gemv_noshuffle_q8_0_f32.cl.h" + }; +#else + const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q8_0_f32.cl"); +#endif - // DV-split decode variant (q1_vec) - auto & m_q1_vec = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec : backend_ctx->fa.f32_q4_0_q1_vec; - const std::string name_q1_vec = name_q1 + "_vec"; - cl_kernel k_q1_vec = clCreateKernel(prog, name_q1_vec.c_str(), &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, - name_q1_vec.c_str(), dk, dv)) { - m_q1_vec[{dk, dv}] = k_q1_vec; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, name_q1_vec.c_str(), dk, dv); - } else { - clReleaseKernel(k_q1_vec); - } - } - - // KV-head-coalesced + flash-decoding split - auto & m_mq_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split - : backend_ctx->fa.f32_q4_0_q1_vec_mq_split; - const std::string name_mq_split = name_q1 + "_vec_mq_split"; - cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, name_mq_split.c_str(), &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, - name_mq_split.c_str(), dk, dv)) { - m_mq_split[{dk, dv}] = k_q1_vec_mq_split; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, name_mq_split.c_str(), dk, dv); - } else { - clReleaseKernel(k_q1_vec_mq_split); - } - } - if (!backend_ctx->fa.f32_merge.count({dk, dv})) { - cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); - if (err == CL_SUCCESS) { - backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; - } - } - // Second compile with MQ_GQA=8, MQ_NSG=3, MQ_NSG_SPLIT=3 - auto & m_mq_split_g8 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8 - : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8; - const std::string opts_mq_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3"; - cl_program prog_mq_g8 = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts_mq_g8, - /*fatal=*/false, is_q8 ? "fa q8_0 MQ_GQA=8" : "fa q4_0 MQ_GQA=8", - backend_ctx->queue); - if (prog_mq_g8) { - const size_t mq_g8_required_wg = 192; - cl_kernel k_g8 = clCreateKernel(prog_mq_g8, name_mq_split.c_str(), &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8, mq_g8_required_wg, - name_mq_split.c_str(), dk, dv)) { - m_mq_split_g8[{dk, dv}] = k_g8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8, name_mq_split.c_str(), dk, dv); - } else { - clReleaseKernel(k_g8); - } - } - clReleaseProgram(prog_mq_g8); - } - // GQA=4 cluster-parallel program (NSG_SPLIT=2 / WG=128) - if (backend_ctx->has_subgroup_shuffle) { - auto & m_c8_gqa4 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8 - : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8; - const std::string name_c8_gqa4 = name_q1 + "_vec_mq_split_c8"; - const std::string opts_c8_gqa4 = opts + " -D MQ_GQA=4 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2" + opts_cl_c_gqa4; - cl_program prog_c8_gqa4 = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8_gqa4, - /*fatal=*/false, is_q8 ? "fa q8_0 c8 GQA4 NSG2" : "fa q4_0 c8 GQA4 NSG2", - backend_ctx->queue); - if (prog_c8_gqa4) { - cl_kernel k_c8_gqa4 = clCreateKernel(prog_c8_gqa4, name_c8_gqa4.c_str(), &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8_gqa4, 128, - name_c8_gqa4.c_str(), dk, dv)) { - m_c8_gqa4[{dk, dv}] = k_c8_gqa4; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8_gqa4, name_c8_gqa4.c_str(), dk, dv); - } else { - clReleaseKernel(k_c8_gqa4); - } - } - clReleaseProgram(prog_c8_gqa4); - } - } - // Cluster-parallel q4_0 decode kernel - if (!is_q8 && backend_ctx->has_subgroup_shuffle) { - const std::string opts_c8 = opts + " -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2"; - cl_program prog_c8 = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8, - /*fatal=*/false, "fa q4_0 c8 NSG2", backend_ctx->queue); - if (prog_c8) { - cl_kernel k_c8 = clCreateKernel(prog_c8, "flash_attn_f32_q4_0_q1_vec_mq_split_c8", &err); - if (err == CL_SUCCESS) { - if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8, 128, - "flash_attn_f32_q4_0_q1_vec_mq_split_c8 (g8 ns2)", dk, dv)) { - backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8[{dk, dv}] = k_c8; - ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8, "flash_attn_f32_q4_0_q1_vec_mq_split_g8_c8", dk, dv); - } else { - clReleaseKernel(k_c8); - } - } - clReleaseProgram(prog_c8); - } - } - break; - } - case FA_VARIANT_F32_F16_SPLIT: { - cl_kernel k; - CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); - backend_ctx->fa.f32_f16_split[{dk, dv}] = k; - backend_ctx->fa.f32_f16_split_wg_size[{dk, dv}] = cfg->bm * cfg->n_split; - backend_ctx->fa.f32_f16_split_nkv_threshold[{dk, dv}] = cfg->nkv_split_threshold; - break; - } - case FA_VARIANT_Q8_0_SPLIT: - case FA_VARIANT_Q4_0_SPLIT: { - const bool is_q8 = variant == FA_VARIANT_Q8_0_SPLIT; - cl_kernel k; - CL_CHECK((k = clCreateKernel(prog, is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0", &err), err)); - auto & split = is_q8 ? backend_ctx->fa.f32_q8_0_split : backend_ctx->fa.f32_q4_0_split; - auto & split_wg = is_q8 ? backend_ctx->fa.f32_q8_0_split_wg_size : backend_ctx->fa.f32_q4_0_split_wg_size; - auto & split_bm = is_q8 ? backend_ctx->fa.f32_q8_0_split_bm : backend_ctx->fa.f32_q4_0_split_bm; - auto & split_thresh = is_q8 ? backend_ctx->fa.f32_q8_0_split_nkv_threshold : backend_ctx->fa.f32_q4_0_split_nkv_threshold; - split[{dk, dv}] = k; - split_wg[{dk, dv}] = cfg->bm * cfg->n_split; - split_bm[{dk, dv}] = cfg->bm; - split_thresh[{dk, dv}] = 0; // quant prefill: always split - break; - } - default: - break; - } - CL_CHECK(clReleaseProgram(prog)); - return true; -} + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); -// Compile a quant FA split kernel with a hand-picked (BLOCK_M, N_SPLIT) that -// overrides the default fa_dims tuning, for the DK values where the default -// N_SPLIT is degenerate for quant prefill: -// DK=256: default N_SPLIT=16 leaves DK/32=8 blocks -> 0 blocks/split. -// Override N_SPLIT=8 (1 block/split), BLOCK_M=16. -// DK=96 : DK/32 = 3 blocks, not divisible by the default N_SPLIT=2 -> -// override N_SPLIT=3. BLOCK_M must be 16, not 32: the N_SPLIT=3 -// QK-partial reduction uses sub_group_shuffle, so all 3 split -// threads of a query must land in one subgroup — WG_SIZE = -// BLOCK_M*N_SPLIT must be <= the 64-lane Adreno subgroup (16*3=48). -static bool ggml_opencl_ensure_fa_quant_split_override( - ggml_backend_opencl_context * backend_ctx, - int dk, int dv, int quant_bm, int quant_n_split, bool is_q8_0 -) { - const std::pair<int, int> dk_dv = {dk, dv}; - if (is_q8_0 && backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { - return true; - } - if (!is_q8_0 && backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { - return true; + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_splitk", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_mc = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_mc", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - const ggml_opencl_fa_variant variant = is_q8_0 ? FA_VARIANT_Q8_0_SPLIT : FA_VARIANT_Q4_0_SPLIT; - const auto attempt_key = std::make_pair(variant, dk_dv); - if (backend_ctx->fa.variant_attempted.count(attempt_key)) { - return false; + // gemm_noshuffle_q4_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_k_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_r1", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_kimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_cok", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - backend_ctx->fa.variant_attempted.insert(attempt_key); + // gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_k_q8_1_dp4a.cl"); +#endif + // X1 uses a smaller N tile to reduce local memory and register use. + int tile_n = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32; + if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) { tile_n = atoi(e); } + GGML_ASSERT(tile_n >= 4 && tile_n <= 64 && tile_n % 4 == 0); + backend_ctx->tile_n_dp4a_q4_k = tile_n; + std::string dp4a_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(tile_n); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), dp4a_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); - std::string shuffle_opts; - if (backend_ctx->has_subgroup_shuffle) { - shuffle_opts = backend_ctx->has_qcom_subgroup_shuffle - ? " -D cl_qcom_subgroup_shuffle=1" - : " -D cl_khr_subgroup_shuffle=1"; - } - const ggml_opencl_fa_dim * cfg = nullptr; - for (const auto & d : g_opencl_fa_dims) { - if (d.dk == dk && d.dv == dv) { - cfg = &d; break; + // Compile a smaller N tile to avoid unused columns in small batches. + int tile_n_narrow = MIN(tile_n, 16); + if (const char * e = getenv("GGML_OPENCL_Q4_K_DP4A_TS_NARROW")) { tile_n_narrow = atoi(e); } + GGML_ASSERT(tile_n_narrow >= 4 && tile_n_narrow <= 64 && tile_n_narrow % 4 == 0); + if (const char * e = getenv("GGML_OPENCL_Q4_K_DP4A_NARROW_MAX")) { + backend_ctx->max_n_dp4a_narrow_q4_k = atoi(e); + } + backend_ctx->tile_n_dp4a_narrow_q4_k = tile_n_narrow; + if (tile_n_narrow != tile_n) { + std::string narrow_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(tile_n_narrow); + cl_program narrow_prog = build_program_from_source(backend_ctx, kernel_src.c_str(), narrow_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow = clCreateKernel(narrow_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg = clCreateKernel(narrow_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err)); + CL_CHECK(clReleaseProgram(narrow_prog)); } + GGML_LOG_CONT("."); } - if (cfg == nullptr) { - return false; + + // gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q8_0_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // BLK_PREPASS_BM is the prepass-kernel BLOCK_M, needed so the quant kernel - // indexes the blk[] classification buffer correctly. - std::string opts = backend_ctx->kernel_compile_opts + shuffle_opts + - " -D DK=" + std::to_string(dk) + - " -D DV=" + std::to_string(dv) + - " -D BLOCK_M=" + std::to_string(quant_bm) + - " -D BLOCK_N=" + std::to_string(cfg->bn) + - " -D N_SPLIT=" + std::to_string(quant_n_split) + - " -D BLK_PREPASS_BM=" + std::to_string(cfg->bm); + // gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - const std::string src = ggml_opencl_fa_kernel_src(variant); - if (src.empty()) { - return false; + // gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - const std::string tag = std::string("fa ") + (is_q8_0 ? "q8_0" : "q4_0") + - " split DK=" + std::to_string(dk); - cl_program prog = build_program_from_source_ex( - backend_ctx->context, backend_ctx->device, src.c_str(), opts, - /*fatal=*/false, tag.c_str(), backend_ctx->queue); - if (!prog) { return false; } - cl_int err; - cl_kernel k; - if (is_q8_0) { - CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q8_0", &err), err)); - backend_ctx->fa.f32_q8_0_split[dk_dv] = k; - backend_ctx->fa.f32_q8_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; - backend_ctx->fa.f32_q8_0_split_bm[dk_dv] = quant_bm; - backend_ctx->fa.f32_q8_0_split_nkv_threshold[dk_dv] = 0; - } else { - CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q4_0", &err), err)); - backend_ctx->fa.f32_q4_0_split[dk_dv] = k; - backend_ctx->fa.f32_q4_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; - backend_ctx->fa.f32_q4_0_split_bm[dk_dv] = quant_bm; - backend_ctx->fa.f32_q4_0_split_nkv_threshold[dk_dv] = 0; + // quant_a_q8_1 (plain activation q8_1 pre-pass for the dense dp4a GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "quant_a_q8_1.cl.h" + }; +#else + const std::string kernel_src = read_file("quant_a_q8_1.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_quant_a_q8_1 = clCreateKernel(prog, "kernel_quant_a_q8_1", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - CL_CHECK(clReleaseProgram(prog)); - return true; -} - -namespace /* anonymous */ { -extern struct ggml_backend_device_i ggml_backend_opencl_device_i; -} -// Look for available and suitable devices. -static std::vector<ggml_backend_device> ggml_opencl_probe_devices(ggml_backend_reg * reg) { - std::vector<ggml_backend_device> found_devices; + // gemv_noshuffle_q4_k_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X) { + CL_gemv_compile_opts += " -DGGML_CL_A6X_CONSTFOLD_FIX"; + } + // dequant-once-per-block mc3 verify GEMV + if (getenv("GGML_OPENCL_Q4_K_MC3_DQ")) { + CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_ONCE "; + } + // LM-staged dequant mc3 verify GEMV, stages the dequantized + // q4_K weights in __local instead of regs + if (getenv("GGML_OPENCL_Q4_K_MC3_LDS")) { + CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_LDS "; + } -#ifdef GGML_OPENCL_PROFILING - GGML_LOG_INFO("ggml_opencl: OpenCL profiling enabled\n"); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32.cl"); #endif - struct cl_device; - struct cl_platform { - cl_platform_id id; - unsigned number; - char name[128]; - char vendor[128]; - struct cl_device * devices; - unsigned n_devices; - struct cl_device * default_device; - }; - - struct cl_device { - struct cl_platform * platform; - cl_device_id id; - unsigned number; - cl_device_type type; - char name[128]; - char version[128]; - }; + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - enum { NPLAT = 16, NDEV = 16 }; + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_mc3", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_splitk", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_splitk_reduce_f32 = clCreateKernel(prog, "kernel_gemv_splitk_reduce_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_glu", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - struct cl_platform platforms[NPLAT]; - unsigned n_platforms = 0; - struct cl_device devices[NDEV]; - unsigned n_devices = 0; - struct cl_device * default_device = NULL; - unsigned default_platform_number = 0; + // gemv_noshuffle_q4_k_f32_o4 — 4-output-per-WI variant for the long-vocab + // q4_K lm_head/embed GEMV (shares one activation read across 4 output rows). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_o4.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_o4.cl"); +#endif + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } + cl_program prog = build_program_from_source( + backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_o4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - cl_platform_id platform_ids[NPLAT]; - if (clGetPlatformIDs(NPLAT, platform_ids, &n_platforms) != CL_SUCCESS) { - GGML_LOG_ERROR("ggml_opencl: platform IDs not available.\n"); - return found_devices; + // gemv_noshuffle_q4_k_f32_tiled — tiled-wide canonical layout, default ON + // (opt out: GGML_OPENCL_Q4K_GEMV_TILED=0; separate convert + GEMV; weights via __global). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_tiled", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - for (unsigned i = 0; i < n_platforms; i++) { - struct cl_platform * p = &platforms[i]; - p->number = i; - p->id = platform_ids[i]; - CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL)); - CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL)); + // gemm_noshuffle_q4_K_f32_tiled_dp4a + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_K_f32_tiled_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_K_f32_tiled_dp4a.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a = + clCreateKernel(prog, "kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - cl_device_id device_ids[NDEV]; - cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices); - if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) { - p->n_devices = 0; - } else { - CL_CHECK(clGetDeviceIDsError); + backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr; + { + if (backend_ctx->has_vector_subgroup_broadcast) { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = + clCreateKernel(prog, "gemv_noshuffle_q4_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL; - p->default_device = NULL; - for (unsigned j = 0; j < p->n_devices; j++) { - struct cl_device * d = &devices[n_devices]; - d->number = n_devices++; - d->id = device_ids[j]; - d->platform = p; - CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL)); - CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL)); - CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_VERSION, sizeof(d->version), &d->version, NULL)); + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) { - p->default_device = d; + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } - } - if (default_device == NULL && p->default_device != NULL) { - default_device = p->default_device; - default_platform_number = i; + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } } } - if (n_devices == 0) { - GGML_LOG_ERROR("ggml_opencl: could find any OpenCL devices.\n"); - return found_devices; - } + backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = nullptr; + { + if (backend_ctx->has_vector_subgroup_broadcast) { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - char * user_platform_string = getenv("GGML_OPENCL_PLATFORM"); - char * user_device_string = getenv("GGML_OPENCL_DEVICE"); - int user_platform_number = -1; - int user_device_number = -1; - cl_device * candidate_devices = nullptr; - unsigned n_candidate_devices = 0; + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - unsigned n; - if (user_platform_string != NULL && sscanf(user_platform_string, " %u", &n) == 1 && n < n_platforms) { - user_platform_number = (int)n; - } - if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1 && n < n_devices) { - user_device_number = (int)n; - } - if (user_platform_number != -1 && user_device_number != -1) { - cl_platform* platform = &platforms[user_platform_number]; - if ((unsigned)user_device_number >= platform->n_devices) { - GGML_LOG_ERROR("ggml_opencl: invalid device number %d\n", user_device_number); - exit(1); - } - default_device = &platform->devices[user_device_number]; - candidate_devices = platform->devices; - n_candidate_devices = platform->n_devices; - } else { - // Choose a platform by matching a substring. - if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) { - for (unsigned i = 0; i < n_platforms; i++) { - struct cl_platform * p = &platforms[i]; - if (strstr(p->name, user_platform_string) != NULL || - strstr(p->vendor, user_platform_string) != NULL) { - user_platform_number = (int)i; - break; - } + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } - if (user_platform_number == -1) { - GGML_LOG_ERROR("ggml_opencl: no platform matching '%s' was found.\n", user_platform_string); - exit(1); + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } } + } - int platform_idx = user_platform_number != -1 ? user_platform_number : default_platform_number; - struct cl_platform * p = &platforms[platform_idx]; - candidate_devices = p->devices; - n_candidate_devices = p->n_devices; - default_device = p->default_device; - if (n_candidate_devices == 0) { - GGML_LOG_ERROR("ggml_opencl: selected platform '%s' does not have any devices.\n", p->name); - exit(1); + backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = nullptr; + { + if (backend_ctx->has_vector_subgroup_broadcast) { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q5_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = + clCreateKernel(prog, "gemv_noshuffle_q5_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) { - for (unsigned i = 0; i < n_candidate_devices; i++) { - struct cl_device * d = &candidate_devices[i]; - if (strstr(d->name, user_device_string) != NULL) { - user_device_number = d->number; - break; - } - } - if (user_device_number == -1) { - GGML_LOG_ERROR("ggml_opencl: no device matching '%s' was found.\n", user_device_string); - exit(1); + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } - } - if (user_device_number != -1) { - candidate_devices = &devices[user_device_number]; - n_candidate_devices = 1; - default_device = &candidate_devices[0]; - } - GGML_ASSERT(n_candidate_devices > 0); + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); - if (default_device == NULL) { - default_device = &candidate_devices[0]; + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } } } - GGML_ASSERT(n_candidate_devices != 0 && candidate_devices); + std::string CL_moe_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -cl-fast-relaxed-math"; - // Put the default device in front. - for (unsigned i = 1; i < n_candidate_devices; i++) { - if (&candidate_devices[i] == default_device) { - std::swap(candidate_devices[0], candidate_devices[i]); - default_device = &candidate_devices[0]; - break; - } + // gemv_moe_q4_1_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q4_1_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q4_1_f32_ns.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_1_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_1_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - GGML_LOG_INFO("ggml_opencl: selected platform: '%s'\n", default_device->platform->name); + // gemm_moe_q4_1_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_1_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_1_f32_ns.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - std::vector<cl_device_id> device_ids; - for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { - device_ids.push_back(dev->id); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - cl_int err; - cl_context shared_context; - cl_context_properties properties[] = { (intptr_t) CL_CONTEXT_PLATFORM, (intptr_t) default_device->platform->id, 0 }; + // gemm_moe_q4_1_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = nullptr; - CL_CHECK( - (shared_context = clCreateContext(properties, device_ids.size(), device_ids.data(), NULL, NULL, &err), err)); + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_1_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { - GGML_LOG_INFO("\nggml_opencl: device: '%s (%s)'\n", dev->name, dev->version); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } - auto dev_ctx = std::unique_ptr<ggml_backend_opencl_device_context>(new ggml_backend_opencl_device_context{ - /*.platform =*/dev->platform->id, - /*.platform_nane =*/dev->platform->name, - /*.device =*/dev->id, - /*.device_name =*/dev->name, - /*.device_type =*/dev->type, - /*.device_version =*/dev->version, - /*.backend_ctx =*/nullptr, - /*.buffer_type =*/{}, - /*.context =*/shared_context, - }); + // gemv_moe_mxfp4_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_mxfp4_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_mxfp4_f32.cl"); +#endif + backend_ctx->program_gemv_moe_mxfp4_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - found_devices.push_back(ggml_backend_device{ - /* .iface = */ ggml_backend_opencl_device_i, - /* .reg = */ reg, - /* .context = */ dev_ctx.get(), - }); + CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemv_moe_mxfp4_f32, "kernel_gemv_moe_mxfp4_f32", &err), err)); + GGML_LOG_CONT("."); + } - if (!ggml_opencl_is_device_supported(&found_devices.back())) { - found_devices.pop_back(); - GGML_LOG_WARN("ggml_opencl: drop unsupported device '%s'.\n", dev->name); - continue; - } + // gemm_moe_mxfp4_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_mxfp4_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_mxfp4_f32.cl"); +#endif + backend_ctx->program_gemm_moe_mxfp4_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - g_ggml_backend_opencl_dev_ctxs.push_back(std::move(dev_ctx)); + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemm_moe_mxfp4_f32, "kernel_gemm_moe_mxfp4_f32", &err), err)); + GGML_LOG_CONT("."); } - if (found_devices.size()) { - auto * dev_ctx = static_cast<ggml_backend_opencl_device_context *>(found_devices.front().context); - GGML_LOG_INFO("ggml_opencl: default device: '%s (%s)'\n", dev_ctx->device_name.c_str(), - dev_ctx->device_version.c_str()); + // gemv_moe_q4_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q4_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q4_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - if (dev_ctx->device_type != CL_DEVICE_TYPE_GPU) { - GGML_LOG_WARN("ggml_opencl: warning, the default device is not a GPU: '%s'.\n", - dev_ctx->device_name.c_str()); - } + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_0_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - return found_devices; -} + // gemm_moe_q4_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -static void ggml_opencl_print_backend_info(ggml_backend_opencl_device_context * dev_ctx) { - GGML_ASSERT(dev_ctx); - GGML_ASSERT(dev_ctx->backend_ctx); - - auto * backend_ctx = dev_ctx->backend_ctx; - - GGML_LOG_INFO("ggml_opencl: OpenCL device: %s\n", - backend_ctx->device_name.c_str()); - GGML_LOG_INFO("ggml_opencl: OpenCL driver: %s\n", - backend_ctx->driver_version.c_str()); - GGML_LOG_INFO("ggml_opencl: vector subgroup broadcast support: %s\n", - backend_ctx->has_vector_subgroup_broadcast ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: subgroup shuffle support: %s\n", - backend_ctx->has_subgroup_shuffle ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: device FP16 support: %s\n", - backend_ctx->fp16_support ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: khr dot product support: %s\n", - backend_ctx->has_integer_dot ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: mem base addr align: %u\n", - backend_ctx->alignment); - GGML_LOG_INFO("ggml_opencl: global mem size: %zu MB\n", - backend_ctx->global_mem_size/1024/1024); - GGML_LOG_INFO("ggml_opencl: max mem alloc size: %zu MB\n", - backend_ctx->max_alloc_size/1024/1024); - GGML_LOG_INFO("ggml_opencl: device max image buffer size (pixels): %zu\n", - backend_ctx->image_max_buffer_size); - GGML_LOG_INFO("ggml_opencl: device max image2d size: %zu x %zu\n", - backend_ctx->image2d_max_width, backend_ctx->image2d_max_height); - GGML_LOG_INFO("ggml_opencl: device max workgroup size: %zu\n", - backend_ctx->max_workgroup_size); - GGML_LOG_INFO("ggml_opencl: SVM coarse grain buffer support: %s\n", - backend_ctx->svm_caps & CL_DEVICE_SVM_COARSE_GRAIN_BUFFER ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: SVM fine grain buffer support: %s\n", - backend_ctx->svm_caps & CL_DEVICE_SVM_FINE_GRAIN_BUFFER ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: SVM fine grain system support: %s\n", - backend_ctx->svm_caps & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: SVM atomics support: %s\n", - backend_ctx->svm_caps & CL_DEVICE_SVM_ATOMICS ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: cl_qcom_subgroup_shuffle support: %s\n", - backend_ctx->has_qcom_subgroup_shuffle ? "true" : "false"); - - // Print out configurations -#ifdef GGML_OPENCL_SOA_Q - GGML_LOG_INFO("ggml_opencl: flattening quantized weights representation as struct of arrays (GGML_OPENCL_SOA_Q)\n"); -#endif // GGML_OPENCL_SOA_Q - -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - GGML_LOG_INFO("ggml_opencl: using kernels optimized for Adreno (GGML_OPENCL_USE_ADRENO_KERNELS)\n"); - if (backend_ctx->adreno_xmem_gemm_enabled) { - GGML_LOG_INFO("ggml_opencl: Adreno xmem F16xF32 GEMM enabled (temporary weight prepack)\n"); - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - - if (backend_ctx->adreno_use_large_buffer) { - if (!backend_ctx->adreno_has_large_buffer) { - GGML_LOG_INFO("ggml_opencl: Adreno large buffer requested but not supported by driver, will use regular buffer\n"); - backend_ctx->adreno_use_large_buffer = false; - } else { - GGML_LOG_INFO("ggml_opencl: Adreno large buffer enabled\n"); - } - } - - if (dev_ctx->opfilter) { - // for information only, the actual regex object is created in ggml_opencl_is_device_supported - GGML_LOG_INFO("ggml_opencl: opfilter regex = \"%s\"\n", dev_ctx->opfilter_str.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } -} - -// check if device should be accepted -static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev) { - GGML_ASSERT(dev); - GGML_ASSERT(dev->context); - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; - GGML_ASSERT(dev_ctx->platform); - GGML_ASSERT(dev_ctx->device); + // gemm_moe_q4_0_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = nullptr; - if (strstr(dev_ctx->device_name.c_str(), "Adreno") || - strstr(dev_ctx->device_name.c_str(), "Qualcomm") || - strstr(dev_ctx->device_version.c_str(), "Adreno")) { - dev_ctx->gpu_family = GPU_FAMILY::ADRENO; + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - // Usually device version contains the detailed device name - dev_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_version.c_str()); - if (dev_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN) { - dev_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_name.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } } - } else if (strstr(dev_ctx->device_name.c_str(), "Intel")) { - dev_ctx->gpu_family = GPU_FAMILY::INTEL; - } else { - GGML_LOG_WARN("ggml_opencl: unsupported GPU '%s'.\n", dev_ctx->device_name.c_str()); - dev_ctx->gpu_family = GPU_FAMILY::UNKNOWN; - return false; } - ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); + // gemm_moe_q8_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q8_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q8_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // Check device OpenCL version, OpenCL 2.0 or above is required - ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, dev_ctx->device); - if (opencl_c_version.major < 2) { - GGML_LOG_WARN("ggml_opencl: OpenCL 2.0 or above is required\n"); - return false; + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q8_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (dev_ctx->gpu_family != GPU_FAMILY::ADRENO) { - GGML_LOG_WARN("ggml_opencl: Adreno-specific kernels should not be enabled for non-Adreno GPUs; " - "run on an Adreno GPU or recompile with CMake option `-DGGML_OPENCL_USE_ADRENO_KERNELS=OFF`\n"); - return false; - } + // gemv_moe_q5_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q5_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q5_0_f32_ns.cl"); #endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - size_t ext_str_size; - CL_CHECK(clGetDeviceInfo(dev_ctx->device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size)); + CL_CHECK((backend_ctx->kernel_gemv_moe_q5_0_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - std::vector<char> ext_buffer(ext_str_size + 1); - CL_CHECK(clGetDeviceInfo(dev_ctx->device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer.data(), NULL)); - ext_buffer[ext_str_size] = '\0'; + // gemm_moe_q5_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q5_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q5_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // Check if ext_buffer contains cl_khr_fp16 - bool fp16_support = strstr(ext_buffer.data(), "cl_khr_fp16") != NULL; - if (!fp16_support) { - GGML_LOG_WARN("ggml_opencl: device does not support FP16\n"); - return false; + CL_CHECK((backend_ctx->kernel_gemm_moe_q5_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - // If OpenCL 3.0 is supported, then check for cl_khr_subgroups, which becomes - // optional in OpenCL 3.0 (cl_khr_subgroup is mandatory in OpenCL 2.x) - if (opencl_c_version.major == 3 && strstr(ext_buffer.data(), "cl_khr_subgroups") == NULL && - strstr(ext_buffer.data(), "cl_intel_subgroups") == NULL) { - GGML_LOG_WARN("ggml_opencl: device does not support subgroups (cl_khr_subgroups or cl_intel_subgroups) " - "(note that subgroups is an optional feature in OpenCL 3.0)\n"); - return false; + // gemv_moe_q5_1_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q5_1_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q5_1_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_moe_q5_1_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_1_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - clGetDeviceInfo(dev_ctx->device, CL_DEVICE_GLOBAL_MEM_SIZE, sizeof(size_t), &dev_ctx->global_mem_size, NULL); + // gemm_moe_q5_1_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q5_1_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q5_1_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - const char * str_opfilter = getenv("GGML_OPENCL_OPFILTER"); - if (str_opfilter) { - dev_ctx->opfilter_str = str_opfilter; - dev_ctx->opfilter = new std::regex(str_opfilter, std::regex_constants::icase); + CL_CHECK((backend_ctx->kernel_gemm_moe_q5_1_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_1_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - return true; -} + // gemv_moe_q4_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q4_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q4_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -// Initialize device if it is supported (returns nullptr if it is not). -static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { - GGML_ASSERT(dev); - GGML_ASSERT(dev->context); + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; - GGML_ASSERT(dev_ctx->platform); - GGML_ASSERT(dev_ctx->device); + // gemm_moe_q4_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - if (dev_ctx->backend_ctx) { - return dev_ctx->backend_ctx; + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - auto backend_ctx = std::make_unique<ggml_backend_opencl_context>(); - backend_ctx->device = dev_ctx->device; - backend_ctx->gpu_family = GPU_FAMILY::UNKNOWN; + // gemm_moe_q4_k_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = nullptr; - // ref_count get increased in ggml_backend_opencl_device_init - // This function is also used to retrieve backend context, so we don't want - // to increase ref_count for each call. We only want to increase ref_count - // when the associated device is initialized - backend_ctx->ref_count = 0; - - backend_ctx->gpu_family = dev_ctx->gpu_family; - backend_ctx->adreno_gen = dev_ctx->adreno_gen; - if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { - ggml_cl_init_fa_dims_table(); + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - // Use wave size of 64 for all Adreno GPUs. - backend_ctx->adreno_wave_size = 64; + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } } - // Populate backend device name - backend_ctx->device_name = dev_ctx->device_name; - - // A local ref of cl_device_id for convenience - cl_device_id device = backend_ctx->device; - - ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); - ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, device); - - backend_ctx->platform_version = platform_version; - backend_ctx->opencl_c_version = opencl_c_version; - - // Check driver version - size_t driver_version_str_size; - CL_CHECK(clGetDeviceInfo(device, CL_DRIVER_VERSION, 0, NULL, &driver_version_str_size)); - std::vector<char> driver_version(driver_version_str_size + 1); - CL_CHECK(clGetDeviceInfo(device, CL_DRIVER_VERSION, driver_version_str_size, driver_version.data(), NULL)); - driver_version[driver_version_str_size] = '\0'; - backend_ctx->driver_version = driver_version.data(); + // gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_k_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - backend_ctx->adreno_cl_compiler_version = get_adreno_cl_compiler_version(driver_version.data()); - backend_ctx->has_vector_subgroup_broadcast = - (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) || - (backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17) || - (backend_ctx->adreno_cl_compiler_version.type == E17); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a - // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41 - // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts - // that do not need the workarounds do not pay for them. The explicit type check is - // required: newer_than_or_same() is false for every non-E031 compiler, so negating it - // alone would enable the workarounds on E17/DX. - backend_ctx->q6_k_flat_old_compiler = - backend_ctx->adreno_cl_compiler_version.type == E031 && - !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0); + // gemm_moe_q4_k_q8_1_dp4a_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr; - size_t ext_str_size; - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size)); - std::vector<char> ext_buffer(ext_str_size + 1); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer.data(), NULL)); - ext_buffer[ext_str_size] = '\0'; // ensure it is null terminated + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_q8_1_dp4a_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - // check support for qcom_subgroup_shuffle - if (strstr(ext_buffer.data(), "cl_qcom_subgroup_shuffle") != NULL) { - backend_ctx->has_qcom_subgroup_shuffle = true; + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } } - // Check if ext_buffer contains cl_khr_fp16 - backend_ctx->fp16_support = strstr(ext_buffer.data(), "cl_khr_fp16") != NULL; + // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_mxfp4_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_mxfp4_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // check Adreno large buffer support - backend_ctx->adreno_has_large_buffer = strstr(ext_buffer.data(), "cl_qcom_large_buffer") != NULL; + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - // subgroup shuffle support (N_SPLIT>1 FA kernel) - backend_ctx->has_qcom_subgroup_shuffle = strstr(ext_buffer.data(), "cl_qcom_subgroup_shuffle") != NULL; - backend_ctx->has_subgroup_shuffle = - strstr(ext_buffer.data(), "cl_khr_subgroup_shuffle") != NULL || - backend_ctx->has_qcom_subgroup_shuffle; + // gemm_moe_mxfp4_q8_1_dp4a_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; - // check for cl_khr_integer_dot_product - // cl_qcom_dot_product8 uses signed * unsigned - // while cl_khr_integer_dot_product uses signed * signed -- we stick with khr for now - backend_ctx->has_integer_dot = - strstr(ext_buffer.data(), "cl_khr_integer_dot_product") != NULL; + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - cl_uint base_align_in_bits; - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL)); - GGML_ASSERT(base_align_in_bits % 8u == 0); - backend_ctx->alignment = base_align_in_bits / 8u; + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } - backend_ctx->global_mem_size = dev_ctx->global_mem_size; + // gemm_moe_mxfp4_q8_1_dp4a_ldck_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr; - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(size_t), &backend_ctx->max_alloc_size, NULL)); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL)); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image2d_max_width, NULL)); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image2d_max_height, NULL)); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL)); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &backend_ctx->svm_caps, 0)); + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - if (opencl_c_version.major >= 3) { - // Assume it is not available for 3.0, since it is optional in 3.0. - // If compiling against 3.0, then we can query. - backend_ctx->non_uniform_workgroups = false; -#if CL_TARGET_OPENCL_VERSION >= 300 - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_NON_UNIFORM_WORK_GROUP_SUPPORT, sizeof(cl_bool), - &backend_ctx->non_uniform_workgroups, 0)); -#endif - } else { - GGML_ASSERT(opencl_c_version.major == 2); - // Non-uniform workgroup sizes is mandatory feature in v2.x. - backend_ctx->non_uniform_workgroups = true; + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Adreno xmem F16xF32 GEMM, default on adreno, opt out with GGML_OPENCL_ADRENO_XMEM_GEMM=0. - // This helps models with f16 attention weights, e.g., gpt-oss-20b-f16 - { - const char * xmem_env = getenv("GGML_OPENCL_ADRENO_XMEM_GEMM"); - backend_ctx->adreno_xmem_gemm_enabled = backend_ctx->gpu_family == GPU_FAMILY::ADRENO && - (xmem_env ? atoi(xmem_env) != 0 : true); - } + // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_0_q8_1_dp4a.cl"); #endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // determine whether to use large buffer for Adreno - backend_ctx->adreno_use_large_buffer = getenv("GGML_OPENCL_ADRENO_USE_LARGE_BUFFER") != nullptr && - backend_ctx->gpu_family == GPU_FAMILY::ADRENO; - - // ragged moe, unspecified or non-zero means enabled, set to 0 to disable - static const char * ragged_fp16_env = getenv("GGML_OPENCL_MOE_RAGGED_FP16"); - backend_ctx->adreno_use_moe_ragged = (ragged_fp16_env == NULL) ? 1 : (atoi(ragged_fp16_env) != 0); - - // ragged moe, tile-skip granularity (columns per skip-group): 8 = quarter (default), - // 16 = half (legacy), 32 = disabled. Override with GGML_OPENCL_MOE_RAGGED_GRAN={8,16,32} - static const char * ragged_gran_env = getenv("GGML_OPENCL_MOE_RAGGED_GRAN"); - backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8; - - // whether fuse moe combine - static const char * fuse_moe_bias_glu_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_GLU"); - backend_ctx->fuse_moe_bias_glu = fuse_moe_bias_glu_env == NULL ? 1 : (atoi(fuse_moe_bias_glu_env) != 0); - - static const char * fuse_moe_bias_combine_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_COMBINE"); - backend_ctx->fuse_moe_bias_combine = fuse_moe_bias_combine_env == NULL ? 1 : (atoi(fuse_moe_bias_combine_env) != 0); - - static const char * fuse_moe_combine_env = getenv("GGML_OPENCL_FUSE_MOE_COMBINE"); - backend_ctx->fuse_moe_combine = fuse_moe_combine_env == NULL ? 1 : (atoi(fuse_moe_combine_env) != 0); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - // ragged moe dp4 variant - static const char * ragged_dp4_env = getenv("GGML_OPENCL_MOE_RAGGED"); - backend_ctx->adreno_use_moe_ragged_dp4 = ragged_dp4_env == NULL ? 1 : (atoi(ragged_dp4_env) != 0); + // gemm_moe_q4_0_q8_1_dp4a_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; -#ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS - // try loading adreno binary kernels if enabled - // if fails to load, builtin kernels will be used - { - dl_handle * kernel_lib_handle = dl_load_library(KERNEL_LIB_NAME); - backend_ctx->adreno_use_bin_kernels = false; + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_q8_1_dp4a_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - if (kernel_lib_handle) { - backend_ctx->get_adreno_bin_kernel_func = (get_adreno_bin_kernel_func_t)dl_get_sym(kernel_lib_handle, "get_adreno_kernels"); - if (backend_ctx->get_adreno_bin_kernel_func) { - GGML_LOG_INFO("ggml_opencl: loaded bin kernel library %s\n", KERNEL_LIB_NAME); - backend_ctx->adreno_use_bin_kernels = true; - } else { - GGML_LOG_INFO("ggml_opencl: bin kernel library %s is invalid, will use builtin kernels\n", KERNEL_LIB_NAME); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - } else { - GGML_LOG_INFO("ggml_opencl: failed to load %s, will use builtin kernels\n", KERNEL_LIB_NAME); } } -#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS - - cl_int err; - // A local ref of cl_context for convenience - cl_context context = backend_ctx->context = dev_ctx->context; + // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q8_1_dp4a.cl"); +#endif + const std::string opts80 = CL_moe_compile_opts + " -DMOE_QT=80"; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), opts80.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 = clCreateKernel(prog, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); - //CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err), - // (err != CL_INVALID_QUEUE_PROPERTIES && err != CL_INVALID_VALUE ? err : - // (queue = clCreateCommandQueue(context, device, 0, &err), err) - //))); - cl_command_queue_properties command_queue_props = 0; -#ifdef GGML_OPENCL_PROFILING - command_queue_props |= CL_QUEUE_PROFILING_ENABLE; -#endif - CL_CHECK((backend_ctx->queue = clCreateCommandQueue(context, device, command_queue_props, &err), err)); + const std::string opts50 = CL_moe_compile_opts + " -DMOE_QT=50"; + cl_program prog50 = + build_program_from_source(backend_ctx, kernel_src.c_str(), opts50.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 = clCreateKernel(prog50, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog50)); - // delay kernel loading until the first buffer is created - // load_cl_kernels(backend_ctx.get()); + const std::string opts5 = CL_moe_compile_opts + " -DMOE_QT=5"; + cl_program prog5 = + build_program_from_source(backend_ctx, kernel_src.c_str(), opts5.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k = clCreateKernel(prog5, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog5)); + GGML_LOG_CONT("."); + } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Allocate intermediate buffers and images - size_t required_A_q_d_bytes = 311164928; - size_t required_A_s_d_bytes = 38895616; - size_t required_B_d_bytes = 45088768; + // moe_reorder_quant_a_q8_1 (fused reorder + q8_1 quant) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_reorder_quant_a_q8_1.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_reorder_quant_a_q8_1.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // Ensure buffer sizes do not exceed the maximum allocation size - size_t max_A_q_d_bytes = MIN(required_A_q_d_bytes, backend_ctx->max_alloc_size); - size_t max_A_s_d_bytes = MIN(required_A_s_d_bytes, backend_ctx->max_alloc_size); - size_t max_B_d_bytes = MIN(required_B_d_bytes, backend_ctx->max_alloc_size); - if (required_A_q_d_bytes > backend_ctx->max_alloc_size) { - GGML_LOG_WARN("ggml_opencl: A_q_d buffer size reduced from %zu to %zu due to device limitations.\n", - required_A_q_d_bytes, max_A_q_d_bytes); - } - if (required_A_s_d_bytes > backend_ctx->max_alloc_size) { - GGML_LOG_WARN("ggml_opencl: A_s_d buffer size reduced from %zu to %zu due to device limitations.\n", - required_A_s_d_bytes, max_A_s_d_bytes); - } - if (required_B_d_bytes > backend_ctx->max_alloc_size) { - GGML_LOG_WARN("ggml_opencl: B_d buffer size reduced from %zu to %zu due to device limitations.\n", - required_B_d_bytes, max_B_d_bytes); + CL_CHECK((backend_ctx->kernel_moe_reorder_quant_a_q8_1 = clCreateKernel(prog, "kernel_moe_reorder_quant_a_q8_1", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - backend_ctx->prealloc_quant_trans.allocate(context, max_A_q_d_bytes); - backend_ctx->prealloc_scales_trans.allocate(context, max_A_s_d_bytes); - backend_ctx->prealloc_act_trans.allocate(context, max_B_d_bytes); -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + // transpose_activation_q8 (per-tile transpose of q8_1 activations for the ldck dp4a GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "transpose_activation_q8.cl.h" + }; +#else + const std::string kernel_src = read_file("transpose_activation_q8.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - backend_ctx->disable_fusion = getenv("GGML_OPENCL_DISABLE_FUSION") != nullptr; - if (const char * env = getenv("GGML_OPENCL_FUSE_MM_GLU")) { - backend_ctx->fuse_mm_glu = atoi(env) != 0; + CL_CHECK((backend_ctx->kernel_transpose_activation_q8 = clCreateKernel(prog, "kernel_transpose_activation_q8", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (const char * env = getenv("GGML_OPENCL_FUSE_RMS_ADD")) { - backend_ctx->fuse_rms_add = atoi(env) != 0; + + // gemv_moe_q5_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q5_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q5_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_moe_q5_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (const char * env = getenv("GGML_OPENCL_F16_MROW")) { - backend_ctx->f16_mrow = atoi(env) != 0; + + // gemm_moe_q5_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q5_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q5_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q5_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (const char * env = getenv("GGML_OPENCL_F16_MROW_RPT")) { - const int v = atoi(env); - backend_ctx->f16_mrow_rpt = (v == 2 || v == 4 || v == 8 || v == 16) ? v : 1; + + // gemv_moe_q6_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q6_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q6_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_moe_q6_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q6_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - dev_ctx->backend_ctx = backend_ctx.release(); - return dev_ctx->backend_ctx; -} + // gemm_moe_q6_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q6_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q6_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -static void ggml_cl_free(ggml_backend_t backend) { - ggml_backend_opencl_context * ctx = (ggml_backend_opencl_context *) backend->context; - ctx->free(); -} + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static void transpose_2d( - ggml_backend_opencl_context * backend_ctx, - cl_kernel kernel, - cl_mem src, cl_mem dst, size_t size, - cl_int stride, cl_int rows, - bool blocking = true, - bool auto_local = false // let driver pick local size for non-uniform workgroups -) { - static ggml_cl_buffer buf; + // gemm_moe_q6_k_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = nullptr; - cl_event evt; - cl_int err; + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - buf.allocate(backend_ctx->context, size); + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } - cl_mem trans; - cl_buffer_region region; + // gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q6_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q6_k_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - region.origin = 0; - region.size = size; - CL_CHECK((trans = clCreateSubBuffer( - buf.buffer, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &stride)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &rows)); + // gemm_moe_q6_k_q8_1_dp4a_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr; - size_t local_size[3] = {64, 1, 1}; - size_t global_size[3] = {(size_t)stride, (size_t)rows, 1};; - CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, - global_size, auto_local ? NULL : local_size, 0, NULL, NULL)); + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_q8_1_dp4a_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - if (blocking) { - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseEvent(evt)); - } else { - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, NULL)); + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } } - CL_CHECK(clReleaseMemObject(trans)); -} + // gemv_moe_mxfp4_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_mxfp4_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_mxfp4_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -static void transpose_2d_as_8b( - ggml_backend_opencl_context * backend_ctx, - cl_mem src, cl_mem dst, size_t size, - cl_int stride, cl_int rows, - bool blocking = true, - bool auto_local = false -) { - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_8_buf, - src, dst, size, stride, rows, blocking, auto_local); -} + CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } -static void transpose_2d_as_16b( - ggml_backend_opencl_context * backend_ctx, - cl_mem src, cl_mem dst, size_t size, - cl_int stride, cl_int rows, - bool blocking = true -) { - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, - src, dst, size, stride, rows, blocking); -} - -static void transpose_2d_as_32b( - ggml_backend_opencl_context * backend_ctx, - cl_mem src, cl_mem dst, size_t size, - cl_int stride, cl_int rows, - bool blocking = true -) { - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, - src, dst, size, stride, rows, blocking); -} -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - -//------------------------------------------------------------------------------ -// Tensor extra management -//------------------------------------------------------------------------------ -struct ggml_tensor_extra_cl { - // The buffer object that holds the data. - cl_mem data_device; - // The offset into the buffer object. This is primarily for scratch buffer - // and view operation. - // NB: this offset no longer includes view offset (view_offs). Whenever this - // offset is used, view_offs should be considered. - cl_ulong offset; - // The actual size of the cl_mem object. This is needed when returning the - // block to the pool. - size_t actual_size; + // gemm_moe_mxfp4_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_mxfp4_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_mxfp4_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - void reset() { - data_device = nullptr; - offset = 0; - actual_size = 0; + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } -}; - -struct ggml_tensor_extra_cl_q1_0 { - cl_mem q = nullptr; - cl_mem q_img = nullptr; - - cl_mem d = nullptr; - cl_mem d_img = nullptr; - size_t size_q = 0; - size_t size_d = 0; + // gemm_moe_mxfp4_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = nullptr; - ~ggml_tensor_extra_cl_q1_0() { - reset(); - } + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } } - q_img = nullptr; - d_img = nullptr; - size_q = 0; - size_d = 0; } -}; -// Additional tensor extra structs for quantized tensors. -// These tensors are loaded from files and should not be allocated in scratch -- -// they should always be allocated from the pool. Hence, they do not have an -// `offset`, which indicate their locations in the scratch buffer. -struct ggml_tensor_extra_cl_q4_0 { - // Quantized values. - cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem q_img = nullptr; - // Scales. - cl_mem d = nullptr; - // Scales in image1d_buffer_t. - cl_mem d_img = nullptr; - // Size of quantized values. - size_t size_q = 0; - // Size of scales. - size_t size_d = 0; + // moe_reorder_b + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_reorder_b.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_reorder_b.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - ~ggml_tensor_extra_cl_q4_0() { - reset(); + CL_CHECK((backend_ctx->kernel_moe_reorder_b = clCreateKernel(prog, "kernel_moe_reorder_b", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - if (q_img != nullptr) { - CL_CHECK(clReleaseMemObject(q_img)); - q_img = nullptr; - } - if (d_img != nullptr) { - CL_CHECK(clReleaseMemObject(d_img)); - d_img = nullptr; + // gemv_noshuffle_q6_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32.cl"); +#endif + + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } - size_q = 0; - size_d = 0; - } -}; -struct ggml_tensor_extra_cl_q4_1 { - // Quantized values. - cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem q_img = nullptr; - // Scales. - cl_mem d = nullptr; - // Scales in image1d_buffer_t. - cl_mem d_img = nullptr; - // Min - cl_mem m = nullptr; - // Min in image1d_buffer_t. - cl_mem m_img = nullptr; - // Size of quantized values. - size_t size_q = 0; - // Size of scales. - size_t size_d = 0; - // Size of min values. - size_t size_m = 0; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - ~ggml_tensor_extra_cl_q4_1() { - reset(); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_mc3", &err), err)); + GGML_LOG_CONT("."); } - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - if (m != nullptr) { - CL_CHECK(clReleaseMemObject(m)); - m = nullptr; - } - if (q_img != nullptr) { - CL_CHECK(clReleaseMemObject(q_img)); - q_img = nullptr; + // gemv_noshuffle_q6_k_f32_o4 — 4-output-per-WI variant, opt-in via + // GGML_OPENCL_Q6K_GEMV_O4=1 (~3x fewer dispatches on long-vocab lm_head). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_o4.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_o4.cl"); +#endif + + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } - // Currently, q_img and d_img are only initialized when SMALL_ALLOC is - // enabled. They point to the images in ggml_backend_opencl_buffer_context. - // So, there is no need to release them here. - // TODO: initialize them for non SMALL_PATH path, or remove them. - d_img = nullptr; - m_img = nullptr; - size_q = 0; - size_d = 0; - size_m = 0; - } -}; -struct ggml_tensor_extra_cl_q5_0 { - // Quantized values. - cl_mem qs = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem qs_img = nullptr; - // 5-th bit values. - cl_mem qh = nullptr; - // 5-th bit values in image1d_buffer_t. - cl_mem qh_img = nullptr; - // Scales. - cl_mem d = nullptr; - // Scales in image1d_buffer_t. - cl_mem d_img = nullptr; - // Uniform per-32-block scale (2/block) + min (1/block, = d*16 for the -16 centering) - // for the generic dp4a MoE GEMM. Built from d. - cl_mem scale = nullptr; - cl_mem min = nullptr; - // Size of quantized values. - size_t size_qs = 0; - // Size of 5-th bit values. - size_t size_qh = 0; - // Size of scales. - size_t size_d = 0; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - ~ggml_tensor_extra_cl_q5_0() { - reset(); - } + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_o4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); - void reset() { - if (qs != nullptr) { - CL_CHECK(clReleaseMemObject(qs)); - qs = nullptr; - } - if (qh != nullptr) { - CL_CHECK(clReleaseMemObject(qh)); - qh = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - if (qs_img != nullptr) { - CL_CHECK(clReleaseMemObject(qs_img)); - qs_img = nullptr; + // Global-read variant: weights read from __global coalesced instead of + // image1d_buffer (the texture cache caps the streaming lm_head read + // bandwidth). Opt-in via GGML_OPENCL_Q6K_GEMV_O4_GLOBAL. + cl_program prog_g = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts + " -DQ6K_O4_GLOBAL"); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global = + clCreateKernel(prog_g, "kernel_gemv_noshuffle_q6_K_f32_o4_global", &err), err)); + CL_CHECK(clReleaseProgram(prog_g)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q6_k_f32_tiled — tiled-wide canonical layout, default ON + // (opt out: GGML_OPENCL_Q6K_GEMV_TILED=0; separate convert + GEMV; weights via __global). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled_mc3", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q6_k_f32_tiled — batched (N>1) GEMM over the same tiled-wide + // canonical layout, so batched lm_head/embed stays correct + on GPU. + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled = + clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_tiled", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q6_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_cok", &err), err)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q5_k_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - if (scale != nullptr) { - CL_CHECK(clReleaseMemObject(scale)); - scale = nullptr; + +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q5_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32.cl"); +#endif + + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32_mc3", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q5_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_k_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // repack + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "flash_attn_repack.cl.h" + }; +#else + const std::string kernel_src = read_file("flash_attn_repack.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->fa.kernel_repack_q_for_wmm = clCreateKernel(prog, "kernel_repack_q_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_k_for_wmm = clCreateKernel(prog, "kernel_repack_k_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_v_for_wmm = clCreateKernel(prog, "kernel_repack_v_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_mask_for_wmm = clCreateKernel(prog, "kernel_repack_mask_for_wmm", &err), err)); + GGML_LOG_CONT("."); + } + + // kernel_flash_attn_f32_f16_bin + { + size_t bin_size = 0; + backend_ctx->fa.kernel_flash_attn_f32_f16_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("flash_attn_f32_f16_wmm", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->fa.kernel_flash_attn_f32_f16_bin = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } } - if (min != nullptr) { - CL_CHECK(clReleaseMemObject(min)); - min = nullptr; + } + load_cl_kernels_gemm_coopk(backend_ctx); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + GGML_LOG_CONT("\n"); + backend_ctx->kernels_loaded = true; +} + +static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev); +static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev); + +// FA per-(dk,dv) tile tuning table + GGML_OPENCL_FA_TUNE override parsing. +#include "fa_tune.h" + +// FA variant key for the per-(dk,dv,variant) lazy compile cache. +// Kernel built on first dispatch to reduce kernel loading time. +// NB - a warmup run is recommended to get all necessary FA variants compiled +// before actual runs. +enum ggml_opencl_fa_variant { + FA_VARIANT_PRE = 0, // prepass kernels (kv_pad, mask_pad, blk) + FA_VARIANT_F16 = 1, + FA_VARIANT_F32 = 2, + FA_VARIANT_F32_F16 = 3, + FA_VARIANT_Q8_0 = 4, + FA_VARIANT_Q4_0 = 5, + FA_VARIANT_F32_F16_SPLIT = 6, + FA_VARIANT_Q8_0_SPLIT = 7, + FA_VARIANT_Q4_0_SPLIT = 8, +}; + +static std::string ggml_opencl_fa_kernel_src(ggml_opencl_fa_variant v) { +#ifdef GGML_OPENCL_EMBED_KERNELS + switch (v) { + case FA_VARIANT_F16: + return std::string{ + #include "flash_attn_f16.cl.h" + }; + case FA_VARIANT_F32: + return std::string{ + #include "flash_attn_f32.cl.h" + }; + case FA_VARIANT_F32_F16: + case FA_VARIANT_F32_F16_SPLIT: + return std::string{ + #include "flash_attn_f32_f16.cl.h" + }; + case FA_VARIANT_PRE: + return std::string{ + #include "flash_attn_pre_f16.cl.h" + }; + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q8_0_SPLIT: + return std::string{ + #include "flash_attn_f32_q8_0.cl.h" + }; + case FA_VARIANT_Q4_0: + case FA_VARIANT_Q4_0_SPLIT: + return std::string{ + #include "flash_attn_f32_q4_0.cl.h" + }; + } + return {}; +#else + switch (v) { + case FA_VARIANT_F16: return read_file("flash_attn_f16.cl"); + case FA_VARIANT_F32: return read_file("flash_attn_f32.cl"); + case FA_VARIANT_F32_F16: + case FA_VARIANT_F32_F16_SPLIT: return read_file("flash_attn_f32_f16.cl"); + case FA_VARIANT_PRE: return read_file("flash_attn_pre_f16.cl"); + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q8_0_SPLIT: return read_file("flash_attn_f32_q8_0.cl"); + case FA_VARIANT_Q4_0: + case FA_VARIANT_Q4_0_SPLIT: return read_file("flash_attn_f32_q4_0.cl"); + } + return {}; +#endif +} + +static std::string ggml_opencl_fa_compile_opts(ggml_backend_opencl_context * backend_ctx, + const ggml_opencl_fa_dim * cfg, + ggml_opencl_fa_variant variant) { + std::string opts = backend_ctx->kernel_compile_opts + + " -D DK=" + std::to_string(cfg->dk) + + " -D DV=" + std::to_string(cfg->dv) + + " -D BLOCK_M=" + std::to_string(cfg->bm) + + " -D BLOCK_N=" + std::to_string(cfg->bn); + + if (backend_ctx->gpu_family == INTEL) { + opts += " -D FA_SG=32"; + } + + const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || + variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0_SPLIT; + if (is_split) { + opts += " -D N_SPLIT=" + std::to_string(cfg->n_split); + } + // Shuffle define for the split tile paths AND the cluster-parallel decode + // kernel (q1_vec_mq_split_c8) in the plain F32_F16 program. Without it the + // c8 kernel is compiled out (HAS_SUBGROUP_SHUFFLE guard) and dispatch + // falls back to the baseline mq_split. + if ((is_split || variant == FA_VARIANT_F32_F16) && backend_ctx->has_subgroup_shuffle) { + opts += backend_ctx->has_qcom_subgroup_shuffle + ? " -D cl_qcom_subgroup_shuffle=1" + : " -D cl_khr_subgroup_shuffle=1"; + } + // X1E drops the explicit sub-group size pin on the c8 kernels, compiler + // routes the fp16-heavy kernel to a slow variant with explicit subgroup size + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) { + opts += " -D FA_C8_NO_SG_PIN"; + } + // Transposed K tile in local memory: the KV rows the QK loop walks together become + // adjacent, so a group of them is ONE 128-bit local read instead of several narrow + // ones. The QK loop is LDS-read-issue-bound (a wrong-math probe that kept every FMA/dp4a + // but removed the LDS reads ran the kernel ~40% faster), so this is worth up to +26% on + // fa=1 prefill. Output is bit-identical -- only the layout moves. + // + // DK <= 128 only. At DK=256 (gemma-3-4b) it measures 1-2% NEGATIVE and reproduces across + // rounds; padding the row stride does not recover it, so the cause is not a simple bank + // conflict and the wider tile does not want this layout. + // + // Default on within that gate; GGML_OPENCL_FA_K_LDS_T=0 restores the row-major tile. + { + const char * e = getenv("GGML_OPENCL_FA_K_LDS_T"); + if ((e == nullptr || e[0] != '0') && cfg->dk <= 128) { + opts += " -D FA_K_LDS_T"; + } + } + return opts; +} + +// only register when the kernel's required dispatch workgroup size is within +// the limit of the device's maximum workgroup size +static bool ggml_opencl_fa_kernel_fits_wg(ggml_backend_opencl_context * backend_ctx, + cl_kernel kernel, size_t required_wg, + const char * name, int dk, int dv) { + if (kernel == NULL) { return false; } + const size_t dev_max = backend_ctx->max_workgroup_size; + if (dev_max < required_wg) { + GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d requires WG %zu > device max %zu; skipping registration (will fall back)\n", + name, dk, dv, required_wg, dev_max); + return false; + } + size_t kwg = 0; + cl_int err = clGetKernelWorkGroupInfo(kernel, backend_ctx->device, + CL_KERNEL_WORK_GROUP_SIZE, + sizeof(kwg), &kwg, NULL); + if (err != CL_SUCCESS) { + GGML_LOG_INFO("ggml_opencl: clGetKernelWorkGroupInfo failed for %s DK=%d DV=%d (err=%d); skipping registration\n", + name, dk, dv, err); + return false; + } + if (kwg < required_wg) { + GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d per-kernel max %zu < required %zu; skipping registration (will fall back)\n", + name, dk, dv, kwg, required_wg); + return false; + } + return true; +} + +// Log private memory for an FA kernel. Enable via `GGML_OPENCL_FA_LOG_SPILL=1`. +// On Adreno non-zero private_mem means spilling to global memory due to resource +// constraint and usually causes performance degradation. +// (per-work-item, no cache locality) — a strong signal to pick a config +// with smaller per-thread state (e.g. larger N_SPLIT). +static void ggml_opencl_log_fa_kernel_spill(ggml_backend_opencl_context * backend_ctx, + cl_kernel kernel, const char * name, int dk, int dv) { + static const bool enabled = []{ + const char * e = std::getenv("GGML_OPENCL_FA_LOG_SPILL"); + return e && e[0] && e[0] != '0'; + }(); + + if (!enabled || kernel == nullptr) { + return; + } + + cl_ulong priv_mem = 0; + if (clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, + sizeof(priv_mem), &priv_mem, NULL) == CL_SUCCESS) { + const char * tag = priv_mem > 0 ? "SPILL" : "ok"; + GGML_LOG_INFO("ggml_opencl: [%s] %s DK=%d DV=%d private_mem=%llu bytes\n", + tag, name, dk, dv, (unsigned long long) priv_mem); + } +} + +static void ggml_opencl_ensure_fa_pre_kernels(ggml_backend_opencl_context * backend_ctx, int dk, int dv) { + const std::pair<int, int> dk_dv = {dk, dv}; + + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; + } + } + + if (cfg == nullptr) { + GGML_ABORT("ggml_opencl: no flash_attn config for DK=%d DV=%d", dk, dv); + } + + // BM-tile metadata is consumed by the prefill dispatch (n_q_blocks / wg + // sizing) regardless of whether the prepass kernels are needed for this + // n_kv — set it unconditionally + backend_ctx->fa.f32_f16_bm[{dk, dv}] = cfg->bm; + backend_ctx->fa.f32_f16_bn[{dk, dv}] = cfg->bn; + backend_ctx->fa.f32_f16_wg_size[{dk, dv}] = cfg->bm; + backend_ctx->fa.bm[{dk, dv}] = cfg->bm; + backend_ctx->fa.bn[{dk, dv}] = cfg->bn; + + if (backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0) { return; } + + GGML_LOG_INFO("ggml_opencl: lazy-compiling flash_attn prepass for DK=%d DV=%d\n", dk, dv); + cl_int err; + const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_PRE); + const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_PRE); + // retry when kernel compile fails + cl_program prog_pre_f16 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts, + /*fatal=*/false, "fa prepass f16", backend_ctx->queue); + if (!prog_pre_f16) { return; } + cl_kernel k_kv_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_kv_pad_f16", &err); + if (err != CL_SUCCESS) { clReleaseProgram(prog_pre_f16); return; } + cl_kernel k_mask_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_mask_pad_f16", &err); + if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseProgram(prog_pre_f16); return; } + cl_kernel k_blk_f16 = clCreateKernel(prog_pre_f16, "flash_attn_blk_f16", &err); + if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseKernel(k_mask_pad_f16); clReleaseProgram(prog_pre_f16); return; } + backend_ctx->fa.kv_pad_f16[{dk, dv}] = k_kv_pad_f16; + backend_ctx->fa.mask_pad_f16[{dk, dv}] = k_mask_pad_f16; + backend_ctx->fa.blk_f16[{dk, dv}] = k_blk_f16; + // optional, the decomposed prefill checks for it and falls back to the fused tile + cl_kernel k_v_transpose_f16 = clCreateKernel(prog_pre_f16, "flash_attn_v_transpose_f16", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.v_transpose_f16[{dk, dv}] = k_v_transpose_f16; + } + if (backend_ctx->kernel_fa_v_transpose_q8 == nullptr && backend_ctx->has_integer_dot) { + backend_ctx->kernel_fa_v_transpose_q8 = clCreateKernel(prog_pre_f16, "flash_attn_v_transpose_q8", &err); + if (err != CL_SUCCESS) { + backend_ctx->kernel_fa_v_transpose_q8 = nullptr; + } + } + clReleaseProgram(prog_pre_f16); +} + +static bool ggml_opencl_ensure_fa_f32_f16_vec_512(ggml_backend_opencl_context * backend_ctx) { + const std::pair<int, int> key = {512, 512}; + auto & fa = backend_ctx->fa; + if (fa.f32_f16_q1_vec.count(key) > 0) { + return true; + } + if (fa.f32_f16_vec_512_attempted || backend_ctx->kernel_compile_opts.empty()) { + return false; + } + fa.f32_f16_vec_512_attempted = true; + + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == 512 && d.dv == 512) { + cfg = &d; + break; + } + } + if (cfg == nullptr) { + return false; + } + + // Compile only vec decode and merge to stay within the Adreno compiler's memory limit. + const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_F32_F16) + + " -D FA_DECODE_ONLY -D FA_VEC_ONLY"; + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, + ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts, + /*fatal=*/false, "fa f32_f16 decode512 vec", backend_ctx->queue); + if (!prog) { + return false; + } + cl_int err; + cl_kernel vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err); + if (err != CL_SUCCESS) { + clReleaseProgram(prog); + return false; + } + cl_kernel merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); + clReleaseProgram(prog); + if (err != CL_SUCCESS) { + clReleaseKernel(vec); + return false; + } + if (!ggml_opencl_fa_kernel_fits_wg(backend_ctx, vec, 256, "flash_attn_f32_f16_q1_vec", 512, 512) || + !ggml_opencl_fa_kernel_fits_wg(backend_ctx, merge, 128, "flash_attn_f32_merge", 512, 512)) { + clReleaseKernel(vec); + clReleaseKernel(merge); + return false; + } + fa.f32_f16_q1_vec[key] = vec; + if (fa.f32_merge.count(key) > 0) { + clReleaseKernel(merge); + } else { + fa.f32_merge[key] = merge; + } + return true; +} + +static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context * backend_ctx, int dk, int dv, int gqa) { + const std::tuple<int, int, int> key = {dk, dv, gqa}; + auto & fa = backend_ctx->fa; + if (fa.f32_f16_mq_decode.count(key) > 0 || fa.f32_f16_mq_decode_attempted.count(key) > 0 || + backend_ctx->kernel_compile_opts.empty()) { + return; + } + if (gqa == 4 && dk != 128 && fa.f32_f16_q1_vec_mq_split.count({dk, dv}) > 0) { + fa.f32_f16_mq_decode[key] = fa.f32_f16_q1_vec_mq_split.at({dk, dv}); + fa.f32_f16_mq_decode_wg[key] = 256; + fa.f32_f16_mq_decode_hs[key] = 1; + return; + } + fa.f32_f16_mq_decode_attempted.insert(key); + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; + break; + } + } + if (cfg == nullptr) { + return; + } + + const bool cluster = dk == 64 || dk == 128; + const int head_sub = gqa == 16 ? (cluster ? 4 : 8) : + cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1); + const int nsg_max = dk == 64 ? 1 : (dk == 128 || (gqa == 8 && dk == 256) ? 2 : 4); + const char * kernel_name = cluster ? "flash_attn_f32_f16_q1_vec_mq_split_c8" : "flash_attn_f32_f16_q1_vec_mq_split"; + const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16); + const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_F32_F16) + + " -D FA_MQ_ONLY -D MQ_GQA=" + std::to_string(gqa / head_sub) + + " -D FA_HEAD_SUB=" + std::to_string(head_sub) + + (cluster ? " -D MQ_NSG=" + std::to_string(nsg_max) + " -D FA_CL_C=16" : " -D FA_MQ_SPLIT_ONLY") + + (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : (cluster && gqa == 16 ? " -D FA_CL_MHRED" : "")) + + (gqa == 8 && dk == 256 ? " -D FA_Q1_Q_REG" : ""); + for (int nsg = nsg_max; nsg >= 1; nsg /= 2) { + const size_t wg = 64 * nsg; + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), + opts + " -D MQ_NSG_SPLIT=" + std::to_string(nsg), + /*fatal=*/false, "fa f32_f16 mq decode", backend_ctx->queue); + if (!prog) { + continue; + } + cl_int err; + cl_kernel kernel = clCreateKernel(prog, kernel_name, &err); + clReleaseProgram(prog); + if (err != CL_SUCCESS) { + continue; + } + if (!ggml_opencl_fa_kernel_fits_wg(backend_ctx, kernel, wg, kernel_name, dk, dv)) { + clReleaseKernel(kernel); + continue; + } + fa.f32_f16_mq_decode[key] = kernel; + fa.f32_f16_mq_decode_wg[key] = wg; + fa.f32_f16_mq_decode_hs[key] = head_sub; + return; + } +} + +// DK=512 prefill BM-tile +static bool ggml_opencl_ensure_fa_f32_f16_prefill_512(ggml_backend_opencl_context * backend_ctx, bool split) { + const int dk = 512, dv = 512; + const std::pair<int, int> dk_dv = {dk, dv}; + auto & target = split ? backend_ctx->fa.f32_f16_split : backend_ctx->fa.f32_f16; + if (target.count(dk_dv) > 0) { return true; } + + static bool failed[2] = { false, false }; + if (failed[split ? 1 : 0]) { return false; } + + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { cfg = &d; break; } + } + if (cfg == nullptr) { failed[split ? 1 : 0] = true; return false; } + if (split && cfg->n_split <= 1) { failed[1] = true; return false; } + + const ggml_opencl_fa_variant variant = split ? FA_VARIANT_F32_F16_SPLIT : FA_VARIANT_F32_F16; + std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + " -D FA_PREFILL_ONLY"; + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, + ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts, + /*fatal=*/false, split ? "fa f32_f16 prefill512 split" : "fa f32_f16 prefill512", + backend_ctx->queue); + if (!prog) { failed[split ? 1 : 0] = true; return false; } + + cl_int err; + cl_kernel k = clCreateKernel(prog, "flash_attn_f32_f16", &err); + if (err != CL_SUCCESS) { clReleaseProgram(prog); failed[split ? 1 : 0] = true; return false; } + target[dk_dv] = k; + if (split) { + backend_ctx->fa.f32_f16_split_wg_size[dk_dv] = cfg->bm * cfg->n_split; + backend_ctx->fa.f32_f16_split_nkv_threshold[dk_dv] = cfg->nkv_split_threshold; + } + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, + split ? "flash_attn_f32_f16 (prefill512 split)" : "flash_attn_f32_f16 (prefill512)", dk, dv); + clReleaseProgram(prog); + + // determine whether to use the K-image variant of the split tile + static const char * pkimg_build_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); + const bool pkimg_build = (pkimg_build_env != NULL) && (pkimg_build_env[0] != '0'); + if (split && pkimg_build && backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) == 0) { + std::string opts_img = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + + " -D FA_PREFILL_ONLY -D FA_K_IMG -D FA_TILE_NAME=flash_attn_f32_f16_k_img"; + cl_program prog_img = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, + ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts_img, + /*fatal=*/false, "fa f32_f16 prefill512 split k_img", backend_ctx->queue); + if (prog_img) { + cl_int err_img; + cl_kernel k_img = clCreateKernel(prog_img, "flash_attn_f32_f16_k_img", &err_img); + if (err_img == CL_SUCCESS) { + backend_ctx->fa.f32_f16_split_k_img[dk_dv] = k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_img, + "flash_attn_f32_f16 (prefill512 split k_img)", dk, dv); + } + clReleaseProgram(prog_img); + } + } + return true; +} + +// Compile one (variant, dk, dv); memoised. false = compiler rejected. +static bool ggml_opencl_ensure_fa_variant(ggml_backend_opencl_context * backend_ctx, int dk, int dv, ggml_opencl_fa_variant variant) { + const std::pair<int, int> dk_dv = {dk, dv}; + + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; + } + } + if (cfg == nullptr) { + return false; + } + + // if a variant has already been compiled + switch (variant) { + case FA_VARIANT_F16: { + if (backend_ctx->fa.f16.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32: { + if (backend_ctx->fa.f32.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32_F16: { + // The DK=512 decode-only program does not create the f32_f16 + // prefill kernel; check the q1 kernel instead so that repeated + // calls return a consistent result. + const bool decode_only = (dk == 512); + if (decode_only ? (backend_ctx->fa.f32_f16_q1.count(dk_dv) > 0) + : (backend_ctx->fa.f32_f16.count(dk_dv) > 0)) { + return true; + } + break; + } + case FA_VARIANT_Q8_0: { + if (backend_ctx->fa.f32_q8_0.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q4_0: { + if (backend_ctx->fa.f32_q4_0.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32_F16_SPLIT: { + if (backend_ctx->fa.f32_f16_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q8_0_SPLIT: { + if (backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q4_0_SPLIT: { + if (backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_PRE: { + ggml_opencl_ensure_fa_pre_kernels(backend_ctx, dk, dv); + return true; + } + } + + // not registered but attempted - meaning these kernels failed to compile + const auto attempt_key = std::make_pair(variant, dk_dv); + if (backend_ctx->fa.variant_attempted.count(attempt_key)) { + return false; + } + backend_ctx->fa.variant_attempted.insert(attempt_key); + + const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || + variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0_SPLIT; + const bool is_quant = variant == FA_VARIANT_Q8_0 || variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0 || variant == FA_VARIANT_Q4_0_SPLIT; + if (is_quant && (dk % 32 != 0 || dv % 32 != 0)) { + return false; + } + if (is_split && cfg->n_split <= 1) { + return false; + } + if ((variant == FA_VARIANT_Q8_0_SPLIT || variant == FA_VARIANT_Q4_0_SPLIT) && + ((dk / 32) % cfg->n_split != 0 || (dv / 4) % cfg->n_split != 0)) { + return false; + } + + const std::string src = ggml_opencl_fa_kernel_src(variant); + if (src.empty()) { return false; } + std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant); + + // bypass kernels for DK=512 + const bool fa_decode_only = (variant == FA_VARIANT_F32_F16 && dk == 512); + if (fa_decode_only) { + opts += " -D FA_DECODE_ONLY -D FA_DECODE_MINIMAL"; + } + + // c8 cluster width (GGML_OPENCL_FA_CL_C overrides): value = GQA4 cluster + // width (kernel default 8); the g8 programs use 2x the value (default 16). + // Wider clusters halve per-lane o_acc at the cost of position streams per + // subgroup + static const int fa_cl_c_env = []{ + const char * e = std::getenv("GGML_OPENCL_FA_CL_C"); + const int x = (e && e[0]) ? atoi(e) : 0; + return (x == 8 || x == 16 || x == 32) ? x : 0; // 0 = per-gen default + }(); + // X2E needs 16 to keep per-lane o_acc at 128B (the compiler spills the + // kernel-default width); X1E does not spill, but C=16 is still a measured + // +28-30% DK128-GQA4 decode win there (X1-85, kv 4096/8192), neutral on + // DK64 / GQA1 / quant-KV. + const int fa_cl_c_gqa4 = fa_cl_c_env ? fa_cl_c_env + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E ? 16 : 0); + const std::string opts_cl_c_gqa4 = fa_cl_c_gqa4 + ? " -D FA_CL_C=" + std::to_string(fa_cl_c_gqa4) : std::string(); + const std::string fa_cl_c_g8_val = std::to_string(fa_cl_c_gqa4 ? fa_cl_c_gqa4 * 2 : 16); + + const char * tag = nullptr; + switch (variant) { + case FA_VARIANT_F16: tag = "fa f16"; break; + case FA_VARIANT_F32: tag = "fa f32"; break; + case FA_VARIANT_F32_F16: tag = "fa f32_f16"; break; + case FA_VARIANT_Q8_0: tag = "fa q8_0"; break; + case FA_VARIANT_Q4_0: tag = "fa q4_0"; break; + case FA_VARIANT_F32_F16_SPLIT: tag = "fa f32_f16 split"; break; + case FA_VARIANT_Q8_0_SPLIT: tag = "fa q8_0 split"; break; + case FA_VARIANT_Q4_0_SPLIT: tag = "fa q4_0 split"; break; + default: break; + } + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts + opts_cl_c_gqa4, + /*fatal=*/false, tag, backend_ctx->queue); + if (!prog) { return false; } + + cl_int err; + switch (variant) { + case FA_VARIANT_F16: { + cl_kernel k, kq1; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f16", &err), err)); + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f16_q1", &err), err)); + backend_ctx->fa.f16[{dk, dv}] = k; + backend_ctx->fa.f16_q1[{dk, dv}] = kq1; + break; + } + case FA_VARIANT_F32: { + cl_kernel k, kq1; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32", &err), err)); + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_q1", &err), err)); + backend_ctx->fa.f32[{dk, dv}] = k; + backend_ctx->fa.f32_q1[{dk, dv}] = kq1; + break; + } + case FA_VARIANT_F32_F16: { + cl_kernel kq1; + // BM-tile prefill kernel is excluded from the decode-only (DK=512) + if (!fa_decode_only) { + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + backend_ctx->fa.f32_f16[{dk, dv}] = k; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, "flash_attn_f32_f16", dk, dv); + } + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_f16_q1", &err), err)); + backend_ctx->fa.f32_f16_q1[{dk, dv}] = kq1; + ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, "flash_attn_f32_f16_q1", dk, dv); + cl_kernel k_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_split", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_f16_q1_split[{dk, dv}] = k_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, "flash_attn_f32_f16_q1_split", dk, dv); + } + // q1_vec decode kernel (DV-split + subgroup reduce) + cl_kernel k_q1_vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, + "flash_attn_f32_f16_q1_vec", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec[{dk, dv}] = k_q1_vec; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, "flash_attn_f32_f16_q1_vec", dk, dv); + } else { + clReleaseKernel(k_q1_vec); + } + } + // KV-head-coalesced vec for high-GQA small models + cl_kernel k_q1_vec_mq = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq, 256, + "flash_attn_f32_f16_q1_vec_mq", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq[{dk, dv}] = k_q1_vec_mq; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq, "flash_attn_f32_f16_q1_vec_mq", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq); + } + } + // KV-head-coalesced + flash-decoding split, reuses merge kernel + cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, + "flash_attn_f32_f16_q1_vec_mq_split", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split[{dk, dv}] = k_q1_vec_mq_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, "flash_attn_f32_f16_q1_vec_mq_split", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split); + } + } + // K-image variant of MQ_GQA=4 split + cl_kernel k_q1_vec_mq_split_k_img = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_k_img, 256, + "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img[{dk, dv}] = k_q1_vec_mq_split_k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_k_img, "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_k_img); + } + } + // Cluster-parallel decode variant + cl_kernel k_q1_vec_mq_split_c8 = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_c8, 256, + "flash_attn_f32_f16_q1_vec_mq_split_c8", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_c8[{dk, dv}] = k_q1_vec_mq_split_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_c8); + } + } + cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; + } + // local-tile decode variant + if (dk == 128 && dv == 128) { + cl_kernel k_lt = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_tile", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lt, 128, + "flash_attn_f32_f16_q1_local_tile", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_tile[{dk, dv}] = k_lt; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lt, "flash_attn_f32_f16_q1_local_tile", dk, dv); + } else { + clReleaseKernel(k_lt); + } + } + // hybrid local-tile + MQ + FD-split + cl_kernel k_lmq = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq, 64, + "flash_attn_f32_f16_q1_local_mq_split", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_mq_split[{dk, dv}] = k_lmq; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq, "flash_attn_f32_f16_q1_local_mq_split", dk, dv); + } else { + clReleaseKernel(k_lmq); + } + } + } + + // second compile of the same source with -DMQ_GQA=8. + // FA_MQ_ONLY keeps only the vec_mq kernels so that the program + // compiles within the Adreno compiler's memory budget at DK>=256. + // FA_CL_C for the g8 program: MQ_GQA=8 doubles the c8 kernel's + // per-lane o_acc, so widen the cluster to keep the register + // footprint inside the 192-thread WG cap (see fa_cl_c_gqa4 above + // for the per-gen default). + const std::string opts_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3 -D FA_MQ_ONLY -D FA_CL_C=" + fa_cl_c_g8_val; + cl_program prog_g8 = fa_decode_only ? nullptr : build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8, + /*fatal=*/false, "fa f32_f16 MQ_GQA=8", backend_ctx->queue); + if (prog_g8) { + const size_t mq_g8_required_wg = 192; // Q1_WG_SIZE(64) * MQ_NSG_SPLIT(3) + cl_kernel k_q1_vec_mq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_g8, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_g8[{dk, dv}] = k_q1_vec_mq_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_g8, "flash_attn_f32_f16_q1_vec_mq_g8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_g8); + } + } + cl_kernel k_q1_vec_mq_split_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq_split (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8[{dk, dv}] = k_q1_vec_mq_split_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8, "flash_attn_f32_f16_q1_vec_mq_split_g8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_g8); + } + } + // K-image variant + cl_kernel k_q1_vec_mq_split_g8_k_img = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_k_img, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq_split_k_img (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img[{dk, dv}] = k_q1_vec_mq_split_g8_k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_k_img, "flash_attn_f32_f16_q1_vec_mq_split_g8_k_img", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_g8_k_img); + } + } + // Cluster-parallel decode, MQ_GQA=8 / FA_CL_C=16 specialization + cl_kernel k_q1_vec_mq_split_g8_c8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_c8, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8[{dk, dv}] = k_q1_vec_mq_split_g8_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_g8_c8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_g8_c8); + } + } + // hybrid local-tile + MQ_GQA=8 + if (dk == 128 && dv == 128) { + cl_kernel k_lmq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_local_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq_g8, 64, + "flash_attn_f32_f16_q1_local_mq_split (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_mq_split_g8[{dk, dv}] = k_lmq_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq_g8, "flash_attn_f32_f16_q1_local_mq_split_g8", dk, dv); + } else { + clReleaseKernel(k_lmq_g8); + } + } + } + clReleaseProgram(prog_g8); + } + // NSG_SPLIT=2 programs for the cluster-parallel kernel: its register + // footprint caps the per-kernel WG at 128 on X2 (< the stock 256/192 + // requirement), so it can never register from the stock programs. + // With FA_CL_NCL position streams per subgroup, 2 subgroups still + // carry 16 in-flight rows per WG (baseline WG has 4). FA_MQ_ONLY + // keeps these compiles minimal; skipped when the stock program c8 + // registered (some other device) or shuffles are absent. + if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && + backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count({dk, dv}) == 0) { + const std::string opts_c8_ns2 = opts + " -D FA_MQ_ONLY -D MQ_GQA=4 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2" + opts_cl_c_gqa4; + cl_program prog_c8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8_ns2, + /*fatal=*/false, "fa f32_f16 c8 NSG2", backend_ctx->queue); + if (prog_c8) { + cl_kernel k_c8 = clCreateKernel(prog_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + // WG = MQ_NSG(2) × Q1_WG_SIZE(=FA_SG): 128 Adreno (64), 64 Intel (32). + const size_t c8_ns2_wg = backend_ctx->gpu_family == INTEL ? 64 : 128; + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8, c8_ns2_wg, + "flash_attn_f32_f16_q1_vec_mq_split_c8 (ns2)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2[{dk, dv}] = k_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8_ns2", dk, dv); + } else { + clReleaseKernel(k_c8); + } + } + clReleaseProgram(prog_c8); + } + } + // FA_CL_C=32 g8 program for the DK=DV=256 GQA=8 + if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && + dk == 256 && dv == 256) { + const std::string opts_g8_c32 = opts + " -D FA_MQ_ONLY -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2 -D FA_CL_C=32"; + cl_program prog_g8_c32 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8_c32, + /*fatal=*/false, "fa f32_f16 c32 g8 d256 NSG2", backend_ctx->queue); + if (prog_g8_c32) { + cl_kernel k_g8_c32 = clCreateKernel(prog_g8_c32, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8_c32, 128, + "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8 c32 d256)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32[{dk, dv}] = k_g8_c32; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8_c32, "flash_attn_f32_f16_q1_vec_mq_split_g8_c32", dk, dv); + } else { + clReleaseKernel(k_g8_c32); + } + } + clReleaseProgram(prog_g8_c32); + } + } + if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count({dk, dv}) == 0) { + const std::string opts_g8_c8_ns2 = opts + " -D FA_MQ_ONLY -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2 -D FA_CL_C=" + fa_cl_c_g8_val; + cl_program prog_g8_c8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8_c8_ns2, + /*fatal=*/false, "fa f32_f16 c8 g8 NSG2", backend_ctx->queue); + if (prog_g8_c8) { + cl_kernel k_g8_c8 = clCreateKernel(prog_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8_c8, 128, + "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8 ns2)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2[{dk, dv}] = k_g8_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_g8_c8_ns2", dk, dv); + } else { + clReleaseKernel(k_g8_c8); + } + } + clReleaseProgram(prog_g8_c8); + } + } + break; + } + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q4_0: { + const bool is_q8 = variant == FA_VARIANT_Q8_0; + const std::string base = is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0"; + const std::string name_q1 = base + "_q1"; + const std::string name_q1_split = base + "_q1_split"; + auto & m_q1 = is_q8 ? backend_ctx->fa.f32_q8_0_q1 : backend_ctx->fa.f32_q4_0_q1; + auto & m_prefill = is_q8 ? backend_ctx->fa.f32_q8_0 : backend_ctx->fa.f32_q4_0; + auto & m_q1_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_split : backend_ctx->fa.f32_q4_0_q1_split; + + cl_kernel k, kq1; + CL_CHECK((kq1 = clCreateKernel(prog, name_q1.c_str(), &err), err)); + CL_CHECK((k = clCreateKernel(prog, base.c_str(), &err), err)); + m_q1[{dk, dv}] = kq1; + m_prefill[{dk, dv}] = k; + ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, name_q1.c_str(), dk, dv); + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, base.c_str(), dk, dv); + cl_kernel k_split = clCreateKernel(prog, name_q1_split.c_str(), &err); + if (err == CL_SUCCESS) { + m_q1_split[{dk, dv}] = k_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, name_q1_split.c_str(), dk, dv); + } + + // DV-split decode variant (q1_vec) + auto & m_q1_vec = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec : backend_ctx->fa.f32_q4_0_q1_vec; + const std::string name_q1_vec = name_q1 + "_vec"; + cl_kernel k_q1_vec = clCreateKernel(prog, name_q1_vec.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, + name_q1_vec.c_str(), dk, dv)) { + m_q1_vec[{dk, dv}] = k_q1_vec; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, name_q1_vec.c_str(), dk, dv); + } else { + clReleaseKernel(k_q1_vec); + } + } + + // KV-head-coalesced + flash-decoding split + auto & m_mq_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split + : backend_ctx->fa.f32_q4_0_q1_vec_mq_split; + const std::string name_mq_split = name_q1 + "_vec_mq_split"; + cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, name_mq_split.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, + name_mq_split.c_str(), dk, dv)) { + m_mq_split[{dk, dv}] = k_q1_vec_mq_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, name_mq_split.c_str(), dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split); + } + } + if (!backend_ctx->fa.f32_merge.count({dk, dv})) { + cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; + } + } + // Second compile with MQ_GQA=8, MQ_NSG=3, MQ_NSG_SPLIT=3 + auto & m_mq_split_g8 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8 + : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8; + const std::string opts_mq_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3"; + cl_program prog_mq_g8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_mq_g8, + /*fatal=*/false, is_q8 ? "fa q8_0 MQ_GQA=8" : "fa q4_0 MQ_GQA=8", + backend_ctx->queue); + if (prog_mq_g8) { + const size_t mq_g8_required_wg = 192; + cl_kernel k_g8 = clCreateKernel(prog_mq_g8, name_mq_split.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8, mq_g8_required_wg, + name_mq_split.c_str(), dk, dv)) { + m_mq_split_g8[{dk, dv}] = k_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8, name_mq_split.c_str(), dk, dv); + } else { + clReleaseKernel(k_g8); + } + } + clReleaseProgram(prog_mq_g8); + } + // GQA=4 cluster-parallel program (NSG_SPLIT=2 / WG=128) + if (backend_ctx->has_subgroup_shuffle) { + auto & m_c8_gqa4 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8 + : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8; + const std::string name_c8_gqa4 = name_q1 + "_vec_mq_split_c8"; + const std::string opts_c8_gqa4 = opts + " -D MQ_GQA=4 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2" + opts_cl_c_gqa4; + cl_program prog_c8_gqa4 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8_gqa4, + /*fatal=*/false, is_q8 ? "fa q8_0 c8 GQA4 NSG2" : "fa q4_0 c8 GQA4 NSG2", + backend_ctx->queue); + if (prog_c8_gqa4) { + cl_kernel k_c8_gqa4 = clCreateKernel(prog_c8_gqa4, name_c8_gqa4.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8_gqa4, 128, + name_c8_gqa4.c_str(), dk, dv)) { + m_c8_gqa4[{dk, dv}] = k_c8_gqa4; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8_gqa4, name_c8_gqa4.c_str(), dk, dv); + } else { + clReleaseKernel(k_c8_gqa4); + } + } + clReleaseProgram(prog_c8_gqa4); + } + } + // Cluster-parallel q4_0 decode kernel + if (!is_q8 && backend_ctx->has_subgroup_shuffle) { + const std::string opts_c8 = opts + " -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2"; + cl_program prog_c8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8, + /*fatal=*/false, "fa q4_0 c8 NSG2", backend_ctx->queue); + if (prog_c8) { + cl_kernel k_c8 = clCreateKernel(prog_c8, "flash_attn_f32_q4_0_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8, 128, + "flash_attn_f32_q4_0_q1_vec_mq_split_c8 (g8 ns2)", dk, dv)) { + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8[{dk, dv}] = k_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8, "flash_attn_f32_q4_0_q1_vec_mq_split_g8_c8", dk, dv); + } else { + clReleaseKernel(k_c8); + } + } + clReleaseProgram(prog_c8); + } + } + break; + } + case FA_VARIANT_F32_F16_SPLIT: { + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + backend_ctx->fa.f32_f16_split[{dk, dv}] = k; + backend_ctx->fa.f32_f16_split_wg_size[{dk, dv}] = cfg->bm * cfg->n_split; + backend_ctx->fa.f32_f16_split_nkv_threshold[{dk, dv}] = cfg->nkv_split_threshold; + break; + } + case FA_VARIANT_Q8_0_SPLIT: + case FA_VARIANT_Q4_0_SPLIT: { + const bool is_q8 = variant == FA_VARIANT_Q8_0_SPLIT; + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0", &err), err)); + auto & split = is_q8 ? backend_ctx->fa.f32_q8_0_split : backend_ctx->fa.f32_q4_0_split; + auto & split_wg = is_q8 ? backend_ctx->fa.f32_q8_0_split_wg_size : backend_ctx->fa.f32_q4_0_split_wg_size; + auto & split_bm = is_q8 ? backend_ctx->fa.f32_q8_0_split_bm : backend_ctx->fa.f32_q4_0_split_bm; + auto & split_thresh = is_q8 ? backend_ctx->fa.f32_q8_0_split_nkv_threshold : backend_ctx->fa.f32_q4_0_split_nkv_threshold; + split[{dk, dv}] = k; + split_wg[{dk, dv}] = cfg->bm * cfg->n_split; + split_bm[{dk, dv}] = cfg->bm; + split_thresh[{dk, dv}] = 0; // quant prefill: always split + break; + } + default: + break; + } + CL_CHECK(clReleaseProgram(prog)); + return true; +} + +// Compile a quant FA split kernel with a hand-picked (BLOCK_M, N_SPLIT) that +// overrides the default fa_dims tuning, for the DK values where the default +// N_SPLIT is degenerate for quant prefill: +// DK=256: default N_SPLIT=16 leaves DK/32=8 blocks -> 0 blocks/split. +// Override N_SPLIT=8 (1 block/split), BLOCK_M=16. +// DK=96 : DK/32 = 3 blocks, not divisible by the default N_SPLIT=2 -> +// override N_SPLIT=3. BLOCK_M must be 16, not 32: the N_SPLIT=3 +// QK-partial reduction uses sub_group_shuffle, so all 3 split +// threads of a query must land in one subgroup — WG_SIZE = +// BLOCK_M*N_SPLIT must be <= the 64-lane Adreno subgroup (16*3=48). +static bool ggml_opencl_ensure_fa_quant_split_override( + ggml_backend_opencl_context * backend_ctx, + int dk, int dv, int quant_bm, int quant_n_split, bool is_q8_0 +) { + const std::pair<int, int> dk_dv = {dk, dv}; + if (is_q8_0 && backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { + return true; + } + if (!is_q8_0 && backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { + return true; + } + + const ggml_opencl_fa_variant variant = is_q8_0 ? FA_VARIANT_Q8_0_SPLIT : FA_VARIANT_Q4_0_SPLIT; + const auto attempt_key = std::make_pair(variant, dk_dv); + if (backend_ctx->fa.variant_attempted.count(attempt_key)) { + return false; + } + + backend_ctx->fa.variant_attempted.insert(attempt_key); + + std::string shuffle_opts; + if (backend_ctx->has_subgroup_shuffle) { + shuffle_opts = backend_ctx->has_qcom_subgroup_shuffle + ? " -D cl_qcom_subgroup_shuffle=1" + : " -D cl_khr_subgroup_shuffle=1"; + } + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; + } + } + if (cfg == nullptr) { + return false; + } + + // BLK_PREPASS_BM is the prepass-kernel BLOCK_M, needed so the quant kernel + // indexes the blk[] classification buffer correctly. + std::string opts = backend_ctx->kernel_compile_opts + shuffle_opts + + " -D DK=" + std::to_string(dk) + + " -D DV=" + std::to_string(dv) + + " -D BLOCK_M=" + std::to_string(quant_bm) + + " -D BLOCK_N=" + std::to_string(cfg->bn) + + " -D N_SPLIT=" + std::to_string(quant_n_split) + + " -D BLK_PREPASS_BM=" + std::to_string(cfg->bm); + + const std::string src = ggml_opencl_fa_kernel_src(variant); + if (src.empty()) { + return false; + } + + const std::string tag = std::string("fa ") + (is_q8_0 ? "q8_0" : "q4_0") + + " split DK=" + std::to_string(dk); + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts, + /*fatal=*/false, tag.c_str(), backend_ctx->queue); + if (!prog) { return false; } + cl_int err; + cl_kernel k; + if (is_q8_0) { + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q8_0", &err), err)); + backend_ctx->fa.f32_q8_0_split[dk_dv] = k; + backend_ctx->fa.f32_q8_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; + backend_ctx->fa.f32_q8_0_split_bm[dk_dv] = quant_bm; + backend_ctx->fa.f32_q8_0_split_nkv_threshold[dk_dv] = 0; + } else { + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q4_0", &err), err)); + backend_ctx->fa.f32_q4_0_split[dk_dv] = k; + backend_ctx->fa.f32_q4_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; + backend_ctx->fa.f32_q4_0_split_bm[dk_dv] = quant_bm; + backend_ctx->fa.f32_q4_0_split_nkv_threshold[dk_dv] = 0; + } + CL_CHECK(clReleaseProgram(prog)); + return true; +} + +namespace /* anonymous */ { +extern struct ggml_backend_device_i ggml_backend_opencl_device_i; +} + +// Look for available and suitable devices. +static std::vector<ggml_backend_device> ggml_opencl_probe_devices(ggml_backend_reg * reg) { + std::vector<ggml_backend_device> found_devices; + +#ifdef GGML_OPENCL_PROFILING + GGML_LOG_INFO("ggml_opencl: OpenCL profiling enabled\n"); +#endif + + struct cl_device; + struct cl_platform { + cl_platform_id id; + unsigned number; + char name[128]; + char vendor[128]; + struct cl_device * devices; + unsigned n_devices; + struct cl_device * default_device; + }; + + struct cl_device { + struct cl_platform * platform; + cl_device_id id; + unsigned number; + cl_device_type type; + char name[128]; + char version[128]; + }; + + enum { NPLAT = 16, NDEV = 16 }; + + struct cl_platform platforms[NPLAT]; + unsigned n_platforms = 0; + struct cl_device devices[NDEV]; + unsigned n_devices = 0; + struct cl_device * default_device = NULL; + unsigned default_platform_number = 0; + + cl_platform_id platform_ids[NPLAT]; + if (clGetPlatformIDs(NPLAT, platform_ids, &n_platforms) != CL_SUCCESS) { + GGML_LOG_ERROR("ggml_opencl: platform IDs not available.\n"); + return found_devices; + } + + for (unsigned i = 0; i < n_platforms; i++) { + struct cl_platform * p = &platforms[i]; + p->number = i; + p->id = platform_ids[i]; + CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL)); + CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL)); + + cl_device_id device_ids[NDEV]; + cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices); + if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) { + p->n_devices = 0; + } else { + CL_CHECK(clGetDeviceIDsError); + } + p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL; + p->default_device = NULL; + + for (unsigned j = 0; j < p->n_devices; j++) { + struct cl_device * d = &devices[n_devices]; + d->number = n_devices++; + d->id = device_ids[j]; + d->platform = p; + CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL)); + CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL)); + CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_VERSION, sizeof(d->version), &d->version, NULL)); + + if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) { + p->default_device = d; + } + } + + if (default_device == NULL && p->default_device != NULL) { + default_device = p->default_device; + default_platform_number = i; + } + } + + if (n_devices == 0) { + GGML_LOG_ERROR("ggml_opencl: could find any OpenCL devices.\n"); + return found_devices; + } + + char * user_platform_string = getenv("GGML_OPENCL_PLATFORM"); + char * user_device_string = getenv("GGML_OPENCL_DEVICE"); + int user_platform_number = -1; + int user_device_number = -1; + cl_device * candidate_devices = nullptr; + unsigned n_candidate_devices = 0; + + unsigned n; + if (user_platform_string != NULL && sscanf(user_platform_string, " %u", &n) == 1 && n < n_platforms) { + user_platform_number = (int)n; + } + if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1 && n < n_devices) { + user_device_number = (int)n; + } + if (user_platform_number != -1 && user_device_number != -1) { + cl_platform* platform = &platforms[user_platform_number]; + if ((unsigned)user_device_number >= platform->n_devices) { + GGML_LOG_ERROR("ggml_opencl: invalid device number %d\n", user_device_number); + exit(1); + } + default_device = &platform->devices[user_device_number]; + candidate_devices = platform->devices; + n_candidate_devices = platform->n_devices; + } else { + // Choose a platform by matching a substring. + if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) { + for (unsigned i = 0; i < n_platforms; i++) { + struct cl_platform * p = &platforms[i]; + if (strstr(p->name, user_platform_string) != NULL || + strstr(p->vendor, user_platform_string) != NULL) { + user_platform_number = (int)i; + break; + } + } + if (user_platform_number == -1) { + GGML_LOG_ERROR("ggml_opencl: no platform matching '%s' was found.\n", user_platform_string); + exit(1); + } + } + + int platform_idx = user_platform_number != -1 ? user_platform_number : default_platform_number; + struct cl_platform * p = &platforms[platform_idx]; + candidate_devices = p->devices; + n_candidate_devices = p->n_devices; + default_device = p->default_device; + if (n_candidate_devices == 0) { + GGML_LOG_ERROR("ggml_opencl: selected platform '%s' does not have any devices.\n", p->name); + exit(1); + } + + if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) { + for (unsigned i = 0; i < n_candidate_devices; i++) { + struct cl_device * d = &candidate_devices[i]; + if (strstr(d->name, user_device_string) != NULL) { + user_device_number = d->number; + break; + } + } + if (user_device_number == -1) { + GGML_LOG_ERROR("ggml_opencl: no device matching '%s' was found.\n", user_device_string); + exit(1); + } + } + if (user_device_number != -1) { + candidate_devices = &devices[user_device_number]; + n_candidate_devices = 1; + default_device = &candidate_devices[0]; + } + + GGML_ASSERT(n_candidate_devices > 0); + + if (default_device == NULL) { + default_device = &candidate_devices[0]; + } + } + + GGML_ASSERT(n_candidate_devices != 0 && candidate_devices); + + // Put the default device in front. + for (unsigned i = 1; i < n_candidate_devices; i++) { + if (&candidate_devices[i] == default_device) { + std::swap(candidate_devices[0], candidate_devices[i]); + default_device = &candidate_devices[0]; + break; + } + } + + GGML_LOG_INFO("ggml_opencl: selected platform: '%s'\n", default_device->platform->name); + + std::vector<cl_device_id> device_ids; + for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { + device_ids.push_back(dev->id); + } + + cl_int err; + cl_context shared_context; + cl_context_properties properties[] = { (intptr_t) CL_CONTEXT_PLATFORM, (intptr_t) default_device->platform->id, 0 }; + + CL_CHECK( + (shared_context = clCreateContext(properties, device_ids.size(), device_ids.data(), NULL, NULL, &err), err)); + + for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { + GGML_LOG_INFO("\nggml_opencl: device: '%s (%s)'\n", dev->name, dev->version); + + auto dev_ctx = std::unique_ptr<ggml_backend_opencl_device_context>(new ggml_backend_opencl_device_context{ + /*.platform =*/dev->platform->id, + /*.platform_nane =*/dev->platform->name, + /*.device =*/dev->id, + /*.device_name =*/dev->name, + /*.device_type =*/dev->type, + /*.device_version =*/dev->version, + /*.backend_ctx =*/nullptr, + /*.buffer_type =*/{}, + /*.context =*/shared_context, + }); + + found_devices.push_back(ggml_backend_device{ + /* .iface = */ ggml_backend_opencl_device_i, + /* .reg = */ reg, + /* .context = */ dev_ctx.get(), + }); + + if (!ggml_opencl_is_device_supported(&found_devices.back())) { + found_devices.pop_back(); + GGML_LOG_WARN("ggml_opencl: drop unsupported device '%s'.\n", dev->name); + continue; + } + + g_ggml_backend_opencl_dev_ctxs.push_back(std::move(dev_ctx)); + } + + if (found_devices.size()) { + auto * dev_ctx = static_cast<ggml_backend_opencl_device_context *>(found_devices.front().context); + GGML_LOG_INFO("ggml_opencl: default device: '%s (%s)'\n", dev_ctx->device_name.c_str(), + dev_ctx->device_version.c_str()); + + if (dev_ctx->device_type != CL_DEVICE_TYPE_GPU) { + GGML_LOG_WARN("ggml_opencl: warning, the default device is not a GPU: '%s'.\n", + dev_ctx->device_name.c_str()); + } + } + + return found_devices; +} + +static void ggml_opencl_print_backend_info(ggml_backend_opencl_device_context * dev_ctx) { + GGML_ASSERT(dev_ctx); + GGML_ASSERT(dev_ctx->backend_ctx); + + auto * backend_ctx = dev_ctx->backend_ctx; + + GGML_LOG_INFO("ggml_opencl: OpenCL device: %s\n", + backend_ctx->device_name.c_str()); + GGML_LOG_INFO("ggml_opencl: OpenCL driver: %s\n", + backend_ctx->driver_version.c_str()); + GGML_LOG_INFO("ggml_opencl: vector subgroup broadcast support: %s\n", + backend_ctx->has_vector_subgroup_broadcast ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: subgroup shuffle support: %s\n", + backend_ctx->has_subgroup_shuffle ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: device FP16 support: %s\n", + backend_ctx->fp16_support ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: khr dot product support: %s\n", + backend_ctx->has_integer_dot ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: mem base addr align: %u\n", + backend_ctx->alignment); + GGML_LOG_INFO("ggml_opencl: global mem size: %zu MB\n", + backend_ctx->global_mem_size/1024/1024); + GGML_LOG_INFO("ggml_opencl: max mem alloc size: %zu MB\n", + backend_ctx->max_alloc_size/1024/1024); + GGML_LOG_INFO("ggml_opencl: device max image buffer size (pixels): %zu\n", + backend_ctx->image_max_buffer_size); + GGML_LOG_INFO("ggml_opencl: device max image2d size: %zu x %zu\n", + backend_ctx->image2d_max_width, backend_ctx->image2d_max_height); + GGML_LOG_INFO("ggml_opencl: device max workgroup size: %zu\n", + backend_ctx->max_workgroup_size); + GGML_LOG_INFO("ggml_opencl: SVM coarse grain buffer support: %s\n", + backend_ctx->svm_caps & CL_DEVICE_SVM_COARSE_GRAIN_BUFFER ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: SVM fine grain buffer support: %s\n", + backend_ctx->svm_caps & CL_DEVICE_SVM_FINE_GRAIN_BUFFER ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: SVM fine grain system support: %s\n", + backend_ctx->svm_caps & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: SVM atomics support: %s\n", + backend_ctx->svm_caps & CL_DEVICE_SVM_ATOMICS ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: cl_qcom_subgroup_shuffle support: %s\n", + backend_ctx->has_qcom_subgroup_shuffle ? "true" : "false"); + + // Print out configurations +#ifdef GGML_OPENCL_SOA_Q + GGML_LOG_INFO("ggml_opencl: flattening quantized weights representation as struct of arrays (GGML_OPENCL_SOA_Q)\n"); +#endif // GGML_OPENCL_SOA_Q + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_LOG_INFO("ggml_opencl: using kernels optimized for Adreno (GGML_OPENCL_USE_ADRENO_KERNELS)\n"); + if (backend_ctx->adreno_xmem_gemm_enabled) { + GGML_LOG_INFO("ggml_opencl: Adreno xmem F16xF32 GEMM enabled (temporary weight prepack)\n"); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + if (backend_ctx->adreno_use_large_buffer) { + if (!backend_ctx->adreno_has_large_buffer) { + GGML_LOG_INFO("ggml_opencl: Adreno large buffer requested but not supported by driver, will use regular buffer\n"); + backend_ctx->adreno_use_large_buffer = false; + } else { + GGML_LOG_INFO("ggml_opencl: Adreno large buffer enabled\n"); + } + } + + if (dev_ctx->opfilter) { + // for information only, the actual regex object is created in ggml_opencl_is_device_supported + GGML_LOG_INFO("ggml_opencl: opfilter regex = \"%s\"\n", dev_ctx->opfilter_str.c_str()); + } +} + +// check if device should be accepted +static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev) { + GGML_ASSERT(dev); + GGML_ASSERT(dev->context); + + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + GGML_ASSERT(dev_ctx->platform); + GGML_ASSERT(dev_ctx->device); + + if (strstr(dev_ctx->device_name.c_str(), "Adreno") || + strstr(dev_ctx->device_name.c_str(), "Qualcomm") || + strstr(dev_ctx->device_version.c_str(), "Adreno")) { + dev_ctx->gpu_family = GPU_FAMILY::ADRENO; + + // Usually device version contains the detailed device name + dev_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_version.c_str()); + if (dev_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN) { + dev_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_name.c_str()); + } + } else if (strstr(dev_ctx->device_name.c_str(), "Intel")) { + dev_ctx->gpu_family = GPU_FAMILY::INTEL; + } else { + GGML_LOG_WARN("ggml_opencl: unsupported GPU '%s'.\n", dev_ctx->device_name.c_str()); + dev_ctx->gpu_family = GPU_FAMILY::UNKNOWN; + return false; + } + + ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); + + // Check device OpenCL version, OpenCL 2.0 or above is required + ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, dev_ctx->device); + if (opencl_c_version.major < 2) { + GGML_LOG_WARN("ggml_opencl: OpenCL 2.0 or above is required\n"); + return false; + } + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (dev_ctx->gpu_family != GPU_FAMILY::ADRENO) { + GGML_LOG_WARN("ggml_opencl: Adreno-specific kernels should not be enabled for non-Adreno GPUs; " + "run on an Adreno GPU or recompile with CMake option `-DGGML_OPENCL_USE_ADRENO_KERNELS=OFF`\n"); + return false; + } +#endif + + size_t ext_str_size; + CL_CHECK(clGetDeviceInfo(dev_ctx->device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size)); + + std::vector<char> ext_buffer(ext_str_size + 1); + CL_CHECK(clGetDeviceInfo(dev_ctx->device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer.data(), NULL)); + ext_buffer[ext_str_size] = '\0'; + + // Check if ext_buffer contains cl_khr_fp16 + bool fp16_support = strstr(ext_buffer.data(), "cl_khr_fp16") != NULL; + if (!fp16_support) { + GGML_LOG_WARN("ggml_opencl: device does not support FP16\n"); + return false; + } + + // If OpenCL 3.0 is supported, then check for cl_khr_subgroups, which becomes + // optional in OpenCL 3.0 (cl_khr_subgroup is mandatory in OpenCL 2.x) + if (opencl_c_version.major == 3 && strstr(ext_buffer.data(), "cl_khr_subgroups") == NULL && + strstr(ext_buffer.data(), "cl_intel_subgroups") == NULL) { + GGML_LOG_WARN("ggml_opencl: device does not support subgroups (cl_khr_subgroups or cl_intel_subgroups) " + "(note that subgroups is an optional feature in OpenCL 3.0)\n"); + return false; + } + + clGetDeviceInfo(dev_ctx->device, CL_DEVICE_GLOBAL_MEM_SIZE, sizeof(size_t), &dev_ctx->global_mem_size, NULL); + + const char * str_opfilter = getenv("GGML_OPENCL_OPFILTER"); + if (str_opfilter) { + dev_ctx->opfilter_str = str_opfilter; + dev_ctx->opfilter = new std::regex(str_opfilter, std::regex_constants::icase); + } + + return true; +} + +// Initialize device if it is supported (returns nullptr if it is not). +static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { + GGML_ASSERT(dev); + GGML_ASSERT(dev->context); + + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + GGML_ASSERT(dev_ctx->platform); + GGML_ASSERT(dev_ctx->device); + + if (dev_ctx->backend_ctx) { + return dev_ctx->backend_ctx; + } + + auto backend_ctx = std::make_unique<ggml_backend_opencl_context>(); + backend_ctx->device = dev_ctx->device; + backend_ctx->gpu_family = GPU_FAMILY::UNKNOWN; + + // ref_count get increased in ggml_backend_opencl_device_init + // This function is also used to retrieve backend context, so we don't want + // to increase ref_count for each call. We only want to increase ref_count + // when the associated device is initialized + backend_ctx->ref_count = 0; + + backend_ctx->gpu_family = dev_ctx->gpu_family; + backend_ctx->adreno_gen = dev_ctx->adreno_gen; + if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { + ggml_cl_init_fa_dims_table(); + + // Use wave size of 64 for all Adreno GPUs. + backend_ctx->adreno_wave_size = 64; + } + + // Populate backend device name + backend_ctx->device_name = dev_ctx->device_name; + + // A local ref of cl_device_id for convenience + cl_device_id device = backend_ctx->device; + + ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); + ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, device); + + backend_ctx->platform_version = platform_version; + backend_ctx->opencl_c_version = opencl_c_version; + + // Check driver version + size_t driver_version_str_size; + CL_CHECK(clGetDeviceInfo(device, CL_DRIVER_VERSION, 0, NULL, &driver_version_str_size)); + std::vector<char> driver_version(driver_version_str_size + 1); + CL_CHECK(clGetDeviceInfo(device, CL_DRIVER_VERSION, driver_version_str_size, driver_version.data(), NULL)); + driver_version[driver_version_str_size] = '\0'; + backend_ctx->driver_version = driver_version.data(); + + backend_ctx->adreno_cl_compiler_version = get_adreno_cl_compiler_version(driver_version.data()); + backend_ctx->has_vector_subgroup_broadcast = + (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) || + (backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17) || + (backend_ctx->adreno_cl_compiler_version.type == E17); + + // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a + // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41 + // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts + // that do not need the workarounds do not pay for them. The explicit type check is + // required: newer_than_or_same() is false for every non-E031 compiler, so negating it + // alone would enable the workarounds on E17/DX. + backend_ctx->q6_k_flat_old_compiler = + backend_ctx->adreno_cl_compiler_version.type == E031 && + !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0); + + size_t ext_str_size; + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size)); + std::vector<char> ext_buffer(ext_str_size + 1); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer.data(), NULL)); + ext_buffer[ext_str_size] = '\0'; // ensure it is null terminated + + // check support for qcom_subgroup_shuffle + if (strstr(ext_buffer.data(), "cl_qcom_subgroup_shuffle") != NULL) { + backend_ctx->has_qcom_subgroup_shuffle = true; + } + + // Check if ext_buffer contains cl_khr_fp16 + backend_ctx->fp16_support = strstr(ext_buffer.data(), "cl_khr_fp16") != NULL; + + // check Adreno large buffer support + backend_ctx->adreno_has_large_buffer = strstr(ext_buffer.data(), "cl_qcom_large_buffer") != NULL; + + // subgroup shuffle support (N_SPLIT>1 FA kernel) + backend_ctx->has_qcom_subgroup_shuffle = strstr(ext_buffer.data(), "cl_qcom_subgroup_shuffle") != NULL; + backend_ctx->has_khr_subgroup_shuffle = strstr(ext_buffer.data(), "cl_khr_subgroup_shuffle") != NULL; + backend_ctx->has_subgroup_shuffle = backend_ctx->has_khr_subgroup_shuffle || backend_ctx->has_qcom_subgroup_shuffle; + + // check for cl_khr_integer_dot_product + // cl_qcom_dot_product8 uses signed * unsigned + // while cl_khr_integer_dot_product uses signed * signed -- we stick with khr for now + backend_ctx->has_integer_dot = + strstr(ext_buffer.data(), "cl_khr_integer_dot_product") != NULL; + + cl_uint base_align_in_bits; + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL)); + GGML_ASSERT(base_align_in_bits % 8u == 0); + backend_ctx->alignment = base_align_in_bits / 8u; + + backend_ctx->global_mem_size = dev_ctx->global_mem_size; + + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(size_t), &backend_ctx->max_alloc_size, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image2d_max_width, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image2d_max_height, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_LOCAL_MEM_SIZE, sizeof(cl_ulong), &backend_ctx->local_mem_size, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &backend_ctx->svm_caps, 0)); + + if (opencl_c_version.major >= 3) { + // Assume it is not available for 3.0, since it is optional in 3.0. + // If compiling against 3.0, then we can query. + backend_ctx->non_uniform_workgroups = false; +#if CL_TARGET_OPENCL_VERSION >= 300 + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_NON_UNIFORM_WORK_GROUP_SUPPORT, sizeof(cl_bool), + &backend_ctx->non_uniform_workgroups, 0)); +#endif + } else { + GGML_ASSERT(opencl_c_version.major == 2); + // Non-uniform workgroup sizes is mandatory feature in v2.x. + backend_ctx->non_uniform_workgroups = true; + } + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno xmem F16xF32 GEMM, default on adreno, opt out with GGML_OPENCL_ADRENO_XMEM_GEMM=0. + // This helps models with f16 attention weights, e.g., gpt-oss-20b-f16 + { + const char * xmem_env = getenv("GGML_OPENCL_ADRENO_XMEM_GEMM"); + backend_ctx->adreno_xmem_gemm_enabled = backend_ctx->gpu_family == GPU_FAMILY::ADRENO && + (xmem_env ? atoi(xmem_env) != 0 : true); + } +#endif + + // determine whether to use large buffer for Adreno + backend_ctx->adreno_use_large_buffer = getenv("GGML_OPENCL_ADRENO_USE_LARGE_BUFFER") != nullptr && + backend_ctx->gpu_family == GPU_FAMILY::ADRENO; + + backend_ctx->kernel_compile_opts = ggml_opencl_make_compile_opts(backend_ctx.get()); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_COMPUTE_UNITS, + sizeof(backend_ctx->fa.compute_units), &backend_ctx->fa.compute_units, NULL)); + + // ragged moe, unspecified or non-zero means enabled, set to 0 to disable + static const char * ragged_fp16_env = getenv("GGML_OPENCL_MOE_RAGGED_FP16"); + backend_ctx->adreno_use_moe_ragged = (ragged_fp16_env == NULL) ? 1 : (atoi(ragged_fp16_env) != 0); + + // ragged moe, tile-skip granularity (columns per skip-group): 8 = quarter (default), + // 16 = half (legacy), 32 = disabled. Override with GGML_OPENCL_MOE_RAGGED_GRAN={8,16,32} + static const char * ragged_gran_env = getenv("GGML_OPENCL_MOE_RAGGED_GRAN"); + backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8; + + // whether fuse moe combine + static const char * fuse_moe_bias_glu_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_GLU"); + backend_ctx->fuse_moe_bias_glu = fuse_moe_bias_glu_env == NULL ? 1 : (atoi(fuse_moe_bias_glu_env) != 0); + + static const char * fuse_moe_bias_combine_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_COMBINE"); + backend_ctx->fuse_moe_bias_combine = fuse_moe_bias_combine_env == NULL ? 1 : (atoi(fuse_moe_bias_combine_env) != 0); + + static const char * fuse_moe_combine_env = getenv("GGML_OPENCL_FUSE_MOE_COMBINE"); + backend_ctx->fuse_moe_combine = fuse_moe_combine_env == NULL ? 1 : (atoi(fuse_moe_combine_env) != 0); + + // ragged moe dp4 variant + static const char * ragged_dp4_env = getenv("GGML_OPENCL_MOE_RAGGED"); + backend_ctx->adreno_use_moe_ragged_dp4 = ragged_dp4_env == NULL ? 1 : (atoi(ragged_dp4_env) != 0); + +#ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS + // try loading adreno binary kernels if enabled + // if fails to load, builtin kernels will be used + { + dl_handle * kernel_lib_handle = dl_load_library(KERNEL_LIB_NAME); + backend_ctx->adreno_use_bin_kernels = false; + + if (kernel_lib_handle) { + backend_ctx->get_adreno_bin_kernel_func = (get_adreno_bin_kernel_func_t)dl_get_sym(kernel_lib_handle, "get_adreno_kernels"); + if (backend_ctx->get_adreno_bin_kernel_func) { + GGML_LOG_INFO("ggml_opencl: loaded bin kernel library %s\n", KERNEL_LIB_NAME); + backend_ctx->adreno_use_bin_kernels = true; + } else { + GGML_LOG_INFO("ggml_opencl: bin kernel library %s is invalid, will use builtin kernels\n", KERNEL_LIB_NAME); + } + } else { + GGML_LOG_INFO("ggml_opencl: failed to load %s, will use builtin kernels\n", KERNEL_LIB_NAME); + } + } +#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS + + cl_int err; + + // A local ref of cl_context for convenience + cl_context context = backend_ctx->context = dev_ctx->context; + + //CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err), + // (err != CL_INVALID_QUEUE_PROPERTIES && err != CL_INVALID_VALUE ? err : + // (queue = clCreateCommandQueue(context, device, 0, &err), err) + //))); + cl_command_queue_properties command_queue_props = 0; +#ifdef GGML_OPENCL_PROFILING + command_queue_props |= CL_QUEUE_PROFILING_ENABLE; +#endif + CL_CHECK((backend_ctx->queue = clCreateCommandQueue(context, device, command_queue_props, &err), err)); + + // delay kernel loading until the first buffer is created + // load_cl_kernels(backend_ctx.get()); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Allocate intermediate buffers and images + size_t required_A_q_d_bytes = 311164928; + size_t required_A_s_d_bytes = 38895616; + size_t required_B_d_bytes = 45088768; + + // Ensure buffer sizes do not exceed the maximum allocation size + size_t max_A_q_d_bytes = MIN(required_A_q_d_bytes, backend_ctx->max_alloc_size); + size_t max_A_s_d_bytes = MIN(required_A_s_d_bytes, backend_ctx->max_alloc_size); + size_t max_B_d_bytes = MIN(required_B_d_bytes, backend_ctx->max_alloc_size); + if (required_A_q_d_bytes > backend_ctx->max_alloc_size) { + GGML_LOG_WARN("ggml_opencl: A_q_d buffer size reduced from %zu to %zu due to device limitations.\n", + required_A_q_d_bytes, max_A_q_d_bytes); + } + if (required_A_s_d_bytes > backend_ctx->max_alloc_size) { + GGML_LOG_WARN("ggml_opencl: A_s_d buffer size reduced from %zu to %zu due to device limitations.\n", + required_A_s_d_bytes, max_A_s_d_bytes); + } + if (required_B_d_bytes > backend_ctx->max_alloc_size) { + GGML_LOG_WARN("ggml_opencl: B_d buffer size reduced from %zu to %zu due to device limitations.\n", + required_B_d_bytes, max_B_d_bytes); + } + + backend_ctx->prealloc_quant_trans.allocate(context, max_A_q_d_bytes); + backend_ctx->prealloc_scales_trans.allocate(context, max_A_s_d_bytes); + backend_ctx->prealloc_act_trans.allocate(context, max_B_d_bytes); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + backend_ctx->disable_fusion = getenv("GGML_OPENCL_DISABLE_FUSION") != nullptr; + if (const char * env = getenv("GGML_OPENCL_FUSE_MM_GLU")) { + backend_ctx->fuse_mm_glu = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_FUSE_RMS_ADD")) { + backend_ctx->fuse_rms_add = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_F16_MROW")) { + backend_ctx->f16_mrow = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_F16_MROW_RPT")) { + const int v = atoi(env); + backend_ctx->f16_mrow_rpt = (v == 2 || v == 4 || v == 8 || v == 16) ? v : 1; + } + + dev_ctx->backend_ctx = backend_ctx.release(); + return dev_ctx->backend_ctx; +} + +static void ggml_cl_free(ggml_backend_t backend) { + ggml_backend_opencl_context * ctx = (ggml_backend_opencl_context *) backend->context; + ctx->free(); +} + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void transpose_2d( + ggml_backend_opencl_context * backend_ctx, + cl_kernel kernel, + cl_mem src, cl_mem dst, size_t size, + cl_int stride, cl_int rows, + bool blocking = true, + bool auto_local = false // let driver pick local size for non-uniform workgroups +) { + static ggml_cl_buffer buf; + + cl_event evt; + cl_int err; + + buf.allocate(backend_ctx->context, size); + + cl_mem trans; + cl_buffer_region region; + + region.origin = 0; + region.size = size; + CL_CHECK((trans = clCreateSubBuffer( + buf.buffer, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &stride)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &rows)); + + size_t local_size[3] = {64, 1, 1}; + size_t global_size[3] = {(size_t)stride, (size_t)rows, 1};; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, + global_size, auto_local ? NULL : local_size, 0, NULL, NULL)); + + if (blocking) { + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + } else { + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, NULL)); + } + + CL_CHECK(clReleaseMemObject(trans)); +} + +static void transpose_2d_as_8b( + ggml_backend_opencl_context * backend_ctx, + cl_mem src, cl_mem dst, size_t size, + cl_int stride, cl_int rows, + bool blocking = true, + bool auto_local = false +) { + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_8_buf, + src, dst, size, stride, rows, blocking, auto_local); +} + +static void transpose_2d_as_16b( + ggml_backend_opencl_context * backend_ctx, + cl_mem src, cl_mem dst, size_t size, + cl_int stride, cl_int rows, + bool blocking = true, + bool auto_local = false +) { + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, + src, dst, size, stride, rows, blocking, auto_local); +} + +static void transpose_2d_as_32b( + ggml_backend_opencl_context * backend_ctx, + cl_mem src, cl_mem dst, size_t size, + cl_int stride, cl_int rows, + bool blocking = true, + bool auto_local = false +) { + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, + src, dst, size, stride, rows, blocking, auto_local); +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +//------------------------------------------------------------------------------ +// Tensor extra management +//------------------------------------------------------------------------------ +struct ggml_tensor_extra_cl { + // The buffer object that holds the data. + cl_mem data_device; + // The offset into the buffer object. This is primarily for scratch buffer + // and view operation. + // NB: this offset no longer includes view offset (view_offs). Whenever this + // offset is used, view_offs should be considered. + cl_ulong offset; + // The actual size of the cl_mem object. This is needed when returning the + // block to the pool. + size_t actual_size; + + void reset() { + data_device = nullptr; + offset = 0; + actual_size = 0; + } +}; + +struct ggml_tensor_extra_cl_q1_0 { + cl_mem q = nullptr; + cl_mem q_img = nullptr; + + cl_mem d = nullptr; + cl_mem d_img = nullptr; + + size_t size_q = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q1_0() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + q_img = nullptr; + d_img = nullptr; + size_q = 0; + size_d = 0; + } +}; + +// Additional tensor extra structs for quantized tensors. +// These tensors are loaded from files and should not be allocated in scratch -- +// they should always be allocated from the pool. Hence, they do not have an +// `offset`, which indicate their locations in the scratch buffer. +struct ggml_tensor_extra_cl_q4_0 { + // Quantized values. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Size of quantized values. + size_t size_q = 0; + // Size of scales. + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q4_0() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + if (d_img != nullptr) { + CL_CHECK(clReleaseMemObject(d_img)); + d_img = nullptr; + } + size_q = 0; + size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_q4_1 { + // Quantized values. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Min + cl_mem m = nullptr; + // Min in image1d_buffer_t. + cl_mem m_img = nullptr; + // Size of quantized values. + size_t size_q = 0; + // Size of scales. + size_t size_d = 0; + // Size of min values. + size_t size_m = 0; + + ~ggml_tensor_extra_cl_q4_1() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (m != nullptr) { + CL_CHECK(clReleaseMemObject(m)); + m = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + // Currently, q_img and d_img are only initialized when SMALL_ALLOC is + // enabled. They point to the images in ggml_backend_opencl_buffer_context. + // So, there is no need to release them here. + // TODO: initialize them for non SMALL_PATH path, or remove them. + d_img = nullptr; + m_img = nullptr; + size_q = 0; + size_d = 0; + size_m = 0; + } +}; + +struct ggml_tensor_extra_cl_q5_0 { + // Quantized values. + cl_mem qs = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem qs_img = nullptr; + // 5-th bit values. + cl_mem qh = nullptr; + // 5-th bit values in image1d_buffer_t. + cl_mem qh_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Uniform per-32-block scale (2/block) + min (1/block, = d*16 for the -16 centering) + // for the generic dp4a MoE GEMM. Built from d. + cl_mem scale = nullptr; + cl_mem min = nullptr; + // Size of quantized values. + size_t size_qs = 0; + // Size of 5-th bit values. + size_t size_qh = 0; + // Size of scales. + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q5_0() { + reset(); + } + + void reset() { + if (qs != nullptr) { + CL_CHECK(clReleaseMemObject(qs)); + qs = nullptr; + } + if (qh != nullptr) { + CL_CHECK(clReleaseMemObject(qh)); + qh = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (qs_img != nullptr) { + CL_CHECK(clReleaseMemObject(qs_img)); + qs_img = nullptr; + } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } + if (min != nullptr) { + CL_CHECK(clReleaseMemObject(min)); + min = nullptr; + } + + qh_img = nullptr; + d_img = nullptr; + size_qs = 0; + size_qh = 0; + size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_q5_1 { + // Quantized values. + cl_mem qs = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem qs_img = nullptr; + // 5-th bit values. + cl_mem qh = nullptr; + // 5-th bit values in image1d_buffer_t. + cl_mem qh_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Min + cl_mem m = nullptr; + // Min in image1d_buffer_t. + cl_mem m_img = nullptr; + // Size of quantized values. + size_t size_qs = 0; + // Size of 5-th bit values. + size_t size_qh = 0; + // Size of scales. + size_t size_d = 0; + // Size of min values. + size_t size_m = 0; + + ~ggml_tensor_extra_cl_q5_1() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (qs != nullptr) { + CL_CHECK(clReleaseMemObject(qs)); + qs = nullptr; + } + if (qh != nullptr) { + CL_CHECK(clReleaseMemObject(qh)); + qh = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (m != nullptr) { + CL_CHECK(clReleaseMemObject(m)); + m = nullptr; + } + if (qs_img != nullptr) { + CL_CHECK(clReleaseMemObject(qs_img)); + qs_img = nullptr; + } + // qh_img, d_img, and m_img are not currently allocated separately. + // TODO: initialize them for non SMALL_PATH path, or remove them. + qh_img = nullptr; + d_img = nullptr; + m_img = nullptr; + size_qs = 0; + size_qh = 0; + size_d = 0; + size_m = 0; + } +}; + +struct ggml_tensor_extra_cl_mxfp4 { + // Quantized values. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales in E8M0. + cl_mem e = nullptr; + // Scales in image1d_buffer_t. + cl_mem e_img = nullptr; + // Size of quantized values. + size_t size_q = 0; + // Size of scales. + size_t size_e = 0; + + ~ggml_tensor_extra_cl_mxfp4() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (e != nullptr) { + CL_CHECK(clReleaseMemObject(e)); + e = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + // Currently, e_img is not used. They can be image1d_buffer_t + // that wraps around q and d to utilize image access path. + e_img = nullptr; + size_q = 0; + size_e = 0; + } +}; + +struct ggml_tensor_extra_cl_q8_0 { + cl_mem q = nullptr; + cl_mem q_img = nullptr; + + cl_mem d = nullptr; + cl_mem d_img = nullptr; + + // Uniform per-16-segment scale (16/superblock) for the generic dp4a MoE GEMM. + // Expanded from d at set_tensor; the int8 codes are reused from q. + // q8_0 is symmetric so no min buffer (has_min=0). + cl_mem scale = nullptr; + + size_t size_q = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q8_0() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } + // Currently, q_img and d_img are not used. They can be image1d_buffer_t + // that wraps around q and d to utilize image access path. + q_img = nullptr; + d_img = nullptr; + size_q = 0; + size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq4_nl { + cl_mem q = nullptr; + cl_mem q_img = nullptr; + + cl_mem d = nullptr; + cl_mem d_img = nullptr; + + size_t size_q = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_iq4_nl() { + reset(); + } + + void reset() { + if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + q_img = nullptr; + d_img = nullptr; + size_q = 0; + size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_q4_K { + // Quantized values + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales for each super block. + cl_mem s = nullptr; + // Scales + cl_mem d = nullptr; + // Min + cl_mem dm = nullptr; + + ~ggml_tensor_extra_cl_q4_K() { + reset(); + } + + void reset() { + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (s != nullptr) { + CL_CHECK(clReleaseMemObject(s)); + s = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (dm != nullptr) { + CL_CHECK(clReleaseMemObject(dm)); + dm = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + } +}; + +struct ggml_tensor_extra_cl_q5_K { + // Lower 4 bits of quantized weights. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Upper 1 bit of quantized weights. + cl_mem qh = nullptr; + // Scales for each block. + cl_mem s = nullptr; + // Scales for each super block. + cl_mem d = nullptr; + // Min for each super block. + cl_mem dm = nullptr; + // Uniform per-32-block scale (2/block) + min (1/block, = dm*mn) decoded from the + // 6-bit packed s[] for the generic dp4a MoE GEMM kernel_gemm_moe_q8_1_dp4a. + // Built from s/d/dm at set_tensor; q/qh are reused as-is. + cl_mem scale = nullptr; + cl_mem min = nullptr; + + size_t size_q = 0; + size_t size_qh = 0; + size_t size_s = 0; + size_t size_d = 0; + size_t size_dm = 0; + + ~ggml_tensor_extra_cl_q5_K() { + reset(); + } + + void reset() { + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (qh != nullptr) { + CL_CHECK(clReleaseMemObject(qh)); + qh = nullptr; + } + if (s != nullptr) { + CL_CHECK(clReleaseMemObject(s)); + s = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (dm != nullptr) { + CL_CHECK(clReleaseMemObject(dm)); + dm = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } + if (min != nullptr) { + CL_CHECK(clReleaseMemObject(min)); + min = nullptr; + } + + size_q = 0; + size_qh = 0; + size_s = 0; + size_d = 0; + size_dm = 0; + } +}; + +struct ggml_tensor_extra_cl_q2_K { + bool noshuffle = false; + cl_mem qs = nullptr; // quantized values + cl_mem sc = nullptr; // one uchar per 16 weights: scale nibble + min nibble + cl_mem dm = nullptr; // half2 per super-block: d, dmin + + size_t size_qs = 0, size_sc = 0, size_dm = 0; + + ~ggml_tensor_extra_cl_q2_K() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (dm != nullptr) { CL_CHECK(clReleaseMemObject(dm)); dm = nullptr; } + size_qs = size_sc = size_dm = 0; + } +}; + +struct ggml_tensor_extra_cl_q3_K { + bool noshuffle = false; + cl_mem qs = nullptr; // quantized values + cl_mem hm = nullptr; // high bits + cl_mem sc = nullptr; // packed 6-bit scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_hm = 0, size_sc = 0, size_d = 0; + + ~ggml_tensor_extra_cl_q3_K() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (hm != nullptr) { CL_CHECK(clReleaseMemObject(hm)); hm = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_hm = size_sc = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq3_xxs { + bool noshuffle = false; + cl_mem qs = nullptr; // grid indices + cl_mem sas = nullptr; // packed sign codes and scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_sas = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq3_xxs() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sas != nullptr) { CL_CHECK(clReleaseMemObject(sas)); sas = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sas = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq3_s { + bool noshuffle = false; + cl_mem qs = nullptr; // grid index low bits + cl_mem sg = nullptr; // sign bits + cl_mem qh = nullptr; // grid index high bits + cl_mem sc = nullptr; // packed scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_sg = 0, size_qh = 0, size_sc = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq3_s() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sg != nullptr) { CL_CHECK(clReleaseMemObject(sg)); sg = nullptr; } + if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sg = size_qh = size_sc = size_d = 0; + } +}; + + +struct ggml_tensor_extra_cl_iq2_xxs { + bool noshuffle = false; + cl_mem qs = nullptr; // grid indices + cl_mem sas = nullptr; // packed sign codes and scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_sas = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq2_xxs() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sas != nullptr) { CL_CHECK(clReleaseMemObject(sas)); sas = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sas = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq2_xs { + bool noshuffle = false; + cl_mem qs = nullptr; // packed grid indices and sign codes + cl_mem sc = nullptr; // packed scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_sc = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq2_xs() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sc = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq2_s { + bool noshuffle = false; + cl_mem qs = nullptr; // grid index low bits + cl_mem sg = nullptr; // sign bits + cl_mem qh = nullptr; // grid index high bits + cl_mem sc = nullptr; // packed scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_sg = 0, size_qh = 0, size_sc = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq2_s() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sg != nullptr) { CL_CHECK(clReleaseMemObject(sg)); sg = nullptr; } + if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sg = size_qh = size_sc = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_q2_0 { + bool noshuffle = false; + cl_mem qs = nullptr; + cl_mem d = nullptr; + + size_t size_qs = 0, size_d = 0; + + ~ggml_tensor_extra_cl_q2_0() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq1_s { + bool noshuffle = false; + cl_mem qs = nullptr; // grid indices + cl_mem qh = nullptr; // high grid bits, scales, and shifts + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_qh = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq1_s() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_qh = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq1_m { + bool noshuffle = false; + cl_mem qs = nullptr; // grid indices + cl_mem qh = nullptr; // high grid bits and shifts + cl_mem sc = nullptr; // packed block and super-block scales + + size_t size_qs = 0, size_qh = 0, size_sc = 0; + + ~ggml_tensor_extra_cl_iq1_m() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + size_qs = size_qh = size_sc = 0; + } +}; + +struct ggml_tensor_extra_cl_iq4_xs { + bool noshuffle = false; + cl_mem qs = nullptr; + cl_mem sl = nullptr; + cl_mem sh = nullptr; + cl_mem d = nullptr; + + size_t size_qs = 0, size_sl = 0, size_sh = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq4_xs() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sl != nullptr) { CL_CHECK(clReleaseMemObject(sl)); sl = nullptr; } + if (sh != nullptr) { CL_CHECK(clReleaseMemObject(sh)); sh = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sl = size_sh = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_q6_K { + // Lower 4 bits of quantized weights. + cl_mem ql = nullptr; + // Lower 4 bits as image1d_buffer_t + cl_mem ql_img = nullptr; + // Upper 2 bits of quantized weights. + cl_mem qh = nullptr; + // Upper 2 bits as image1d_buffer_t + cl_mem qh_img = nullptr; + // Scales for each block. + cl_mem s = nullptr; + // Scales for each super block. + cl_mem d = nullptr; + + size_t size_ql = 0; + size_t size_qh = 0; + size_t size_s = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q6_K() { + reset(); + } + + void reset() { + if (ql != nullptr) { + CL_CHECK(clReleaseMemObject(ql)); + ql = nullptr; + } + if (qh != nullptr) { + CL_CHECK(clReleaseMemObject(qh)); + qh = nullptr; + } + if (s != nullptr) { + CL_CHECK(clReleaseMemObject(s)); + s = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (ql_img != nullptr) { + CL_CHECK(clReleaseMemObject(ql_img)); + ql_img = nullptr; + } + if (qh_img != nullptr) { + CL_CHECK(clReleaseMemObject(qh_img)); + qh_img = nullptr; + } + + size_ql = 0; + size_qh = 0; + size_s = 0; + size_d = 0; + } +}; + +//------------------------------------------------------------------------------ +// Backend API +//------------------------------------------------------------------------------ + +// +// backend +// +static const char * ggml_backend_opencl_name(ggml_backend_t backend) { + return "OpenCL"; + + UNUSED(backend); +} + +static void ggml_backend_opencl_free(ggml_backend_t backend) { + ggml_cl_free(backend); +} + +static void ggml_backend_opencl_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + GGML_UNUSED(backend); + GGML_UNUSED(tensor); + GGML_UNUSED(data); + GGML_UNUSED(offset); + GGML_UNUSED(size); +} + +static void ggml_backend_opencl_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + GGML_UNUSED(backend); + GGML_UNUSED(tensor); + GGML_UNUSED(data); + GGML_UNUSED(offset); + GGML_UNUSED(size); +} + +static bool ggml_backend_opencl_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { + GGML_UNUSED(backend); + GGML_UNUSED(src); + GGML_UNUSED(dst); + return false; +} + +static void ggml_backend_opencl_synchronize(ggml_backend_t backend) { + auto * backend_ctx = static_cast<ggml_backend_opencl_context *>(backend->context); + + cl_event evt; + CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); +} + +// Synchronizes the 'backend_ctx's device with others so that commands +// enqueued to it won't start until commands in the other devices have +// completed. +static void sync_with_other_backends(ggml_backend_opencl_context * backend_ctx) { + if (g_ggml_backend_opencl_devices.size() < 2) { + return; // No other devices to synchronize with. + } + + std::vector<cl_event> events; + events.reserve(g_ggml_backend_opencl_devices.size()); + + for (ggml_backend_device & backend_dev : g_ggml_backend_opencl_devices) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) backend_dev.context; + auto * other_backend_ctx = dev_ctx->backend_ctx; + + if (backend_ctx != other_backend_ctx) { + cl_event ev; + CL_CHECK(clEnqueueMarkerWithWaitList(other_backend_ctx->queue, 0, nullptr, &ev)); + CL_CHECK(clFlush(other_backend_ctx->queue)); + events.push_back(ev); + } + } + + CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, events.size(), events.data(), nullptr)); + for (auto ev : events) { + CL_CHECK(clReleaseEvent(ev)); + } +} + +static void sync_with_other_backends(ggml_backend_t backend) { + auto * backend_ctx = static_cast<ggml_backend_opencl_context *>(backend->context); + sync_with_other_backends(backend_ctx); +} + +// True if two tensors share a device buffer with overlapping byte ranges. The pool +// allocator may place a fused op's output over a sequentially-dead input (safe for the +// original separate kernels, but a read/write race inside one fused kernel). +static bool ggml_cl_tensors_overlap(const ggml_tensor * x, const ggml_tensor * y) { + ggml_tensor_extra_cl * ex = (ggml_tensor_extra_cl *)x->extra; + ggml_tensor_extra_cl * ey = (ggml_tensor_extra_cl *)y->extra; + if (!ex || !ey || ex->data_device != ey->data_device) { return false; } + const cl_ulong xo = ex->offset + x->view_offs, xe = xo + ggml_nbytes(x); + const cl_ulong yo = ey->offset + y->view_offs, ye = yo + ggml_nbytes(y); + return xo < ye && yo < xe; +} + +// Detect the MoE combine epilogue: router-weight MUL ([n_embd,k,nt] * [1,k,nt]) followed +// by k VIEWs of it and a (k-1)-long ADD reduction chain producing [n_embd, nt]. When it +// matches (and the output does not alias the inputs), the whole subgraph collapses to one +// weighted-sum-across-experts kernel. +static bool ggml_opencl_can_fuse_moe_combine(const struct ggml_cgraph * cgraph, int node_idx, + const ggml_tensor ** out_final_add) { + const ggml_tensor * mul = cgraph->nodes[node_idx]; + if (mul->op != GGML_OP_MUL) { return false; } + const ggml_tensor * experts = mul->src[0]; + const ggml_tensor * weights = mul->src[1]; + if (!experts || !weights) { return false; } + if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { return false; } + + const int64_t n_embd = experts->ne[0]; + const int64_t k = experts->ne[1]; + const int64_t nt = experts->ne[2]; + if (k < 2 || k > 64 || experts->ne[3] != 1 || n_embd % 4 != 0) { return false; } + if (weights->ne[0] != 1 || weights->ne[1] != k || weights->ne[2] != nt || weights->ne[3] != 1) { return false; } + if (mul->ne[0] != n_embd || mul->ne[1] != k || mul->ne[2] != nt) { return false; } + // the fused kernel needs contiguous experts/weights and a contiguous 2D dst + if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights)) { return false; } + + const int n_nodes = 1 + (int)k + (int)(k - 1); // MUL + k*VIEW + (k-1)*ADD + if (n_nodes >= 32) { return false; } + if (node_idx + n_nodes > cgraph->n_nodes) { return false; } + + enum ggml_op ops[1 + 64 + 63]; + int n = 0; + ops[n++] = GGML_OP_MUL; + for (int j = 0; j < (int)k; ++j) { ops[n++] = GGML_OP_VIEW; } + for (int j = 0; j < (int)k - 1; ++j) { ops[n++] = GGML_OP_ADD; } + const int outs[] = { node_idx + n_nodes - 1 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { return false; } + + for (int j = 0; j < (int)k; ++j) { + const ggml_tensor * vw = cgraph->nodes[node_idx + 1 + j]; + if (vw->op != GGML_OP_VIEW || vw->src[0] != mul || vw->ne[0] != n_embd || vw->ne[1] != nt) { return false; } + } + const ggml_tensor * final_add = cgraph->nodes[node_idx + n_nodes - 1]; + if (final_add->op != GGML_OP_ADD || final_add->type != GGML_TYPE_F32 || + final_add->ne[0] != n_embd || final_add->ne[1] != nt || final_add->ne[2] != 1) { return false; } + if (!ggml_is_contiguous(final_add)) { return false; } + // the fused kernel reads experts + writes final_add in one pass; bail if the + // pool allocator overlapped the output with the (large) experts input -- would race. + // The small weights input is copied to a private scratch in the dispatch, so its own + // aliasing with the output is handled there and does not block the fusion. + if (ggml_cl_tensors_overlap(experts, final_add)) { return false; } + + *out_final_add = final_add; + return true; +} + +// Detect the gpt-oss MoE bias+activation epilogue on the PREFILL path: +// {MUL_MAT_ID(gate), ADD_ID(gate_bias), MUL_MAT_ID(up), ADD_ID(up_bias), GLU(swiglu_oai)}. +// The two matmuls still run as their own dispatches (the prefill GEMM is the vendor's); +// what collapses is the epilogue — both add_id passes are in-place read-modify-writes of a +// tensor the GLU immediately reads again, so they are three full passes over the same +// [n_ff, n_expert_used, n_tokens] f32 tensor where one suffices. +// +// The decode counterpart is handled by the mxfp4 fused GEMV arm in ggml_opencl_can_fuse, +// which folds the matmul too; this one deliberately fires only when that cannot (ne[2] > 1). +static bool ggml_opencl_can_fuse_moe_bias_glu(const struct ggml_cgraph * cgraph, int node_idx) { + if (node_idx + 4 >= cgraph->n_nodes) { + return false; + } + + const enum ggml_op mg_ops[] = { GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_GLU }; + const int mg_out[] = { node_idx + 4 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, 5, mg_ops, mg_out, 1)) { + return false; + } + + const ggml_tensor * gmm = cgraph->nodes[node_idx]; + const ggml_tensor * gad = cgraph->nodes[node_idx+1]; + const ggml_tensor * umm = cgraph->nodes[node_idx+2]; + const ggml_tensor * uad = cgraph->nodes[node_idx+3]; + const ggml_tensor * glu = cgraph->nodes[node_idx+4]; + + if (ggml_get_glu_op(glu) != GGML_GLU_OP_SWIGLU_OAI) { + return false; + } + // Prefill only — at one token the mxfp4 arm above folds the matmul as well. + if (gmm->src[1]->ne[2] == 1) { + return false; + } + // Wiring: both matmuls share the activation and the expert selection, each add_id + // biases its own matmul, and the GLU consumes the two biased results as separate + // operands (so the same-buffer ne00_off/ne10_off split path is not in play). + if (gad->src[0] != gmm || uad->src[0] != umm || + glu->src[0] != gad || glu->src[1] != uad || + umm->src[1] != gmm->src[1] || umm->src[2] != gmm->src[2]) { + return false; + } + // A swapped GLU would exchange the gate/up roles the fused kernel hard-codes. + if (ggml_get_op_params_i32(glu, 1)) { + return false; + } + if (gad->type != GGML_TYPE_F32 || uad->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { + return false; + } + if (!gad->src[1] || gad->src[1]->type != GGML_TYPE_F32 || + !uad->src[1] || uad->src[1]->type != GGML_TYPE_F32) { + return false; + } + if (!gad->src[2] || gad->src[2]->type != GGML_TYPE_I32 || uad->src[2] != gad->src[2]) { + return false; + } + // Full width on both operands: the kernel writes one output element per input pair. + if (!ggml_are_same_shape(gad, uad) || glu->ne[0] != gad->ne[0] || + glu->ne[1] != gad->ne[1] || glu->ne[2] != gad->ne[2] || glu->ne[3] != gad->ne[3]) { + return false; + } + if (gad->ne[3] != 1) { + return false; + } + // The destination is addressed by (expert slot, token) rather than the GLU's flat row + // walk; those agree only for a contiguous destination. + if (!ggml_is_contiguous(glu) || !ggml_is_contiguous(gmm) || !ggml_is_contiguous(umm)) { + return false; + } + return true; +} + +static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + +// Runs the gate and up matmuls unchanged, then one kernel in place of +// add_id(gate) + add_id(up) + swiglu_oai. See ggml_opencl_can_fuse_moe_bias_glu. +static void ggml_cl_moe_bias_glu_fused(ggml_backend_t backend, ggml_tensor * gate_mm, const ggml_tensor * gate_add, + ggml_tensor * up_mm, const ggml_tensor * up_add, const ggml_tensor * glu) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_cl_mul_mat_id(backend, gate_mm->src[0], gate_mm->src[1], gate_mm); + ggml_cl_mul_mat_id(backend, up_mm->src[0], up_mm->src[1], up_mm); + + const ggml_tensor * gbias = gate_add->src[1]; + const ggml_tensor * ubias = up_add->src[1]; + const ggml_tensor * ids = gate_add->src[2]; + + ggml_tensor_extra_cl * eg = (ggml_tensor_extra_cl *)gate_mm->extra; + ggml_tensor_extra_cl * egb = (ggml_tensor_extra_cl *)gbias->extra; + ggml_tensor_extra_cl * eu = (ggml_tensor_extra_cl *)up_mm->extra; + ggml_tensor_extra_cl * eub = (ggml_tensor_extra_cl *)ubias->extra; + ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra; + ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)glu->extra; + + cl_ulong off_g = eg->offset + gate_mm->view_offs; + cl_ulong off_gb = egb->offset + gbias->view_offs; + cl_ulong off_u = eu->offset + up_mm->view_offs; + cl_ulong off_ub = eub->offset + ubias->view_offs; + cl_ulong off_i = ei->offset + ids->view_offs; + cl_ulong off_d = ed->offset + glu->view_offs; + + const cl_ulong nb01_g = gate_mm->nb[1]; + const cl_ulong nb02_g = gate_mm->nb[2]; + const cl_ulong nb01_u = up_mm->nb[1]; + const cl_ulong nb02_u = up_mm->nb[2]; + const cl_ulong nb11_g = gbias->nb[1]; + const cl_ulong nb11_u = ubias->nb[1]; + const cl_ulong nb21 = ids->nb[1]; + const cl_ulong nbd1 = glu->nb[1]; + const cl_ulong nbd2 = glu->nb[2]; + + const int ne0 = (int)glu->ne[0]; + const float alpha = ggml_get_op_params_f32(glu, 2); + const float limit = ggml_get_op_params_f32(glu, 3); + + cl_kernel kernel = backend_ctx->kernel_add_id_add_id_swiglu_oai; + + int i = 0; + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eg->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_g)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &egb->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_gb)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eu->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_u)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eub->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_ub)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &ei->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_i)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &ed->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_d)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_g)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_g)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_u)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_u)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_g)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_u)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd1)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd2)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float), &limit)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float), &alpha)); + + const int nth = MIN(ne0, (int) backend_ctx->get_kernel_workgroup_size(kernel)); + size_t global_work_size[] = { (size_t)glu->ne[1]*nth, (size_t)glu->ne[2], 1 }; + size_t local_work_size[] = { (size_t)nth, 1, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, glu); +} + +// Fusion B: the MoE down-projection bias add feeding the combine. +// +// The graph runs ADD_ID(down_bias) and then immediately the combine subgraph +// {MUL(router weights), k VIEWs, k-1 ADDs}, and the ADD_ID's only consumer is that +// MUL. Since the ADD_ID is an in-place read-modify-write of a tensor the combine +// reads once more, the bias can be added inside the combine instead, dropping a +// full pass over [n_embd, k, n_tokens]. +// +// Shape checks for the combine tail are delegated to ggml_opencl_can_fuse_moe_combine +// (which also owns the n_nodes >= 32 bail and the experts/dst aliasing bail); what is +// added here is the ADD_ID wiring plus a subgraph check over the WHOLE run, so that +// the intermediate bias result is confirmed not to escape. +static bool ggml_opencl_can_fuse_moe_bias_combine(const struct ggml_cgraph * cgraph, int node_idx, + const ggml_tensor ** out_final_add) { + if (node_idx + 1 >= cgraph->n_nodes) { + return false; + } + const ggml_tensor * add = cgraph->nodes[node_idx]; + if (add->op != GGML_OP_ADD_ID) { + return false; + } + const ggml_tensor * mul = cgraph->nodes[node_idx+1]; + if (mul->op != GGML_OP_MUL || mul->src[0] != add) { + return false; + } + + const ggml_tensor * final_add = NULL; + if (!ggml_opencl_can_fuse_moe_combine(cgraph, node_idx+1, &final_add)) { + return false; + } + + const ggml_tensor * raw = add->src[0]; + const ggml_tensor * bias = add->src[1]; + const ggml_tensor * ids = add->src[2]; + if (!raw || !bias || !ids) { + return false; + } + if (raw->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 || + ids->type != GGML_TYPE_I32 || add->type != GGML_TYPE_F32) { + return false; + } + // The combine reads the raw matmul output with the strides it computed from the + // add_id result, so the two must have the same layout. + if (!ggml_are_same_shape(raw, add) || !ggml_is_contiguous(raw)) { + return false; + } + if (raw->nb[1] != add->nb[1] || raw->nb[2] != add->nb[2]) { + return false; + } + // ids is indexed as [expert slot, token]; the combine walks the same two axes. + if (ids->ne[0] < add->ne[1] || ids->ne[1] < add->ne[2]) { + return false; + } + + // Whole-run escape check: ADD_ID + MUL + k VIEWs + (k-1) ADDs, only the last node escapes. + const int k = (int)add->ne[1]; + const int n_nodes = 2 + k + (k - 1); + if (n_nodes >= 32 || node_idx + n_nodes > cgraph->n_nodes) { + return false; + } + enum ggml_op ops[32]; + int n = 0; + ops[n++] = GGML_OP_ADD_ID; + ops[n++] = GGML_OP_MUL; + for (int j = 0; j < k; ++j) ops[n++] = GGML_OP_VIEW; + for (int j = 0; j < k - 1; ++j) ops[n++] = GGML_OP_ADD; + const int outs[] = { node_idx + n_nodes - 1 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { + return false; + } + + *out_final_add = final_add; + return true; +} + + +// Fusion B dispatch: the combine, reading the RAW matmul output and adding the +// per-expert bias row inline. See ggml_opencl_can_fuse_moe_bias_combine. +static void ggml_cl_moe_bias_combine_fused(ggml_backend_t backend, const ggml_tensor * add, + const ggml_tensor * mul, const ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + + const ggml_tensor * experts = add->src[0]; // raw matmul output, bias not yet applied + const ggml_tensor * bias = add->src[1]; + const ggml_tensor * ids = add->src[2]; + const ggml_tensor * weights = mul->src[1]; + + ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; + ggml_tensor_extra_cl * eb = (ggml_tensor_extra_cl *)bias->extra; + ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra; + ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; + ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong off_e = ee->offset + experts->view_offs; + cl_ulong off_b = eb->offset + bias->view_offs; + cl_ulong off_i = ei->offset + ids->view_offs; + cl_ulong off_w = ew->offset + weights->view_offs; + cl_ulong off_d = ed->offset + dst->view_offs; + + const int n_embd4 = (int)(experts->ne[0] / 4); + const int k = (int)experts->ne[1]; + const int nt = (int)experts->ne[2]; + const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); + const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); + const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); + const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); + const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); + const cl_ulong nb_b1 = bias->nb[1]; + const cl_ulong nb_i1 = ids->nb[1]; + + const size_t w_bytes = ggml_nbytes(weights); + backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, + off_w, 0, w_bytes, 0, NULL, NULL)); + cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; + cl_ulong w_off = 0; + + cl_kernel kernel = backend_ctx->kernel_moe_combine_bias_f32; + int a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &eb->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_b)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ei->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_b1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_i1)); + + size_t lws[2] = { 64, 1 }; + size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); +} + + +static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor * mul, const ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + const ggml_tensor * experts = mul->src[0]; + const ggml_tensor * weights = mul->src[1]; + + ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; + ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; + ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong off_e = ee->offset + experts->view_offs; + cl_ulong off_w = ew->offset + weights->view_offs; + cl_ulong off_d = ed->offset + dst->view_offs; + + const int n_embd4 = (int)(experts->ne[0] / 4); + const int k = (int)experts->ne[1]; + const int nt = (int)experts->ne[2]; + const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); + const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); + const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); + const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); + const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); + + // The router weights are tiny ([1,k,nt]) and may share a pool buffer with the output; + // copy them into a private scratch so the fused kernel never reads aliased memory. + const size_t w_bytes = ggml_nbytes(weights); + backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, + off_w, 0, w_bytes, 0, NULL, NULL)); + cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; + cl_ulong w_off = 0; + + cl_kernel kernel = backend_ctx->kernel_moe_combine_f32; + int a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); + + size_t lws[2] = { 64, 1 }; + size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); +} + +inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check) +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); +inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below + +static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) { + + // glu(mul_mat(Wg,x), mul_mat(Wu,x)) — the FFN gate/up GEMVs + GLU. This is a + // non-linear subgraph (up does NOT consume gate), so the contiguous + // ggml_can_fuse below rejects it; use ggml_can_fuse_subgraph with the glu as + // the sole output and validate the edges explicitly. q4_K decode only; + // byte-identical to the per-op path. + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_MUL_MAT && + ops.begin()[1] == GGML_OP_MUL_MAT && ops.begin()[2] == GGML_OP_GLU) { + const enum ggml_op glu_ops[] = { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }; + const int glu_out[] = { node_idx + 2 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, 3, glu_ops, glu_out, 1)) { + return false; + } + + const ggml_tensor *gate = cgraph->nodes[node_idx]; + const ggml_tensor *up = cgraph->nodes[node_idx+1]; + const ggml_tensor *glu = cgraph->nodes[node_idx+2]; + + // decode GEMV path only (single token); prefill GEMM is separate + if (gate->ne[1] != 1 || up->ne[1] != 1) { + return false; + } + // both projections must be q4_K weights, f32 activation/output + if (gate->src[0]->type != GGML_TYPE_Q4_K || up->src[0]->type != GGML_TYPE_Q4_K || + gate->src[1]->type != GGML_TYPE_F32 || up->src[1]->type != GGML_TYPE_F32 || + gate->type != GGML_TYPE_F32 || up->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { + return false; + } + // gate and up must share the same activation and have matching shape/stride + if (gate->src[1] != up->src[1] || + !ggml_are_same_shape(gate->src[0], up->src[0]) || + !ggml_are_same_stride(gate->src[0], up->src[0])) { + return false; + } + // GLU must read gate as src[0] and up as src[1], no swap (the fused + // epilogue applies the activation to gate, multiplies by up) + if (glu->src[0] != gate || glu->src[1] != up) { + return false; + } + if (ggml_get_op_params_i32(glu, 1) /* swapped */) { + return false; + } + // SWIGLU_OAI carries extra alpha/limit params -> not handled by the fused kernel + if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI) { + return false; + } + // the fused kernel reads the standard noshuffle image layout; the tiled + // layout packs weights differently -> defer those to the per-op path + if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) { + return false; + } + // q4_K bin kernel requires 32b transposed layout, not compatible with the fused gemv + if (use_q4_k_bin_kernels(backend_ctx, gate->src[0]) || use_q4_k_bin_kernels(backend_ctx, up->src[0])) { + return false; + } + // that noshuffle layout is only produced at set_tensor time when + // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512). + // Smaller weights stay in the plain q4_K layout, which this kernel would + // misread -> defer them to the per-op path. Real FFN gate/up weights are + // far above the threshold, so production dispatch is unchanged. + if (!use_adreno_kernels(backend_ctx, gate->src[0]) || + !use_adreno_kernels(backend_ctx, up->src[0])) { + return false; + } + return true; + } + + if (!ggml_can_fuse(cgraph, node_idx, ops)) { + return false; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { + const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + + // rms_norm only supports f32 + if (mul->src[0]->type != GGML_TYPE_F32 || + mul->src[1]->type != GGML_TYPE_F32 || + mul->type != GGML_TYPE_F32) { + return false; + } + + // if rms_norm is the B operand, then we don't handle broadcast + if (rms_norm == mul->src[1] && + !ggml_are_same_shape(mul->src[0], rms_norm)) { + return false; + } + + // rms_norm assumes contiguous rows + if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { + return false; + } + } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { + const ggml_tensor *norm = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + const ggml_tensor *add = cgraph->nodes[node_idx+2]; + const ggml_tensor *w = mul->src[0] == norm ? mul->src[1] : mul->src[0]; + const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + + // norm fusion only supports F32 + if (norm->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { + return false; + } + + if (norm->src[0]->ne[0] % 4 != 0) { + return false; + } + + if (!ggml_is_contiguous(norm->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { + return false; + } + } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { + // rms_norm(x) * w + b, fused (residual). Mirrors the RMS_NORM+MUL gate + // plus the residual-add operand's constraints. + const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + const ggml_tensor *add = cgraph->nodes[node_idx+2]; + const ggml_tensor *w = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0]; + const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + + if (w->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 || + b->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32) { + return false; + } + if (rms_norm->src[0]->ne[0] % 4 != 0) { + return false; + } + // if rms_norm is the B operand of mul, broadcast is not handled + if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { + return false; + } + // the residual must match the normed output shape (no add broadcast) + if (!ggml_are_same_shape(b, add)) { + return false; + } + // rms_norm assumes contiguous rows + if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1]) || + !ggml_is_contiguous_rows(b)) { + return false; + } + } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_GROUP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { + const ggml_tensor *gn = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + const ggml_tensor *add = cgraph->nodes[node_idx+2]; + const ggml_tensor *w = mul->src[0] == gn ? mul->src[1] : mul->src[0]; + const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + + if (gn->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { + return false; + } + + if (!ggml_is_contiguous(gn->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { + return false; + } + } + + return true; +} + +static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor); +static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); +static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); + +static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * gate_tensor, ggml_tensor * up_tensor, ggml_tensor * glu_tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_ASSERT(gate_tensor && up_tensor && glu_tensor); + + const ggml_tensor * Wg = gate_tensor->src[0]; + const ggml_tensor * Wu = up_tensor->src[0]; + const ggml_tensor * src1 = gate_tensor->src[1]; // == up_tensor->src[1] + const ggml_tensor * dst = glu_tensor; + + GGML_ASSERT(Wg && Wg->extra); + GGML_ASSERT(Wu && Wu->extra); + GGML_ASSERT(src1 && src1->extra); + GGML_ASSERT(dst && dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_K * extra_g = (ggml_tensor_extra_cl_q4_K *)Wg->extra; + ggml_tensor_extra_cl_q4_K * extra_u = (ggml_tensor_extra_cl_q4_K *)Wu->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int K = Wg->ne[0]; // ne00 + const int M = Wg->ne[1]; // ne01 (= ffn intermediate width) + const int N = 1; // decode GEMV + + const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; + const int glu_op = (int)ggml_get_glu_op(dst); + + cl_context context = backend_ctx->context; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + // q images for the two weight matrices (standard noshuffle layout) + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * K / 2 / 4; + img_desc.buffer = extra_g->q; + cl_mem qg_img = nullptr, qu_img = nullptr; + CL_CHECK((qg_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_desc.buffer = extra_u->q; + CL_CHECK((qu_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // shared activation image (one column at decode) + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + cl_mem b_sub_buf = nullptr, b_img = nullptr; + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qg_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_g->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_g->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra_g->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &qu_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra_u->d)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_u->dm)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extra_u->s)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &glu_op)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_uchar), &mask_hi2)); + + // K-split = nsg_y subgroups. HARD-CAP at 8 (512 work-items): the fused + // kernel's cross-subgroup reduce uses a float4 reduceLM (gate+up packed) = + // 2x the LDS of the base GEMV's float2 reduce, so 16 co-resident subgroups + // exceed the per-CU LDS budget on X2 and the WG barrier DEADLOCKS -> GPU TDR + // (reproduced on upstream gemma-4 E4B decode, K=2560 M=10240). This used to + // be masked: get_kernel_workgroup_size reported 896 for this kernel (so the + // cap loop fell to 8), but it now returns 1024 and the Adreno per-kernel WG + // query is unreliable (over-reports), so cap explicitly instead of trusting + // it. nsg_y < 16 also means the cross-subgroup accumulation grouping differs + // from the standalone wide (nsg=16) GEMV, so the output is coherent but NOT + // byte-identical to the per-op path. Keep the maxwg query as a further floor + // for any driver that reports < 512. + size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); + // X2E: 4 is faster than 8 on every q4_K FFN shape measured there + size_t nsg_y = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? 4 : 8; + while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } + size_t local_work_size[3] = { 64, nsg_y, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 }; + + if (getenv("GGML_OPENCL_FUSE_DEBUG")) { + static int dbg = 0; + if (dbg < 3) { fprintf(stderr, "[FUSE_MM_GLU] fired #%d K=%d M=%d glu_op=%d nsg=%zu maxwg=%zu\n", ++dbg, K, M, glu_op, nsg_y, maxwg); fflush(stderr); } + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(qg_img)); + CL_CHECK(clReleaseMemObject(qu_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +#else + GGML_UNUSED(backend); + GGML_UNUSED(gate_tensor); + GGML_UNUSED(up_tensor); + GGML_UNUSED(glu_tensor); +#endif +} + + +static void ggml_opencl_op_rms_norm_mul_add_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { + GGML_ASSERT(rms_norm_tensor && mul_tensor && add_tensor); + + const ggml_tensor * src0 = rms_norm_tensor->src[0]; + const ggml_tensor * src1 = mul_tensor->src[0] == rms_norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; + const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; + const ggml_tensor * dst = add_tensor; + + GGML_ASSERT(src0 && src0->extra); + GGML_ASSERT(src1 && src1->extra); + GGML_ASSERT(src2 && src2->extra); + GGML_ASSERT(dst && dst->extra); + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + float eps; + memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); + + const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; + const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; + const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; + const cl_ulong nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + GGML_ASSERT(ne00 % 4 == 0); + + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) sgs = 64; + else if (backend_ctx->gpu_family == INTEL) sgs = 32; + else GGML_ASSERT(false && "Unsupported GPU"); + + cl_kernel kernel = backend_ctx->kernel_rms_norm_mul_add; + + int nth = sgs; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth < ne00 && nth < max_workgroup_size) nth *= 2; + nth = MIN(nth, max_workgroup_size); + nth = MIN(nth, ne00); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(float)*sgs, NULL)); + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + + // NOTE: this may oversynchronize by synchronizing with + // backends/devices which don't compute 'cgraph's + // dependencies. + sync_with_other_backends(backend); + + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + continue; + } + + if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } + // Fuse the MoE combine: router-weight mul + cross-expert add chain -> + // one weighted-sum-across-experts kernel. + // Fold the gpt-oss MoE bias epilogue: add_id(gate_bias) + add_id(up_bias) + + // glu(swiglu_oai) -> one kernel, leaving the two matmuls as their own dispatches. + // Both add_ids are in-place passes over a tensor the GLU reads again, so this + // drops two full read+write passes per layer. Opt out GGML_OPENCL_FUSE_MOE_BIAS_GLU=0. + if (backend_ctx->fuse_moe_bias_glu && !backend_ctx->disable_fusion && + ggml_opencl_can_fuse_moe_bias_glu(cgraph, i)) { + ggml_cl_moe_bias_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2], + cgraph->nodes[i+3], cgraph->nodes[i+4]); + i += 4; + continue; + } + + // Fold the MoE down-projection bias into the combine: add_id(down_bias) + the whole + // combine subgraph -> one kernel. Checked before the plain combine arm so the longer + // pattern wins. Opt out GGML_OPENCL_FUSE_MOE_BIAS_COMBINE=0. + if (backend_ctx->fuse_moe_bias_combine && backend_ctx->fuse_moe_combine && + !backend_ctx->disable_fusion) { + const ggml_tensor * bias_combine_out = nullptr; + if (ggml_opencl_can_fuse_moe_bias_combine(cgraph, i, &bias_combine_out)) { + ggml_cl_moe_bias_combine_fused(backend, node, cgraph->nodes[i+1], bias_combine_out); + i += 2 * (int)node->ne[1]; // ADD_ID + MUL + k VIEWs + (k-1) ADDs + continue; + } + } + + if (backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) { + const ggml_tensor * combine_out = nullptr; + if (ggml_opencl_can_fuse_moe_combine(cgraph, i, &combine_out)) { + ggml_cl_moe_combine_fused(backend, node, combine_out); + i += 2 * (int)node->ne[1] - 1; // skip the k VIEWs + (k-1) ADDs + continue; + } + } + + // Fuse rms_norm + mul(weight) + add(residual). Checked before the + // rms_norm+mul fuse so the 3-op pattern wins over its 2-op prefix. + // Default on, opt-out GGML_OPENCL_FUSE_RMS_ADD=0. + if (!backend_ctx->disable_fusion && backend_ctx->fuse_rms_add && + ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + ggml_opencl_op_rms_norm_mul_add_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { + ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]); + i++; + continue; + } + // Fuse mul_mat(Wg,x) + mul_mat(Wu,x) + glu — fold the FFN's two decode + // GEMVs and the GLU into one dispatch. q4_K only (guarded below); the + // fused kernel uses the same accumulation/reduction order and the same + // scalar GLU formula -> coherent. Default on, opt-out GGML_OPENCL_FUSE_MM_GLU=0. +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // The fused executor (ggml_cl_mul_mat_q4_k_glu_fused) is image-path / + // Adreno-only (GGML_ABORT on the non-Adreno #else); gate the dispatch to + // match so the FFN GLU subgraph stays dormant on Intel/other drivers. + if (backend_ctx->fuse_mm_glu && !backend_ctx->disable_fusion && + ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU })) { + ggml_cl_mul_mat_q4_k_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } +#endif + + bool ok = ggml_cl_compute_forward(backend, node); + if (!ok) { + GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); + } + GGML_ASSERT(ok); + } + + return GGML_STATUS_SUCCESS; +} + +// The optimized gemm and gemv kernels are used for large matrices without batch. +// tensor is the quantized weights matrix. +inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + int64_t threshold_ne0 = 512; + int64_t threshold_ne1 = 512; + if (!backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) && + backend_ctx->adreno_cl_compiler_version.type != DX) { + threshold_ne0 = 128; + threshold_ne1 = 128; + } + // K floor 256 on X2E: the gemma-4 E2B/E4B MTP drafters' weights are all K = 256. + // Default on for X2E, GGML_OPENCL_K_FLOOR_256=0/1 forces it. + static const char * k_floor_256_env = getenv("GGML_OPENCL_K_FLOOR_256"); + const bool k_floor_256 = k_floor_256_env ? atoi(k_floor_256_env) != 0 : backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E; + if (k_floor_256 && threshold_ne0 > 256) { + threshold_ne0 = 256; + } + bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && + tensor->ne[2] == 1 && tensor->ne[3] == 1; + + // the transposed layout needs K % 32 == 0 and M % 4 == 0 + if (tensor->ne[0] % 32 != 0 || tensor->ne[1] % 4 != 0) { + return false; + } + + // The noshuffle layout packs 2 rows per 32-bit texel and the GEMV reads it at an + // ne1/2 texel stride with an exact-cover dispatch, so it is only addressable when + // ne1 is a multiple of 64; an unaligned ne1 truncates the stride and the weight is + // read misaligned. That is a property of the layout, not of one quant -- q4_K, q5_K + // and q8_0 read the same packing as q6_K. The bound is 64, not 128: a q8_0 attention + // weight of ne1 = 2880 is a multiple of 64 but not 128 and is correct. + switch (tensor->type) { + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q8_0: + return threashold_ok && tensor->ne[1] % 64 == 0; + default: + break; + } + return threashold_ok; +} + +static bool adreno_e17_compiler_quirks(const ggml_backend_opencl_context *backend_ctx) { + if (!backend_ctx || backend_ctx->gpu_family != GPU_FAMILY::ADRENO || + backend_ctx->adreno_cl_compiler_version.type != ADRENO_CL_COMPILER_TYPE::E17) { + return false; + } + const char * env = getenv("GGML_OPENCL_ADRENO_E17_QUIRKS"); + return !(env && env[0] == '0'); +} + +static bool ggml_cl_q2_k_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_Q2_K_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_Q2_K && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_Q3_K_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_Q3_K && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq4_xs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ4_XS_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ4_XS && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq3_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ3_S_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ3_S && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq3_xxs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ3_XXS_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ3_XXS && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq1_m_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ1_M_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ1_M && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq1_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ1_S_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ1_S && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_q2_0_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_Q2_0_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_Q2_0 && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq2_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ2_S_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ2_S && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq2_xs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ2_XS_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ2_XS && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq2_xxs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ2_XXS_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ2_XXS && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + // The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*. + // Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights. + // So, exclude A6x and A7x from using Adreno MoE kernels for now. + // The quants that have a general mul_mat_id kernel fallback to the general version; the + // rest fallback to CPU. + if (backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN)) { + return false; + } + + if (adreno_e17_compiler_quirks(backend_ctx)) { + return false; + } + + int ne01 = tensor->ne[1]; + return (((strstr(tensor->name, "ffn") != NULL) && (strstr(tensor->name, "exps") != NULL)) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 32 == 0); +} + +// Device default for the tiled-wide lm_head/embed GEMV layout: ON for X2E and A8X. +// +// These kernels were previously off everywhere on the grounds that they compute +// wrong values at multi-superblock K. They do not: that NMSE ~2 came from the +// backend having no get_tensor restore path for the tiled layout, so +// test-backend-ops (which builds its CPU reference by copying the weights back +// out of the backend) compared a correct GPU result against a reference +// dequantized from tiled bytes. With the restore path added, MUL_MAT passes with +// the tiled kernels on, unmodified, on both devices. +// +// Perf, Qwen3-4B-Q4_K_M (q6_K lm_head 151936x2560), tg128, matched pairs with +// alternating lead, tiled vs o4: +// +// A8X +11.9% 6/6 pairs positive, order bias -0.06% (16.93 vs 15.14 tok/s) +// X2E +6.9% 4/4 pairs positive, order bias -0.03% (35.24 vs 32.87 tok/s) +// +// Measure this one on a COLD device. These kernels are far more clock-sensitive +// than the o4 route they replace: on a heat-soaked A8X (CPU cap at 1.5-1.9 GHz) +// tiled pins at ~14.2 tok/s while o4 still makes ~14.9, which reads as a 4-5% +// LOSS and inverts the ranking. The same box, after a reboot and a gate that +// waits for policy6 to return to 4396800, reports the +11.9% above with no +// order bias. A7X regresses hard on this layout and stays off. +// GGML_OPENCL_{Q4K,Q6K}_GEMV_TILED forces either way (=0 off, any other value on). +inline bool tiled_gemv_default_on(const ggml_backend_opencl_context *backend_ctx) { + return backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X); +} + +// Tiled-wide q6_K GEMV (default OFF; GGML_OPENCL_Q6K_GEMV_TILED forces either +// way: =0 off everywhere, any other value on everywhere). +// Both the convert (set_tensor) and the GEMV dispatch must agree on this so the +// buffer layout matches the kernel. +inline bool q6k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { + static const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_TILED"); + if (e && e[0] != '\0') { + return e[0] != '0'; + } + return tiled_gemv_default_on(backend_ctx); +} + +// Only the long-vocab lm_head/embed shapes use the tiled layout; ne01 % 64 == 0 +// is required by the 64-row tiling (no row padding in the buffers). +// use_adreno_kernels is required: only the Adreno GEMV path can read the tiled +// layout, so converting a weight it would decline (e.g. ne00 < 512) leaves the +// generic kernel reading tiled bytes as plain SOA. +inline bool use_q6k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + return q6k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q6_K && + tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && + use_adreno_kernels(backend_ctx, tensor); +} + +// Select untiled Q4_K before upload if the batched DP4A reader is unavailable. +inline bool q4k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { + if (!backend_ctx->has_integer_dot) { + return false; + } + static const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_TILED"); + if (e && e[0] != '\0') { + return e[0] != '0'; + } + return tiled_gemv_default_on(backend_ctx); +} +inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + return q4k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q4_K && + tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && + use_adreno_kernels(backend_ctx, tensor); +} + +inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + + bool adreno_kernel = use_adreno_kernels(backend_ctx, tensor); + + size_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]; + + // The 2D weight transpose (transpose_2d_as_*) tiles rows by 4 over a 2D matrix, + // so it requires K(ne0)%32==0, M(ne1)%4==0 and ne2==ne3==1. + const bool shape_ok = (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 4 == 0) && + (tensor->ne[2] == 1) && (tensor->ne[3] == 1); + + return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27 +} + +inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + if (!use_adreno_kernels(backend_ctx, tensor)) { + return false; + } + + const size_t elem_num = ggml_nelements(tensor); + const size_t q_img_width = elem_num / 8; + const size_t qh_img_width = elem_num / 16; + const bool shape_ok = tensor->ne[0] % 32 == 0 && tensor->ne[1] % 4 == 0 && + tensor->ne[2] == 1 && tensor->ne[3] == 1; + + return shape_ok && q_img_width <= backend_ctx->image_max_buffer_size && + qh_img_width <= backend_ctx->image_max_buffer_size; +} + +inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || + !(backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin || + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin)) { + return false; + } + + // bin kernels require ne2 == 1 and ne3 == 1 for weights + return tensor->ne[2] == 1 && tensor->ne[3] == 1 && + (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v) { + if (backend_ctx->fa.kernel_flash_attn_f32_f16_bin == nullptr) { + return false; + } + + const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + + const int n_q = q->ne[1]; + const int dk = q->ne[0]; + const int dv = v->ne[0]; + + // do not choose bin FA kernels to avoid repacking the KV cache for small + // verify batches on X2 + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && is_mixed && + (dk == 256 || dk == 512) && n_q <= 4) { + return false; + } + + constexpr bool prefill_only = true; + + return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO && + (is_mixed || is_q8_0) && (dk == dv) + && (dk == 64 || dk == 128 || dk == 256 || dk == 512) + && (!prefill_only || n_q != 1)); +} +#endif + +// The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it +// is SLOWER than the route it replaces, not because it is unsafe. It was first +// parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that +// was a misattribution (the test-backend-ops dst sentinel was tripped by the o4 +// GEMV's unguarded tail store, fixed separately - and at the shape it was blamed +// for, k=1536, this predicate returns false anyway, so the flat route never ran). +// +// The escape's original rationale, "gemv_noshuffle perf drops for large M", +// predates the o4 kernel, which now covers the same long-vocab shapes and beats +// this route on every device measured (Qwen3-4B-Q4_K_M, q6_K lm_head +// 151936x2560, tg128, matched pairs vs o4): A8X -10.3% (0/3 pairs), X2E -3.7% +// (0/3). Keep it reachable for shapes o4 declines, but do not default it on. +static inline bool flat_large_m_enabled() { + static const char * e = getenv("GGML_OPENCL_FLAT_LARGE_M"); + static const bool en = e != nullptr && atoi(e) != 0; + return en; +} + +// The noshuffle Q4_K weight image stores eight weights per uint texel. +static inline bool q4_K_weight_image_fits(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + const size_t texels = (size_t) ggml_nelements(tensor) / 8; + return texels != 0 && texels <= backend_ctx->image_max_buffer_size; +} + +static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + if (tensor->ne[1] % 4 != 0 && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + return true; + } + + if (tensor->ne[2] != 1 || tensor->ne[3] != 1 || use_q4k_tiled(backend_ctx, tensor)) { + return false; + } + + // The image limit is a correctness guard, independent of the large-M opt-in. + if (!q4_K_weight_image_fits(backend_ctx, tensor)) { + return true; + } + + // gemv_noshuffle variant perf drops for large M, use flat variant for large M. + // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. + // note that this forces large M weights to use LM GEMM. + return flat_large_m_enabled() && tensor->ne[1] >= 32768; +} + +static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + // NOTE on ordering: the ne01 % 128 escape below is a CORRECTNESS guard, not a + // performance one, so it must be reachable regardless of flat_large_m_enabled(). + // The opt-in gate therefore sits after it, and after the tiled deferral. + // gemv_noshuffle variant perf drops for large M, use flat variant for large M. + // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. + // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold. + // note that this forces large M weights to use LM GEMM. + // When this branch's tiled-canonical lm_head/embed layout is active, the weight is + // converted to the 64-row tiled layout, which the flat gemv would misread as + // garbage. use_q6k_tiled owns these large-M weights (it requires ne01 % 64 == 0, + // so it never claims an odd-vocab weight), so defer to it first. + if (use_q6k_tiled(backend_ctx, tensor)) { + return false; + } + // The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the + // gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of + // ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles. + // It is therefore only correct for ne01 % 128 == 0: an odd ne01 (e.g. granitemoe + // lm_head [1536, 49155] -- odd vocab) truncates the texel stride, misaligning every + // odd column of the transposed layout (gross garbage) and dropping the last row; + // other non-multiples over-dispatch and write past the end of dst. Route such + // tensors to the flat GEMV + regular convert; the matching GEMM (ne1>1) falls back + // to CPU (see supports_op). All standard even-vocab/hidden dims are multiples of + // 128 and keep the noshuffle path. + if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + return true; + } + + if (!flat_large_m_enabled()) { + return false; + } + + // The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a + // proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head. + // Add a direct size escape so such weights also take the flat path, without changing + // which weights ne0 >= 2048 already routes there. + // The size escape is not taken on the A7X since its compiler miscompiles the flat K-quant GEMV + return tensor->ne[1] >= 32768 + && (tensor->ne[0] >= 2048 || (backend_ctx->adreno_gen != ADRENO_GPU_GEN::A7X && ggml_nbytes(tensor) >= (256ull << 20))) + && tensor->ne[2] == 1 && tensor->ne[3] == 1; +} + +inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin) { + return false; + } + // bin kernels require ne2 == 1 and ne3 == 1 for weights + return tensor->ne[2] == 1 && tensor->ne[3] == 1 && + (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && + !use_q6k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) { + return false; + } + // bin kernels require ne2 == 1 and ne3 == 1 for weights + return tensor->ne[2] == 1 && tensor->ne[3] == 1 && + (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && + !use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +inline bool use_q5_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_mul_mat_can_use_fwht(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * op) { + const ggml_tensor * a = op->src[0]; + const ggml_tensor * b = op->src[1]; + const int64_t n = a->ne[0]; + + return ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && + a->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + (b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16) && + n >= 2 && n <= std::numeric_limits<int>::max() && (n & (n - 1)) == 0 && a->ne[1] == n && + (uint64_t)n <= backend_ctx->local_mem_size/sizeof(float) && + ggml_is_contiguous_rows(b) && ggml_is_contiguous(op) && ggml_are_same_shape(b, op); +} + +static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context; + ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; + + // reject ops that match the opfilter regex + if (dev_ctx->opfilter && std::regex_match(std::string(ggml_op_desc(op)), *dev_ctx->opfilter)) { + return false; + } + + switch (op->op) { + case GGML_OP_NONE: + return true; + case GGML_OP_GET_ROWS: + switch (op->src[0]->type) { + case GGML_TYPE_I32: + return op->type == GGML_TYPE_I32; + case GGML_TYPE_F32: + case GGML_TYPE_F16: + return true; + case GGML_TYPE_Q4_0: +#ifdef GGML_OPENCL_SOA_Q + // We do not support flattened Q4_0 (and possibly other Q's) + return false; +#else // GGML_OPENCL_SOA_Q + return true; +#endif // GGML_OPENCL_SOA_Q + default: + return false; + } + case GGML_OP_SET_ROWS: + { + // TODO: add support + // ref: https://github.com/ggml-org/llama.cpp/pull/14274 +#pragma message("TODO: implement BF16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, IQ4_NL support (https://github.com/ggml-org/llama.cpp/pull/14661)") + if (op->src[0]->type != GGML_TYPE_F32) { + return false; + } + switch (op->type) { + case GGML_TYPE_F16: + case GGML_TYPE_F32: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_0: + return (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); + default: + return false; + } + } + case GGML_OP_CPY: + case GGML_OP_DUP: + case GGML_OP_CONT: + switch (op->src[0]->type) { + case GGML_TYPE_F32: + switch (op->type) { + case GGML_TYPE_I32: + case GGML_TYPE_F16: + case GGML_TYPE_F32: + return true; + default: + return false; + } + case GGML_TYPE_F16: + switch (op->type) { + case GGML_TYPE_F16: + case GGML_TYPE_F32: + return true; + default: + return false; + } + case GGML_TYPE_I32: + switch (op->type) { + case GGML_TYPE_F32: + case GGML_TYPE_I32: + return true; + default: + return false; + } + default: + return false; + } + case GGML_OP_SET: { + return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32) && + op->type == op->src[0]->type && + op->type == op->src[1]->type; + } + case GGML_OP_SCALE: + return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); + case GGML_OP_ADD: + if (op->type == GGML_TYPE_F16) { + const bool src0_ok = op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32; + const bool src1_ok = op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32; + if (src0_ok && src1_ok) { + return true; + } + } + case GGML_OP_MUL: + case GGML_OP_DIV: + case GGML_OP_SUB: + return (op->src[0]->type == op->src[1]->type) && + (op->src[0]->type == op->type) && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); + case GGML_OP_ADD_ID: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_SQR: + case GGML_OP_SQRT: + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + ggml_is_contiguous(op->src[0]); + case GGML_OP_UNARY: + switch (ggml_get_unary_op(op)) { + case GGML_UNARY_OP_GELU: + case GGML_UNARY_OP_SILU: + case GGML_UNARY_OP_RELU: + case GGML_UNARY_OP_GELU_ERF: + case GGML_UNARY_OP_GELU_QUICK: + return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; + case GGML_UNARY_OP_SIGMOID: + return op->src[0]->type == op->type && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16); + case GGML_UNARY_OP_TANH: + case GGML_UNARY_OP_NEG: + case GGML_UNARY_OP_EXP: + // Adreno F16 exp/expm1 overflow even post-half->float convert. + return op->src[0]->type == GGML_TYPE_F32; + case GGML_UNARY_OP_EXPM1: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_UNARY_OP_ABS: + case GGML_UNARY_OP_SGN: + case GGML_UNARY_OP_STEP: + case GGML_UNARY_OP_ELU: + case GGML_UNARY_OP_HARDSWISH: + case GGML_UNARY_OP_HARDSIGMOID: + case GGML_UNARY_OP_FLOOR: + case GGML_UNARY_OP_CEIL: + case GGML_UNARY_OP_ROUND: + case GGML_UNARY_OP_TRUNC: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; + case GGML_UNARY_OP_SOFTPLUS: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; + default: + return false; + } + case GGML_OP_GLU: + switch (ggml_get_glu_op(op)) { + case GGML_GLU_OP_GEGLU: + case GGML_GLU_OP_REGLU: + case GGML_GLU_OP_SWIGLU: + case GGML_GLU_OP_SWIGLU_OAI: + case GGML_GLU_OP_GEGLU_ERF: + case GGML_GLU_OP_GEGLU_QUICK: + case GGML_GLU_OP_SWIGLU_CLAMP: + return ggml_is_contiguous_1(op->src[0]) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); + default: + return false; + } + case GGML_OP_TRI: + return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + case GGML_OP_FILL: + return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + case GGML_OP_ARANGE: + return op->type == GGML_TYPE_F32; + case GGML_OP_CLAMP: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_SOFT_MAX: + case GGML_OP_NORM: + return true; + case GGML_OP_RMS_NORM: + return op->ne[0] % 4 == 0 && ggml_is_contiguous_rows(op->src[0]); + case GGML_OP_L2_NORM: + return ggml_is_contiguous_rows(op->src[0]); + case GGML_OP_REPEAT: + switch (op->type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_I32: + case GGML_TYPE_I16: + case GGML_TYPE_BF16: + return op->src[0]->type == op->type; + default: + return false; + } + case GGML_OP_PAD: + // TODO: add circular padding support for opencl, see https://github.com/ggml-org/llama.cpp/pull/16985 + if (ggml_get_op_params_i32(op, 8) != 0) { + return false; + } + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_UPSCALE: { + ggml_scale_mode mode = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & 0xFF); + const bool antialias = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & GGML_SCALE_FLAG_ANTIALIAS); + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + (mode == GGML_SCALE_MODE_NEAREST || mode == GGML_SCALE_MODE_BILINEAR) && !antialias; + } + case GGML_OP_CONV_2D: + return (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16) || + (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) || + (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); + case GGML_OP_CONV_2D_DW: + return op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && + op->src[1]->nb[0] == sizeof(float) && op->nb[0] == sizeof(float) && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); + case GGML_OP_SSM_CONV: + return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); + case GGML_OP_SSM_SCAN: { + if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || + op->src[1]->type != GGML_TYPE_F32 || op->src[2]->type != GGML_TYPE_F32 || + op->src[3]->type != GGML_TYPE_F32 || op->src[4]->type != GGML_TYPE_F32 || + op->src[5]->type != GGML_TYPE_F32 || op->src[6]->type != GGML_TYPE_I32) { + return false; + } + + const int64_t d_state = op->src[0]->ne[0]; + return d_state >= 1 && d_state <= 256 && (d_state & (d_state - 1)) == 0; + } + case GGML_OP_GATED_DELTA_NET: + { + // Match the Vulkan backend: only F32 -> F32, S_v in {16, 32, 64, 128}. + if (op->src[0]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) { + return false; + } + const int64_t S_v = op->src[2]->ne[0]; + return S_v == 16 || S_v == 32 || S_v == 64 || S_v == 128; + } + case GGML_OP_CONCAT: + { + const ggml_type t = op->src[0]->type; + return op->src[1]->type == t && op->type == t && + !ggml_is_quantized(t) && ggml_blck_size(t) == 1 && + (ggml_type_size(t) == 1 || ggml_type_size(t) == 2 || + ggml_type_size(t) == 4 || ggml_type_size(t) == 8); + } + case GGML_OP_TIMESTEP_EMBEDDING: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_GROUP_NORM: + return ggml_is_contiguous(op->src[0]); + case GGML_OP_MUL_MAT: + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; + } + if (ggml_cl_mul_mat_can_use_fwht(backend_ctx, op)) { + return true; + } + if (op->src[0]->type == GGML_TYPE_F16) { + return true; + } else if (op->src[0]->type == GGML_TYPE_BF16) { + return true; + } else if (op->src[0]->type == GGML_TYPE_F32) { + return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q1_0) { + return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q4_0) { + // Non-contig src0 routes through on-device dequant-to-f16. + return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_IQ2_XXS) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq2_xxs_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ2_XS) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq2_xs_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ2_S) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq2_s_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_Q2_0) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_q2_0_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ1_S) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq1_s_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ1_M) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq1_m_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ3_S) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq3_s_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ3_XXS) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq3_xxs_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ4_XS) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq4_xs_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_Q3_K) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_Q2_K) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_q2_k_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && + ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_Q4_1 || + op->src[0]->type == GGML_TYPE_Q5_0 || op->src[0]->type == GGML_TYPE_Q5_1 || + op->src[0]->type == GGML_TYPE_MXFP4 || + op->src[0]->type == GGML_TYPE_IQ4_NL || + op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q6_K) { + // The E031.41 compiler (usually with A7x) miscompiles the flat K-quant + // GEMV kernels (kernel_mul_mv_q*_K_f32_flat) and makes lm_head run much + // slower than it should. So, make it fallback to CPU to preserve performance + // for this compiler series. + static const char * a7x_lmhead_env = getenv("GGML_OPENCL_A7X_LMHEAD_CPU"); + static const bool a7x_lmhead_cpu = (a7x_lmhead_env == nullptr || a7x_lmhead_env[0] != '0'); + if (a7x_lmhead_cpu && + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q6_K) && + op->src[0]->ne[1] >= 32768) { // vocab-scale weight; no FFN/attn weight is this tall + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_Q8_0) { + // ggml_cl_mul_mat_q8_0_f32_adreno now honors src1/dst view_offs (the + // activation sub-buffer starts at offset1 and the kernels take offsetd), + // so a broadcast q8_0 matmul (src1 batch > src0 batch, e.g. Qwen3.5-9B-UD + // / Qwen3.6-35B q8_0 GDN ssm_out) runs on GPU via the per-slice broadcast + // iteration in ggml_cl_mul_mat. No special-casing needed. + return op->src[1]->type == GGML_TYPE_F32; + } + return false; + case GGML_OP_MUL_MAT_ID: + if (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) { + return op->src[1]->type == GGML_TYPE_F32 && op->src[2]->type == GGML_TYPE_I32 && op->type == GGML_TYPE_F32 && + op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && op->src[1]->nb[0] == sizeof(float) && + op->nb[0] == sizeof(float) && op->src[0]->ne[0] <= INT_MAX - 256 && op->ne[0] <= INT_MAX - 64 && + op->src[1]->ne[1] <= INT_MAX && op->ne[1] <= INT_MAX && op->ne[2] <= INT_MAX - 64; + } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if ((ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) && + use_adreno_moe_kernels(backend_ctx, op->src[0])) { + return false; + } +#endif + if (op->src[0]->type == GGML_TYPE_Q4_0 || + op->src[0]->type == GGML_TYPE_Q8_0 || + op->src[0]->type == GGML_TYPE_MXFP4) { + if (op->src[1]->type == GGML_TYPE_F32) { + return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); + } + } + // q4_0, q8_0 and mxfp4 have general MUL_MAT_ID support, + // the quantizations here currently do not - they are only supported by Adreno with certain shapes + if (op->src[0]->type == GGML_TYPE_Q4_1 || + op->src[0]->type == GGML_TYPE_Q5_0 || + op->src[0]->type == GGML_TYPE_Q5_1 || + op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q6_K) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (op->src[1]->type == GGML_TYPE_F32) { + return use_adreno_moe_kernels(backend_ctx, op->src[0]) + && ggml_is_contiguous(op->src[0]) + && ggml_is_contiguous(op->src[1]); + } +#endif + return false; + } + return false; + case GGML_OP_RESHAPE: + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + return true; + case GGML_OP_DIAG: + return true; + case GGML_OP_ROPE: { + const int mode = ((const int32_t *) op->op_params)[2]; + if (!ggml_rope_get_truncate(op)) { + return false; + } + const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; + const bool is_vision = mode == GGML_ROPE_TYPE_VISION; + if (is_mrope && !is_vision) { + if (op->src[0]->type == GGML_TYPE_F32 || + op->src[0]->type == GGML_TYPE_F16) { + return true; + } + return false; + } + if (is_vision) { + if (op->src[0]->type == GGML_TYPE_F32 || + op->src[0]->type == GGML_TYPE_F16) { + return true; + } + return false; + } + return true; } + case GGML_OP_SOLVE_TRI: + return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); + case GGML_OP_IM2COL: + return true; + case GGML_OP_IM2COL_3D: + return true; + case GGML_OP_ARGMAX: + load_cl_kernels_argmax(backend_ctx); + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_I32 && + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op) && + backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_argmax_f32_i32) >= 64; + case GGML_OP_LIGHTNING_INDEXER: + return op->src[0]->type == GGML_TYPE_F32 && + (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_BF16) && + op->src[2]->type == GGML_TYPE_F32 && op->src[3]->type == GGML_TYPE_F16 && + op->src[0]->nb[0] == sizeof(float) && op->src[1]->nb[0] == ggml_type_size(op->src[1]->type) && + op->src[2]->nb[0] == sizeof(float) && op->src[3]->nb[0] == sizeof(ggml_fp16_t); + case GGML_OP_TOP_K: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_I32; + case GGML_OP_TOP_K_MASK: + return (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32) && + op->src[0]->type == op->type && + op->src[0]->nb[0] == ggml_type_size(op->type) && op->nb[0] == ggml_type_size(op->type) && + op->src[1]->type == GGML_TYPE_I32 && + (!op->src[2] || op->src[2]->type == GGML_TYPE_F32) && + (!op->src[3] || op->src[3]->type == GGML_TYPE_I32) && + (!op->src[4] || op->src[4]->type == GGML_TYPE_I32); + case GGML_OP_ARGSORT: { + load_cl_kernels_argsort(backend_ctx); - qh_img = nullptr; - d_img = nullptr; - size_qs = 0; - size_qh = 0; - size_d = 0; - } -}; - -struct ggml_tensor_extra_cl_q5_1 { - // Quantized values. - cl_mem qs = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem qs_img = nullptr; - // 5-th bit values. - cl_mem qh = nullptr; - // 5-th bit values in image1d_buffer_t. - cl_mem qh_img = nullptr; - // Scales. - cl_mem d = nullptr; - // Scales in image1d_buffer_t. - cl_mem d_img = nullptr; - // Min - cl_mem m = nullptr; - // Min in image1d_buffer_t. - cl_mem m_img = nullptr; - // Size of quantized values. - size_t size_qs = 0; - // Size of 5-th bit values. - size_t size_qh = 0; - // Size of scales. - size_t size_d = 0; - // Size of min values. - size_t size_m = 0; + cl_kernel kernel = backend_ctx->kernel_argsort_f32_i32; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - ~ggml_tensor_extra_cl_q5_1() { - reset(); - } + int cols = 1; + while (cols < op->ne[0]) { + cols *= 2; + } - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (qs != nullptr) { - CL_CHECK(clReleaseMemObject(qs)); - qs = nullptr; - } - if (qh != nullptr) { - CL_CHECK(clReleaseMemObject(qh)); - qh = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - if (m != nullptr) { - CL_CHECK(clReleaseMemObject(m)); - m = nullptr; - } - if (qs_img != nullptr) { - CL_CHECK(clReleaseMemObject(qs_img)); - qs_img = nullptr; + return cols <= max_workgroup_size && op->src[0]->type == GGML_TYPE_F32; } - // qh_img, d_img, and m_img are not currently allocated separately. - // TODO: initialize them for non SMALL_PATH path, or remove them. - qh_img = nullptr; - d_img = nullptr; - m_img = nullptr; - size_qs = 0; - size_qh = 0; - size_d = 0; - size_m = 0; - } -}; + case GGML_OP_SUM_ROWS: + case GGML_OP_CUMSUM: + return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + for (int i = 0; i < (op->op == GGML_OP_DSV4_HC_PRE ? 2 : 4); ++i) { + if (op->src[i] && op->src[i]->type != GGML_TYPE_F32) { + return false; + } + } + return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + case GGML_OP_MEAN: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_FLASH_ATTN_EXT: { + // K/V row lists (ggml_flash_attn_ext_rows) are not supported + if (op->src[5] != nullptr) { + return false; + } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_fa_bin_kernels_prefill(backend_ctx, op->src[0], op->src[1], op->src[2])) { + return true; + } +#endif + // The E17 compilers segfault while building FA kernels, skip E17 for now + if (adreno_e17_compiler_quirks(backend_ctx)) { + return false; + } + const ggml_tensor * q = op->src[0]; + const ggml_tensor * k = op->src[1]; + const ggml_tensor * v = op->src[2]; -struct ggml_tensor_extra_cl_mxfp4 { - // Quantized values. - cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem q_img = nullptr; - // Scales in E8M0. - cl_mem e = nullptr; - // Scales in image1d_buffer_t. - cl_mem e_img = nullptr; - // Size of quantized values. - size_t size_q = 0; - // Size of scales. - size_t size_e = 0; + const int dk = q->ne[0]; + const int dv = v->ne[0]; - ~ggml_tensor_extra_cl_mxfp4() { - reset(); - } + const struct { int dk; int dv; } supported_dims[] = { + { 40, 40}, { 64, 64}, { 80, 80}, { 96, 96}, + {112, 112}, {128, 128}, {192, 128}, + {192, 192}, {256, 256}, + {512, 512}, + }; - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (e != nullptr) { - CL_CHECK(clReleaseMemObject(e)); - e = nullptr; - } - if (q_img != nullptr) { - CL_CHECK(clReleaseMemObject(q_img)); - q_img = nullptr; - } - // Currently, e_img is not used. They can be image1d_buffer_t - // that wraps around q and d to utilize image access path. - e_img = nullptr; - size_q = 0; - size_e = 0; - } -}; + bool dims_supported = false; + for (size_t i = 0; i < sizeof(supported_dims)/sizeof(supported_dims[0]); ++i) { + if (supported_dims[i].dk == dk && supported_dims[i].dv == dv) { + dims_supported = true; + break; + } + } + if (!dims_supported) { + return false; + } -struct ggml_tensor_extra_cl_q8_0 { - cl_mem q = nullptr; - cl_mem q_img = nullptr; + const bool is_f32_f32 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && + v->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + const bool is_f16_f16 = q->type == GGML_TYPE_F16 && k->type == GGML_TYPE_F16 && + v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; + const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && + v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; - cl_mem d = nullptr; - cl_mem d_img = nullptr; + const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && + v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 && + dk % 32 == 0 && dv % 32 == 0; + const bool is_f32_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && + v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 && + dk % 32 == 0 && dv % 32 == 0; - // Uniform per-16-segment scale (16/superblock) for the generic dp4a MoE GEMM. - // Expanded from d at set_tensor; the int8 codes are reused from q. - // q8_0 is symmetric so no min buffer (has_min=0). - cl_mem scale = nullptr; + // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram + // building the flash_attn programs whose KV path is mixed-type or + // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it + // is DK-independent). It is a driver crash, not codegen-wrong-output, so + // it cannot be caught in-process (fatal=false only handles clean compile + // errors). The uniform f16_f16 / f32_f32 programs compile fine on this + // compiler, so decline only the KV-convert variants; ggml then runs + // those (f16-KV / quant-KV) attention layers on the CPU backend. + // Negative compiler carve-out, same idiom as the Intel DK=512 decline + // below and the X1E driver-quirk guards. + if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { + return false; + } - size_t size_q = 0; - size_t size_d = 0; + // Asymmetric KV: host-dequants both sides to F32, uses f32 kernel. + auto is_kv_type_ok = [](ggml_type t) { + return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 || + t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q8_0; + }; + const bool is_f32_asym = q->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + k->type != v->type && + is_kv_type_ok(k->type) && is_kv_type_ok(v->type); - ~ggml_tensor_extra_cl_q8_0() { - reset(); - } + const bool kv_combo_ok = is_f32_f32 || is_f16_f16 || is_f32_f16 || + is_f32_q8_0 || is_f32_q4_0 || is_f32_asym; + if (!kv_combo_ok) { + return false; + } - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - if (scale != nullptr) { - CL_CHECK(clReleaseMemObject(scale)); - scale = nullptr; + // Some compilers for A7x (Adreno 740, compiler E031.41) crashes when + // building FA kernels with mixed or quant types (f32_f16, f32_q8_0, f32_q4_0) + // Here we skip all A7x for these kernels to avoid crash + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { + return false; + } + + if (dk == 512) { + if (backend_ctx->gpu_family == INTEL) { + return false; + } + if (!is_f32_f16) { + return false; + } + if (q->ne[1] == 1) { + const char * decode_env = getenv("GGML_OPENCL_FA_DK512_DECODE"); + const int64_t gqa = k->ne[2] > 0 ? q->ne[2] / k->ne[2] : 0; + // GQA 2 and 16 decode on the X2E MQ kernels (see mq_decode_x2e in ggml_cl_flash_attn) + const bool gqa_ok = gqa == 4 || gqa == 8 || + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && (gqa == 2 || gqa == 16)); + if ((decode_env && decode_env[0] == '0') || + backend_ctx->gpu_family != ADRENO || k->ne[2] <= 0 || + !gqa_ok || q->ne[2] % k->ne[2] != 0 || + !ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)) { + return false; + } + } else { + // prefill, BM-tile in its own FA_PREFILL_ONLY program + if (!ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false)) { + return false; + } + } + } + return true; } - // Currently, q_img and d_img are not used. They can be image1d_buffer_t - // that wraps around q and d to utilize image access path. - q_img = nullptr; - d_img = nullptr; - size_q = 0; - size_d = 0; + default: + return false; } -}; +} -struct ggml_tensor_extra_cl_iq4_nl { - cl_mem q = nullptr; - cl_mem q_img = nullptr; +// Forward declaration - implementation appears later in the file. +static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type); - cl_mem d = nullptr; - cl_mem d_img = nullptr; +static ggml_guid_t ggml_backend_opencl_guid() { + static ggml_guid guid = { 0xde, 0xe0, 0x70, 0xa2, 0x73, 0x4e, 0x4d, 0xbc, 0xb0, 0xc7, 0x4f, 0xd4, 0x6d, 0x4e, 0x90, 0xfe }; + return &guid; +} - size_t size_q = 0; - size_t size_d = 0; +static ggml_backend_i ggml_backend_opencl_i = { + /* .get_name = */ ggml_backend_opencl_name, + /* .free = */ ggml_backend_opencl_free, + /* .set_tensor_async = */ NULL, /* ggml_backend_opencl_set_tensor_async */ + /* .get_tensor_async = */ NULL, /* ggml_backend_opencl_get_tensor_async */ + /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, + /* .cpy_tensor_async = */ NULL, /* ggml_backend_opencl_cpy_tensor_async */ + /* .synchronize = */ ggml_backend_opencl_synchronize, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_update = */ NULL, + /* .graph_plan_compute = */ NULL, + /* .graph_compute = */ ggml_backend_opencl_graph_compute, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .graph_optimize = */ NULL, +}; - ~ggml_tensor_extra_cl_iq4_nl() { - reset(); - } +ggml_backend_t ggml_backend_opencl_init(void) { + ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_opencl_reg(), 0); + ggml_backend_opencl_context *backend_ctx = ggml_cl_init(dev); + backend_ctx->ref_count++; - void reset() { - if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } - if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } - q_img = nullptr; - d_img = nullptr; - size_q = 0; - size_d = 0; - } -}; + ggml_backend_t backend = new ggml_backend { + /* .guid = */ ggml_backend_opencl_guid(), + /* .iface = */ ggml_backend_opencl_i, + /* .device = */ dev, + /* .context = */ backend_ctx + }; -struct ggml_tensor_extra_cl_q4_K { - // Quantized values - cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem q_img = nullptr; - // Scales for each super block. - cl_mem s = nullptr; - // Scales - cl_mem d = nullptr; - // Min - cl_mem dm = nullptr; + return backend; +} - ~ggml_tensor_extra_cl_q4_K() { - reset(); +bool ggml_backend_is_opencl(ggml_backend_t backend) { + return backend && backend->iface.get_name == ggml_backend_opencl_name; +} + +// +// buffer +// +struct ggml_backend_opencl_buffer_context { + // A buffer context can hold multiple cl_mem objects. This is for flattening + // quantized weights and should be used with GGML_OPENCL_SMALL_ALLOC where + // each tensor is allocated a separate buffer. When flattening is enabled + // with small allocation, each tensor is backed by two cl_mem objects (for + // quants and scales) packed into a backend_opencl_buffer. + ggml_backend_opencl_buffer_context(cl_mem buf) + : name("OpenCL") { + buffer.push_back(buf); } - void reset() { - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; + ~ggml_backend_opencl_buffer_context() { + for (cl_mem buf : buffer) { + CL_CHECK(clReleaseMemObject(buf)); } - if (s != nullptr) { - CL_CHECK(clReleaseMemObject(s)); - s = nullptr; + for (cl_mem im : img) { + CL_CHECK(clReleaseMemObject(im)); } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; + + // Delete all extras to trigger their destructors + for (ggml_tensor_extra_cl * e : temp_tensor_extras) { + delete e; } - if (dm != nullptr) { - CL_CHECK(clReleaseMemObject(dm)); - dm = nullptr; + for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { + delete e; } - if (q_img != nullptr) { - CL_CHECK(clReleaseMemObject(q_img)); - q_img = nullptr; + for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0) { + delete e; } - } -}; - -struct ggml_tensor_extra_cl_q5_K { - // Lower 4 bits of quantized weights. - cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem q_img = nullptr; - // Upper 1 bit of quantized weights. - cl_mem qh = nullptr; - // Scales for each block. - cl_mem s = nullptr; - // Scales for each super block. - cl_mem d = nullptr; - // Min for each super block. - cl_mem dm = nullptr; - // Uniform per-32-block scale (2/block) + min (1/block, = dm*mn) decoded from the - // 6-bit packed s[] for the generic dp4a MoE GEMM kernel_gemm_moe_q8_1_dp4a. - // Built from s/d/dm at set_tensor; q/qh are reused as-is. - cl_mem scale = nullptr; - cl_mem min = nullptr; - - size_t size_q = 0; - size_t size_qh = 0; - size_t size_s = 0; - size_t size_d = 0; - size_t size_dm = 0; - - ~ggml_tensor_extra_cl_q5_K() { - reset(); - } - - void reset() { - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; + for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { + delete e; } - if (qh != nullptr) { - CL_CHECK(clReleaseMemObject(qh)); - qh = nullptr; + for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1) { + delete e; } - if (s != nullptr) { - CL_CHECK(clReleaseMemObject(s)); - s = nullptr; + for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0) { + delete e; + } + for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1) { + delete e; + } + for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4) { + delete e; + } + for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0) { + delete e; + } + for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0) { + delete e; + } + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl) { + delete e; + } + for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K) { + delete e; + } + for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K) { + delete e; + } + for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K) { + delete e; + } + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K) { + delete e; + } + for (ggml_tensor_extra_cl_iq2_s * e : temp_tensor_extras_iq2_s) { + delete e; + } + for (ggml_tensor_extra_cl_iq2_xs * e : temp_tensor_extras_iq2_xs) { + delete e; + } + for (ggml_tensor_extra_cl_iq2_xxs * e : temp_tensor_extras_iq2_xxs) { + delete e; + } + for (ggml_tensor_extra_cl_q2_0 * e : temp_tensor_extras_q2_0) { + delete e; + } + for (ggml_tensor_extra_cl_iq1_m * e : temp_tensor_extras_iq1_m) { + delete e; + } + for (ggml_tensor_extra_cl_iq1_s * e : temp_tensor_extras_iq1_s) { + delete e; + } + for (ggml_tensor_extra_cl_iq3_s * e : temp_tensor_extras_iq3_s) { + delete e; + } + for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs) { + delete e; + } + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs) { + delete e; + } + for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) { + delete e; } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) { + delete e; } - if (dm != nullptr) { - CL_CHECK(clReleaseMemObject(dm)); - dm = nullptr; + for (ggml_tensor_extra_cl_iq2_s * e : temp_tensor_extras_iq2_s_in_use) { + delete e; } - if (q_img != nullptr) { - CL_CHECK(clReleaseMemObject(q_img)); - q_img = nullptr; + for (ggml_tensor_extra_cl_iq2_xs * e : temp_tensor_extras_iq2_xs_in_use) { + delete e; } - if (scale != nullptr) { - CL_CHECK(clReleaseMemObject(scale)); - scale = nullptr; + for (ggml_tensor_extra_cl_iq2_xxs * e : temp_tensor_extras_iq2_xxs_in_use) { + delete e; } - if (min != nullptr) { - CL_CHECK(clReleaseMemObject(min)); - min = nullptr; + for (ggml_tensor_extra_cl_q2_0 * e : temp_tensor_extras_q2_0_in_use) { + delete e; } - - size_q = 0; - size_qh = 0; - size_s = 0; - size_d = 0; - size_dm = 0; - } -}; - -struct ggml_tensor_extra_cl_q6_K { - // Lower 4 bits of quantized weights. - cl_mem ql = nullptr; - // Lower 4 bits as image1d_buffer_t - cl_mem ql_img = nullptr; - // Upper 2 bits of quantized weights. - cl_mem qh = nullptr; - // Upper 2 bits as image1d_buffer_t - cl_mem qh_img = nullptr; - // Scales for each block. - cl_mem s = nullptr; - // Scales for each super block. - cl_mem d = nullptr; - - size_t size_ql = 0; - size_t size_qh = 0; - size_t size_s = 0; - size_t size_d = 0; - - ~ggml_tensor_extra_cl_q6_K() { - reset(); - } - - void reset() { - if (ql != nullptr) { - CL_CHECK(clReleaseMemObject(ql)); - ql = nullptr; + for (ggml_tensor_extra_cl_iq1_m * e : temp_tensor_extras_iq1_m_in_use) { + delete e; } - if (qh != nullptr) { - CL_CHECK(clReleaseMemObject(qh)); - qh = nullptr; + for (ggml_tensor_extra_cl_iq1_s * e : temp_tensor_extras_iq1_s_in_use) { + delete e; } - if (s != nullptr) { - CL_CHECK(clReleaseMemObject(s)); - s = nullptr; + for (ggml_tensor_extra_cl_iq3_s * e : temp_tensor_extras_iq3_s_in_use) { + delete e; } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; + for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs_in_use) { + delete e; } - if (ql_img != nullptr) { - CL_CHECK(clReleaseMemObject(ql_img)); - ql_img = nullptr; + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs_in_use) { + delete e; } - if (qh_img != nullptr) { - CL_CHECK(clReleaseMemObject(qh_img)); - qh_img = nullptr; + for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { + delete e; + } + for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { + delete e; } - - size_ql = 0; - size_qh = 0; - size_s = 0; - size_d = 0; } -}; -//------------------------------------------------------------------------------ -// Backend API -//------------------------------------------------------------------------------ - -// -// backend -// -static const char * ggml_backend_opencl_name(ggml_backend_t backend) { - return "OpenCL"; + ggml_tensor_extra_cl * ggml_opencl_alloc_temp_tensor_extra() { + ggml_tensor_extra_cl * extra; + if (temp_tensor_extras.empty()) { + extra = new ggml_tensor_extra_cl(); + } else { + extra = temp_tensor_extras.back(); + temp_tensor_extras.pop_back(); + } - UNUSED(backend); -} + temp_tensor_extras_in_use.push_back(extra); -static void ggml_backend_opencl_free(ggml_backend_t backend) { - ggml_cl_free(backend); -} + extra->reset(); + return extra; + } -static void ggml_backend_opencl_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - GGML_UNUSED(backend); - GGML_UNUSED(tensor); - GGML_UNUSED(data); - GGML_UNUSED(offset); - GGML_UNUSED(size); -} + ggml_tensor_extra_cl_q1_0 * ggml_opencl_alloc_temp_tensor_extra_q1_0() { + ggml_tensor_extra_cl_q1_0 * extra; + if (temp_tensor_extras_q1_0.empty()) { + extra = new ggml_tensor_extra_cl_q1_0(); + } else { + extra = temp_tensor_extras_q1_0.back(); + temp_tensor_extras_q1_0.pop_back(); + } -static void ggml_backend_opencl_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - GGML_UNUSED(backend); - GGML_UNUSED(tensor); - GGML_UNUSED(data); - GGML_UNUSED(offset); - GGML_UNUSED(size); -} + temp_tensor_extras_q1_0_in_use.push_back(extra); -static bool ggml_backend_opencl_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { - GGML_UNUSED(backend); - GGML_UNUSED(src); - GGML_UNUSED(dst); - return false; -} + extra->reset(); + return extra; + } -static void ggml_backend_opencl_synchronize(ggml_backend_t backend) { - auto * backend_ctx = static_cast<ggml_backend_opencl_context *>(backend->context); + ggml_tensor_extra_cl_q4_0 * ggml_opencl_alloc_temp_tensor_extra_q4_0() { + ggml_tensor_extra_cl_q4_0 * extra; + if (temp_tensor_extras_q4_0.empty()) { + extra = new ggml_tensor_extra_cl_q4_0(); + } else { + extra = temp_tensor_extras_q4_0.back(); + temp_tensor_extras_q4_0.pop_back(); + } - cl_event evt; - CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, 0, nullptr, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseEvent(evt)); -} + temp_tensor_extras_q4_0_in_use.push_back(extra); -// Synchronizes the 'backend_ctx's device with others so that commands -// enqueued to it won't start until commands in the other devices have -// completed. -static void sync_with_other_backends(ggml_backend_opencl_context * backend_ctx) { - if (g_ggml_backend_opencl_devices.size() < 2) { - return; // No other devices to synchronize with. + extra->reset(); + return extra; } - std::vector<cl_event> events; - events.reserve(g_ggml_backend_opencl_devices.size()); + ggml_tensor_extra_cl_q4_1 * ggml_opencl_alloc_temp_tensor_extra_q4_1() { + ggml_tensor_extra_cl_q4_1 * extra; + if (temp_tensor_extras_q4_1.empty()) { + extra = new ggml_tensor_extra_cl_q4_1(); + } else { + extra = temp_tensor_extras_q4_1.back(); + temp_tensor_extras_q4_1.pop_back(); + } - for (ggml_backend_device & backend_dev : g_ggml_backend_opencl_devices) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) backend_dev.context; - auto * other_backend_ctx = dev_ctx->backend_ctx; + temp_tensor_extras_q4_1_in_use.push_back(extra); - if (backend_ctx != other_backend_ctx) { - cl_event ev; - CL_CHECK(clEnqueueMarkerWithWaitList(other_backend_ctx->queue, 0, nullptr, &ev)); - CL_CHECK(clFlush(other_backend_ctx->queue)); - events.push_back(ev); - } + extra->reset(); + return extra; } - CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, events.size(), events.data(), nullptr)); - for (auto ev : events) { - CL_CHECK(clReleaseEvent(ev)); - } -} + ggml_tensor_extra_cl_q5_0 * ggml_opencl_alloc_temp_tensor_extra_q5_0() { + ggml_tensor_extra_cl_q5_0 * extra; + if (temp_tensor_extras_q5_0.empty()) { + extra = new ggml_tensor_extra_cl_q5_0(); + } else { + extra = temp_tensor_extras_q5_0.back(); + temp_tensor_extras_q5_0.pop_back(); + } -static void sync_with_other_backends(ggml_backend_t backend) { - auto * backend_ctx = static_cast<ggml_backend_opencl_context *>(backend->context); - sync_with_other_backends(backend_ctx); -} + temp_tensor_extras_q5_0_in_use.push_back(extra); -// True if two tensors share a device buffer with overlapping byte ranges. The pool -// allocator may place a fused op's output over a sequentially-dead input (safe for the -// original separate kernels, but a read/write race inside one fused kernel). -static bool ggml_cl_tensors_overlap(const ggml_tensor * x, const ggml_tensor * y) { - ggml_tensor_extra_cl * ex = (ggml_tensor_extra_cl *)x->extra; - ggml_tensor_extra_cl * ey = (ggml_tensor_extra_cl *)y->extra; - if (!ex || !ey || ex->data_device != ey->data_device) { return false; } - const cl_ulong xo = ex->offset + x->view_offs, xe = xo + ggml_nbytes(x); - const cl_ulong yo = ey->offset + y->view_offs, ye = yo + ggml_nbytes(y); - return xo < ye && yo < xe; -} + extra->reset(); + return extra; + } -// Detect the MoE combine epilogue: router-weight MUL ([n_embd,k,nt] * [1,k,nt]) followed -// by k VIEWs of it and a (k-1)-long ADD reduction chain producing [n_embd, nt]. When it -// matches (and the output does not alias the inputs), the whole subgraph collapses to one -// weighted-sum-across-experts kernel. -static bool ggml_opencl_can_fuse_moe_combine(const struct ggml_cgraph * cgraph, int node_idx, - const ggml_tensor ** out_final_add) { - const ggml_tensor * mul = cgraph->nodes[node_idx]; - if (mul->op != GGML_OP_MUL) { return false; } - const ggml_tensor * experts = mul->src[0]; - const ggml_tensor * weights = mul->src[1]; - if (!experts || !weights) { return false; } - if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { return false; } + ggml_tensor_extra_cl_q5_1 * ggml_opencl_alloc_temp_tensor_extra_q5_1() { + ggml_tensor_extra_cl_q5_1 * extra; + if (temp_tensor_extras_q5_1.empty()) { + extra = new ggml_tensor_extra_cl_q5_1(); + } else { + extra = temp_tensor_extras_q5_1.back(); + temp_tensor_extras_q5_1.pop_back(); + } - const int64_t n_embd = experts->ne[0]; - const int64_t k = experts->ne[1]; - const int64_t nt = experts->ne[2]; - if (k < 2 || k > 64 || experts->ne[3] != 1 || n_embd % 4 != 0) { return false; } - if (weights->ne[0] != 1 || weights->ne[1] != k || weights->ne[2] != nt || weights->ne[3] != 1) { return false; } - if (mul->ne[0] != n_embd || mul->ne[1] != k || mul->ne[2] != nt) { return false; } - // the fused kernel needs contiguous experts/weights and a contiguous 2D dst - if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights)) { return false; } + temp_tensor_extras_q5_1_in_use.push_back(extra); + + extra->reset(); + return extra; + } - const int n_nodes = 1 + (int)k + (int)(k - 1); // MUL + k*VIEW + (k-1)*ADD - if (n_nodes >= 32) { return false; } - if (node_idx + n_nodes > cgraph->n_nodes) { return false; } + ggml_tensor_extra_cl_mxfp4 * ggml_opencl_alloc_temp_tensor_extra_mxfp4() { + ggml_tensor_extra_cl_mxfp4 * extra; + if (temp_tensor_extras_mxfp4.empty()) { + extra = new ggml_tensor_extra_cl_mxfp4(); + } else { + extra = temp_tensor_extras_mxfp4.back(); + temp_tensor_extras_mxfp4.pop_back(); + } - enum ggml_op ops[1 + 64 + 63]; - int n = 0; - ops[n++] = GGML_OP_MUL; - for (int j = 0; j < (int)k; ++j) { ops[n++] = GGML_OP_VIEW; } - for (int j = 0; j < (int)k - 1; ++j) { ops[n++] = GGML_OP_ADD; } - const int outs[] = { node_idx + n_nodes - 1 }; - if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { return false; } + temp_tensor_extras_mxfp4_in_use.push_back(extra); - for (int j = 0; j < (int)k; ++j) { - const ggml_tensor * vw = cgraph->nodes[node_idx + 1 + j]; - if (vw->op != GGML_OP_VIEW || vw->src[0] != mul || vw->ne[0] != n_embd || vw->ne[1] != nt) { return false; } + extra->reset(); + return extra; } - const ggml_tensor * final_add = cgraph->nodes[node_idx + n_nodes - 1]; - if (final_add->op != GGML_OP_ADD || final_add->type != GGML_TYPE_F32 || - final_add->ne[0] != n_embd || final_add->ne[1] != nt || final_add->ne[2] != 1) { return false; } - if (!ggml_is_contiguous(final_add)) { return false; } - // the fused kernel reads experts + writes final_add in one pass; bail if the - // pool allocator overlapped the output with the (large) experts input -- would race. - // The small weights input is copied to a private scratch in the dispatch, so its own - // aliasing with the output is handled there and does not block the fusion. - if (ggml_cl_tensors_overlap(experts, final_add)) { return false; } - *out_final_add = final_add; - return true; -} + ggml_tensor_extra_cl_q8_0 * ggml_opencl_alloc_temp_tensor_extra_q8_0() { + ggml_tensor_extra_cl_q8_0 * extra; + if (temp_tensor_extras_q8_0.empty()) { + extra = new ggml_tensor_extra_cl_q8_0(); + } else { + extra = temp_tensor_extras_q8_0.back(); + temp_tensor_extras_q8_0.pop_back(); + } -// Detect the gpt-oss MoE bias+activation epilogue on the PREFILL path: -// {MUL_MAT_ID(gate), ADD_ID(gate_bias), MUL_MAT_ID(up), ADD_ID(up_bias), GLU(swiglu_oai)}. -// The two matmuls still run as their own dispatches (the prefill GEMM is the vendor's); -// what collapses is the epilogue — both add_id passes are in-place read-modify-writes of a -// tensor the GLU immediately reads again, so they are three full passes over the same -// [n_ff, n_expert_used, n_tokens] f32 tensor where one suffices. -// -// The decode counterpart is handled by the mxfp4 fused GEMV arm in ggml_opencl_can_fuse, -// which folds the matmul too; this one deliberately fires only when that cannot (ne[2] > 1). -static bool ggml_opencl_can_fuse_moe_bias_glu(const struct ggml_cgraph * cgraph, int node_idx) { - if (node_idx + 4 >= cgraph->n_nodes) { - return false; - } + temp_tensor_extras_q8_0_in_use.push_back(extra); - const enum ggml_op mg_ops[] = { GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_GLU }; - const int mg_out[] = { node_idx + 4 }; - if (!ggml_can_fuse_subgraph(cgraph, node_idx, 5, mg_ops, mg_out, 1)) { - return false; + extra->reset(); + return extra; } - const ggml_tensor * gmm = cgraph->nodes[node_idx]; - const ggml_tensor * gad = cgraph->nodes[node_idx+1]; - const ggml_tensor * umm = cgraph->nodes[node_idx+2]; - const ggml_tensor * uad = cgraph->nodes[node_idx+3]; - const ggml_tensor * glu = cgraph->nodes[node_idx+4]; + ggml_tensor_extra_cl_iq4_nl * ggml_opencl_alloc_temp_tensor_extra_iq4_nl() { + ggml_tensor_extra_cl_iq4_nl * extra; + if (temp_tensor_extras_iq4_nl.empty()) { + extra = new ggml_tensor_extra_cl_iq4_nl(); + } else { + extra = temp_tensor_extras_iq4_nl.back(); + temp_tensor_extras_iq4_nl.pop_back(); + } - if (ggml_get_glu_op(glu) != GGML_GLU_OP_SWIGLU_OAI) { - return false; - } - // Prefill only — at one token the mxfp4 arm above folds the matmul as well. - if (gmm->src[1]->ne[2] == 1) { - return false; - } - // Wiring: both matmuls share the activation and the expert selection, each add_id - // biases its own matmul, and the GLU consumes the two biased results as separate - // operands (so the same-buffer ne00_off/ne10_off split path is not in play). - if (gad->src[0] != gmm || uad->src[0] != umm || - glu->src[0] != gad || glu->src[1] != uad || - umm->src[1] != gmm->src[1] || umm->src[2] != gmm->src[2]) { - return false; - } - // A swapped GLU would exchange the gate/up roles the fused kernel hard-codes. - if (ggml_get_op_params_i32(glu, 1)) { - return false; - } - if (gad->type != GGML_TYPE_F32 || uad->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { - return false; - } - if (!gad->src[1] || gad->src[1]->type != GGML_TYPE_F32 || - !uad->src[1] || uad->src[1]->type != GGML_TYPE_F32) { - return false; - } - if (!gad->src[2] || gad->src[2]->type != GGML_TYPE_I32 || uad->src[2] != gad->src[2]) { - return false; - } - // Full width on both operands: the kernel writes one output element per input pair. - if (!ggml_are_same_shape(gad, uad) || glu->ne[0] != gad->ne[0] || - glu->ne[1] != gad->ne[1] || glu->ne[2] != gad->ne[2] || glu->ne[3] != gad->ne[3]) { - return false; - } - if (gad->ne[3] != 1) { - return false; - } - // The destination is addressed by (expert slot, token) rather than the GLU's flat row - // walk; those agree only for a contiguous destination. - if (!ggml_is_contiguous(glu) || !ggml_is_contiguous(gmm) || !ggml_is_contiguous(umm)) { - return false; - } - return true; -} + temp_tensor_extras_iq4_nl_in_use.push_back(extra); -static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + extra->reset(); + return extra; + } -// Runs the gate and up matmuls unchanged, then one kernel in place of -// add_id(gate) + add_id(up) + swiglu_oai. See ggml_opencl_can_fuse_moe_bias_glu. -static void ggml_cl_moe_bias_glu_fused(ggml_backend_t backend, ggml_tensor * gate_mm, const ggml_tensor * gate_add, - ggml_tensor * up_mm, const ggml_tensor * up_add, const ggml_tensor * glu) { - ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q4_K * ggml_opencl_alloc_temp_tensor_extra_q4_K() { + ggml_tensor_extra_cl_q4_K * extra; + if (temp_tensor_extras_q4_K.empty()) { + extra = new ggml_tensor_extra_cl_q4_K(); + } else { + extra = temp_tensor_extras_q4_K.back(); + temp_tensor_extras_q4_K.pop_back(); + } - ggml_cl_mul_mat_id(backend, gate_mm->src[0], gate_mm->src[1], gate_mm); - ggml_cl_mul_mat_id(backend, up_mm->src[0], up_mm->src[1], up_mm); + temp_tensor_extras_q4_K_in_use.push_back(extra); - const ggml_tensor * gbias = gate_add->src[1]; - const ggml_tensor * ubias = up_add->src[1]; - const ggml_tensor * ids = gate_add->src[2]; + extra->reset(); + return extra; + } - ggml_tensor_extra_cl * eg = (ggml_tensor_extra_cl *)gate_mm->extra; - ggml_tensor_extra_cl * egb = (ggml_tensor_extra_cl *)gbias->extra; - ggml_tensor_extra_cl * eu = (ggml_tensor_extra_cl *)up_mm->extra; - ggml_tensor_extra_cl * eub = (ggml_tensor_extra_cl *)ubias->extra; - ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra; - ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)glu->extra; + ggml_tensor_extra_cl_q5_K * ggml_opencl_alloc_temp_tensor_extra_q5_K() { + ggml_tensor_extra_cl_q5_K * extra; + if (temp_tensor_extras_q5_K.empty()) { + extra = new ggml_tensor_extra_cl_q5_K(); + } else { + extra = temp_tensor_extras_q5_K.back(); + temp_tensor_extras_q5_K.pop_back(); + } - cl_ulong off_g = eg->offset + gate_mm->view_offs; - cl_ulong off_gb = egb->offset + gbias->view_offs; - cl_ulong off_u = eu->offset + up_mm->view_offs; - cl_ulong off_ub = eub->offset + ubias->view_offs; - cl_ulong off_i = ei->offset + ids->view_offs; - cl_ulong off_d = ed->offset + glu->view_offs; + temp_tensor_extras_q5_K_in_use.push_back(extra); - const cl_ulong nb01_g = gate_mm->nb[1]; - const cl_ulong nb02_g = gate_mm->nb[2]; - const cl_ulong nb01_u = up_mm->nb[1]; - const cl_ulong nb02_u = up_mm->nb[2]; - const cl_ulong nb11_g = gbias->nb[1]; - const cl_ulong nb11_u = ubias->nb[1]; - const cl_ulong nb21 = ids->nb[1]; - const cl_ulong nbd1 = glu->nb[1]; - const cl_ulong nbd2 = glu->nb[2]; + extra->reset(); + return extra; + } - const int ne0 = (int)glu->ne[0]; - const float alpha = ggml_get_op_params_f32(glu, 2); - const float limit = ggml_get_op_params_f32(glu, 3); + ggml_tensor_extra_cl_q2_K * ggml_opencl_alloc_temp_tensor_extra_q2_K() { + ggml_tensor_extra_cl_q2_K * extra; + if (temp_tensor_extras_q2_K.empty()) { + extra = new ggml_tensor_extra_cl_q2_K(); + } else { + extra = temp_tensor_extras_q2_K.back(); + temp_tensor_extras_q2_K.pop_back(); + } - cl_kernel kernel = backend_ctx->kernel_add_id_add_id_swiglu_oai; + temp_tensor_extras_q2_K_in_use.push_back(extra); - int i = 0; - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eg->data_device)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_g)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &egb->data_device)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_gb)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eu->data_device)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_u)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eub->data_device)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_ub)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &ei->data_device)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_i)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &ed->data_device)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_d)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_g)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_g)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_u)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_u)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_g)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_u)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb21)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd1)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd2)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float), &limit)); - CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float), &alpha)); + extra->reset(); + return extra; + } - const int nth = MIN(ne0, (int) backend_ctx->get_kernel_workgroup_size(kernel)); - size_t global_work_size[] = { (size_t)glu->ne[1]*nth, (size_t)glu->ne[2], 1 }; - size_t local_work_size[] = { (size_t)nth, 1, 1 }; + ggml_tensor_extra_cl_q3_K * ggml_opencl_alloc_temp_tensor_extra_q3_K() { + ggml_tensor_extra_cl_q3_K * extra; + if (temp_tensor_extras_q3_K.empty()) { + extra = new ggml_tensor_extra_cl_q3_K(); + } else { + extra = temp_tensor_extras_q3_K.back(); + temp_tensor_extras_q3_K.pop_back(); + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, glu); -} + temp_tensor_extras_q3_K_in_use.push_back(extra); -// Fusion B: the MoE down-projection bias add feeding the combine. -// -// The graph runs ADD_ID(down_bias) and then immediately the combine subgraph -// {MUL(router weights), k VIEWs, k-1 ADDs}, and the ADD_ID's only consumer is that -// MUL. Since the ADD_ID is an in-place read-modify-write of a tensor the combine -// reads once more, the bias can be added inside the combine instead, dropping a -// full pass over [n_embd, k, n_tokens]. -// -// Shape checks for the combine tail are delegated to ggml_opencl_can_fuse_moe_combine -// (which also owns the n_nodes >= 32 bail and the experts/dst aliasing bail); what is -// added here is the ADD_ID wiring plus a subgraph check over the WHOLE run, so that -// the intermediate bias result is confirmed not to escape. -static bool ggml_opencl_can_fuse_moe_bias_combine(const struct ggml_cgraph * cgraph, int node_idx, - const ggml_tensor ** out_final_add) { - if (node_idx + 1 >= cgraph->n_nodes) { - return false; - } - const ggml_tensor * add = cgraph->nodes[node_idx]; - if (add->op != GGML_OP_ADD_ID) { - return false; - } - const ggml_tensor * mul = cgraph->nodes[node_idx+1]; - if (mul->op != GGML_OP_MUL || mul->src[0] != add) { - return false; + extra->reset(); + return extra; } - const ggml_tensor * final_add = NULL; - if (!ggml_opencl_can_fuse_moe_combine(cgraph, node_idx+1, &final_add)) { - return false; + ggml_tensor_extra_cl_iq2_s * ggml_opencl_alloc_temp_tensor_extra_iq2_s() { + ggml_tensor_extra_cl_iq2_s * extra; + if (temp_tensor_extras_iq2_s.empty()) { + extra = new ggml_tensor_extra_cl_iq2_s(); + } else { + extra = temp_tensor_extras_iq2_s.back(); + temp_tensor_extras_iq2_s.pop_back(); + } + + temp_tensor_extras_iq2_s_in_use.push_back(extra); + + extra->reset(); + return extra; } - const ggml_tensor * raw = add->src[0]; - const ggml_tensor * bias = add->src[1]; - const ggml_tensor * ids = add->src[2]; - if (!raw || !bias || !ids) { - return false; + ggml_tensor_extra_cl_iq2_xs * ggml_opencl_alloc_temp_tensor_extra_iq2_xs() { + ggml_tensor_extra_cl_iq2_xs * extra; + if (temp_tensor_extras_iq2_xs.empty()) { + extra = new ggml_tensor_extra_cl_iq2_xs(); + } else { + extra = temp_tensor_extras_iq2_xs.back(); + temp_tensor_extras_iq2_xs.pop_back(); + } + + temp_tensor_extras_iq2_xs_in_use.push_back(extra); + + extra->reset(); + return extra; } - if (raw->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 || - ids->type != GGML_TYPE_I32 || add->type != GGML_TYPE_F32) { - return false; + + ggml_tensor_extra_cl_iq2_xxs * ggml_opencl_alloc_temp_tensor_extra_iq2_xxs() { + ggml_tensor_extra_cl_iq2_xxs * extra; + if (temp_tensor_extras_iq2_xxs.empty()) { + extra = new ggml_tensor_extra_cl_iq2_xxs(); + } else { + extra = temp_tensor_extras_iq2_xxs.back(); + temp_tensor_extras_iq2_xxs.pop_back(); + } + + temp_tensor_extras_iq2_xxs_in_use.push_back(extra); + + extra->reset(); + return extra; } - // The combine reads the raw matmul output with the strides it computed from the - // add_id result, so the two must have the same layout. - if (!ggml_are_same_shape(raw, add) || !ggml_is_contiguous(raw)) { - return false; + + ggml_tensor_extra_cl_q2_0 * ggml_opencl_alloc_temp_tensor_extra_q2_0() { + ggml_tensor_extra_cl_q2_0 * extra; + if (temp_tensor_extras_q2_0.empty()) { + extra = new ggml_tensor_extra_cl_q2_0(); + } else { + extra = temp_tensor_extras_q2_0.back(); + temp_tensor_extras_q2_0.pop_back(); + } + + temp_tensor_extras_q2_0_in_use.push_back(extra); + + extra->reset(); + return extra; } - if (raw->nb[1] != add->nb[1] || raw->nb[2] != add->nb[2]) { - return false; + + ggml_tensor_extra_cl_iq1_m * ggml_opencl_alloc_temp_tensor_extra_iq1_m() { + ggml_tensor_extra_cl_iq1_m * extra; + if (temp_tensor_extras_iq1_m.empty()) { + extra = new ggml_tensor_extra_cl_iq1_m(); + } else { + extra = temp_tensor_extras_iq1_m.back(); + temp_tensor_extras_iq1_m.pop_back(); + } + + temp_tensor_extras_iq1_m_in_use.push_back(extra); + + extra->reset(); + return extra; } - // ids is indexed as [expert slot, token]; the combine walks the same two axes. - if (ids->ne[0] < add->ne[1] || ids->ne[1] < add->ne[2]) { - return false; + + ggml_tensor_extra_cl_iq1_s * ggml_opencl_alloc_temp_tensor_extra_iq1_s() { + ggml_tensor_extra_cl_iq1_s * extra; + if (temp_tensor_extras_iq1_s.empty()) { + extra = new ggml_tensor_extra_cl_iq1_s(); + } else { + extra = temp_tensor_extras_iq1_s.back(); + temp_tensor_extras_iq1_s.pop_back(); + } + + temp_tensor_extras_iq1_s_in_use.push_back(extra); + + extra->reset(); + return extra; } - // Whole-run escape check: ADD_ID + MUL + k VIEWs + (k-1) ADDs, only the last node escapes. - const int k = (int)add->ne[1]; - const int n_nodes = 2 + k + (k - 1); - if (n_nodes >= 32 || node_idx + n_nodes > cgraph->n_nodes) { - return false; + ggml_tensor_extra_cl_iq3_s * ggml_opencl_alloc_temp_tensor_extra_iq3_s() { + ggml_tensor_extra_cl_iq3_s * extra; + if (temp_tensor_extras_iq3_s.empty()) { + extra = new ggml_tensor_extra_cl_iq3_s(); + } else { + extra = temp_tensor_extras_iq3_s.back(); + temp_tensor_extras_iq3_s.pop_back(); + } + + temp_tensor_extras_iq3_s_in_use.push_back(extra); + + extra->reset(); + return extra; } - enum ggml_op ops[32]; - int n = 0; - ops[n++] = GGML_OP_ADD_ID; - ops[n++] = GGML_OP_MUL; - for (int j = 0; j < k; ++j) ops[n++] = GGML_OP_VIEW; - for (int j = 0; j < k - 1; ++j) ops[n++] = GGML_OP_ADD; - const int outs[] = { node_idx + n_nodes - 1 }; - if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { - return false; + + ggml_tensor_extra_cl_iq3_xxs * ggml_opencl_alloc_temp_tensor_extra_iq3_xxs() { + ggml_tensor_extra_cl_iq3_xxs * extra; + if (temp_tensor_extras_iq3_xxs.empty()) { + extra = new ggml_tensor_extra_cl_iq3_xxs(); + } else { + extra = temp_tensor_extras_iq3_xxs.back(); + temp_tensor_extras_iq3_xxs.pop_back(); + } + + temp_tensor_extras_iq3_xxs_in_use.push_back(extra); + extra->reset(); + return extra; } - *out_final_add = final_add; - return true; -} + ggml_tensor_extra_cl_iq4_xs * ggml_opencl_alloc_temp_tensor_extra_iq4_xs() { + ggml_tensor_extra_cl_iq4_xs * extra; + if (temp_tensor_extras_iq4_xs.empty()) { + extra = new ggml_tensor_extra_cl_iq4_xs(); + } else { + extra = temp_tensor_extras_iq4_xs.back(); + temp_tensor_extras_iq4_xs.pop_back(); + } + temp_tensor_extras_iq4_xs_in_use.push_back(extra); -// Fusion B dispatch: the combine, reading the RAW matmul output and adding the -// per-expert bias row inline. See ggml_opencl_can_fuse_moe_bias_combine. -static void ggml_cl_moe_bias_combine_fused(ggml_backend_t backend, const ggml_tensor * add, - const ggml_tensor * mul, const ggml_tensor * dst) { - ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + extra->reset(); + return extra; + } - const ggml_tensor * experts = add->src[0]; // raw matmul output, bias not yet applied - const ggml_tensor * bias = add->src[1]; - const ggml_tensor * ids = add->src[2]; - const ggml_tensor * weights = mul->src[1]; + ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() { + ggml_tensor_extra_cl_q6_K * extra; + if (temp_tensor_extras_q6_K.empty()) { + extra = new ggml_tensor_extra_cl_q6_K(); + } else { + extra = temp_tensor_extras_q6_K.back(); + temp_tensor_extras_q6_K.pop_back(); + } - ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; - ggml_tensor_extra_cl * eb = (ggml_tensor_extra_cl *)bias->extra; - ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra; - ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; - ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; - cl_ulong off_e = ee->offset + experts->view_offs; - cl_ulong off_b = eb->offset + bias->view_offs; - cl_ulong off_i = ei->offset + ids->view_offs; - cl_ulong off_w = ew->offset + weights->view_offs; - cl_ulong off_d = ed->offset + dst->view_offs; + temp_tensor_extras_q6_K_in_use.push_back(extra); - const int n_embd4 = (int)(experts->ne[0] / 4); - const int k = (int)experts->ne[1]; - const int nt = (int)experts->ne[2]; - const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); - const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); - const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); - const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); - const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); - const cl_ulong nb_b1 = bias->nb[1]; - const cl_ulong nb_i1 = ids->nb[1]; + extra->reset(); + return extra; + } - const size_t w_bytes = ggml_nbytes(weights); - backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, - off_w, 0, w_bytes, 0, NULL, NULL)); - cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; - cl_ulong w_off = 0; + void reset() { + for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { + temp_tensor_extras.push_back(e); + } + temp_tensor_extras_in_use.clear(); - cl_kernel kernel = backend_ctx->kernel_moe_combine_bias_f32; - int a = 0; - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &eb->data_device)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_b)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ei->data_device)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_i)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_b1)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_i1)); + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { + temp_tensor_extras_q1_0.push_back(e); + } + temp_tensor_extras_q1_0_in_use.clear(); - size_t lws[2] = { 64, 1 }; - size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); -} + for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { + temp_tensor_extras_q4_0.push_back(e); + } + temp_tensor_extras_q4_0_in_use.clear(); + for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { + temp_tensor_extras_q4_1.push_back(e); + } + temp_tensor_extras_q4_1_in_use.clear(); -static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor * mul, const ggml_tensor * dst) { - ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; - const ggml_tensor * experts = mul->src[0]; - const ggml_tensor * weights = mul->src[1]; + for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0_in_use) { + temp_tensor_extras_q5_0.push_back(e); + } + temp_tensor_extras_q5_0_in_use.clear(); - ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; - ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; - ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; - cl_ulong off_e = ee->offset + experts->view_offs; - cl_ulong off_w = ew->offset + weights->view_offs; - cl_ulong off_d = ed->offset + dst->view_offs; + for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1_in_use) { + temp_tensor_extras_q5_1.push_back(e); + } + temp_tensor_extras_q5_1_in_use.clear(); - const int n_embd4 = (int)(experts->ne[0] / 4); - const int k = (int)experts->ne[1]; - const int nt = (int)experts->ne[2]; - const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); - const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); - const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); - const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); - const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); + for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { + temp_tensor_extras_mxfp4.push_back(e); + } + temp_tensor_extras_mxfp4_in_use.clear(); - // The router weights are tiny ([1,k,nt]) and may share a pool buffer with the output; - // copy them into a private scratch so the fused kernel never reads aliased memory. - const size_t w_bytes = ggml_nbytes(weights); - backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, - off_w, 0, w_bytes, 0, NULL, NULL)); - cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; - cl_ulong w_off = 0; + for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { + temp_tensor_extras_q8_0.push_back(e); + } + temp_tensor_extras_q8_0_in_use.clear(); - cl_kernel kernel = backend_ctx->kernel_moe_combine_f32; - int a = 0; - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); - CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); + for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl_in_use) { + temp_tensor_extras_iq4_nl.push_back(e); + } + temp_tensor_extras_iq4_nl_in_use.clear(); - size_t lws[2] = { 64, 1 }; - size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); -} + for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { + temp_tensor_extras_q4_K.push_back(e); + } + temp_tensor_extras_q4_K_in_use.clear(); + + for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { + temp_tensor_extras_q5_K.push_back(e); + } + temp_tensor_extras_q5_K_in_use.clear(); + + for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { + temp_tensor_extras_q6_K.push_back(e); + } + temp_tensor_extras_q6_K_in_use.clear(); -inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check) -inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); -inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below + for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) { + temp_tensor_extras_q2_K.push_back(e); + } + temp_tensor_extras_q2_K_in_use.clear(); -static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) { + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) { + temp_tensor_extras_q3_K.push_back(e); + } + temp_tensor_extras_q3_K_in_use.clear(); - // glu(mul_mat(Wg,x), mul_mat(Wu,x)) — the FFN gate/up GEMVs + GLU. This is a - // non-linear subgraph (up does NOT consume gate), so the contiguous - // ggml_can_fuse below rejects it; use ggml_can_fuse_subgraph with the glu as - // the sole output and validate the edges explicitly. q4_K decode only; - // byte-identical to the per-op path. - if (ops.size() == 3 && ops.begin()[0] == GGML_OP_MUL_MAT && - ops.begin()[1] == GGML_OP_MUL_MAT && ops.begin()[2] == GGML_OP_GLU) { - const enum ggml_op glu_ops[] = { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }; - const int glu_out[] = { node_idx + 2 }; - if (!ggml_can_fuse_subgraph(cgraph, node_idx, 3, glu_ops, glu_out, 1)) { - return false; + for (ggml_tensor_extra_cl_iq2_s * e : temp_tensor_extras_iq2_s_in_use) { + temp_tensor_extras_iq2_s.push_back(e); } + temp_tensor_extras_iq2_s_in_use.clear(); - const ggml_tensor *gate = cgraph->nodes[node_idx]; - const ggml_tensor *up = cgraph->nodes[node_idx+1]; - const ggml_tensor *glu = cgraph->nodes[node_idx+2]; + for (ggml_tensor_extra_cl_iq2_xs * e : temp_tensor_extras_iq2_xs_in_use) { + temp_tensor_extras_iq2_xs.push_back(e); + } + temp_tensor_extras_iq2_xs_in_use.clear(); - // decode GEMV path only (single token); prefill GEMM is separate - if (gate->ne[1] != 1 || up->ne[1] != 1) { - return false; + for (ggml_tensor_extra_cl_iq2_xxs * e : temp_tensor_extras_iq2_xxs_in_use) { + temp_tensor_extras_iq2_xxs.push_back(e); } - // both projections must be q4_K weights, f32 activation/output - if (gate->src[0]->type != GGML_TYPE_Q4_K || up->src[0]->type != GGML_TYPE_Q4_K || - gate->src[1]->type != GGML_TYPE_F32 || up->src[1]->type != GGML_TYPE_F32 || - gate->type != GGML_TYPE_F32 || up->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { - return false; + temp_tensor_extras_iq2_xxs_in_use.clear(); + + for (ggml_tensor_extra_cl_q2_0 * e : temp_tensor_extras_q2_0_in_use) { + temp_tensor_extras_q2_0.push_back(e); } - // gate and up must share the same activation and have matching shape/stride - if (gate->src[1] != up->src[1] || - !ggml_are_same_shape(gate->src[0], up->src[0]) || - !ggml_are_same_stride(gate->src[0], up->src[0])) { - return false; + temp_tensor_extras_q2_0_in_use.clear(); + + for (ggml_tensor_extra_cl_iq1_m * e : temp_tensor_extras_iq1_m_in_use) { + temp_tensor_extras_iq1_m.push_back(e); } - // GLU must read gate as src[0] and up as src[1], no swap (the fused - // epilogue applies the activation to gate, multiplies by up) - if (glu->src[0] != gate || glu->src[1] != up) { - return false; + temp_tensor_extras_iq1_m_in_use.clear(); + + for (ggml_tensor_extra_cl_iq1_s * e : temp_tensor_extras_iq1_s_in_use) { + temp_tensor_extras_iq1_s.push_back(e); } - if (ggml_get_op_params_i32(glu, 1) /* swapped */) { - return false; + temp_tensor_extras_iq1_s_in_use.clear(); + + for (ggml_tensor_extra_cl_iq3_s * e : temp_tensor_extras_iq3_s_in_use) { + temp_tensor_extras_iq3_s.push_back(e); } - // SWIGLU_OAI carries extra alpha/limit params -> not handled by the fused kernel - if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI) { - return false; + temp_tensor_extras_iq3_s_in_use.clear(); + + for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs_in_use) { + temp_tensor_extras_iq3_xxs.push_back(e); } - // the fused kernel reads the standard noshuffle image layout; the tiled - // layout packs weights differently -> defer those to the per-op path - if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) { - return false; + temp_tensor_extras_iq3_xxs_in_use.clear(); + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs_in_use) { + temp_tensor_extras_iq4_xs.push_back(e); } - // q4_K bin kernel requires 32b transposed layout, not compatible with the fused gemv - if (use_q4_k_bin_kernels(backend_ctx, gate->src[0]) || use_q4_k_bin_kernels(backend_ctx, up->src[0])) { - return false; + temp_tensor_extras_iq4_xs_in_use.clear(); + + q8_0_soa_tensors.clear(); + q4_0_soa_tensors.clear(); + } + + // Pools for extras. Available extras are in `temp_tensor_extras`. Extras + // being used are in `temp_tensor_extras_in_use`. At the first run, new + // extras get created and put in `in_use`. When the buffer is reset via + // the `reset` callback, all extras in `in_use` get moved to available extras + // for reuse. + std::vector<ggml_tensor_extra_cl *> temp_tensor_extras; + std::vector<ggml_tensor_extra_cl *> temp_tensor_extras_in_use; + std::vector<ggml_tensor_extra_cl_q1_0 *> temp_tensor_extras_q1_0; + std::vector<ggml_tensor_extra_cl_q1_0 *> temp_tensor_extras_q1_0_in_use; + std::vector<ggml_tensor_extra_cl_q4_0 *> temp_tensor_extras_q4_0; + std::vector<ggml_tensor_extra_cl_q4_0 *> temp_tensor_extras_q4_0_in_use; + std::vector<ggml_tensor_extra_cl_q4_1 *> temp_tensor_extras_q4_1; + std::vector<ggml_tensor_extra_cl_q4_1 *> temp_tensor_extras_q4_1_in_use; + std::vector<ggml_tensor_extra_cl_q5_0 *> temp_tensor_extras_q5_0; + std::vector<ggml_tensor_extra_cl_q5_0 *> temp_tensor_extras_q5_0_in_use; + std::vector<ggml_tensor_extra_cl_q5_1 *> temp_tensor_extras_q5_1; + std::vector<ggml_tensor_extra_cl_q5_1 *> temp_tensor_extras_q5_1_in_use; + std::vector<ggml_tensor_extra_cl_mxfp4 *> temp_tensor_extras_mxfp4; + std::vector<ggml_tensor_extra_cl_mxfp4 *> temp_tensor_extras_mxfp4_in_use; + std::vector<ggml_tensor_extra_cl_q8_0 *> temp_tensor_extras_q8_0; + std::vector<ggml_tensor_extra_cl_q8_0 *> temp_tensor_extras_q8_0_in_use; + std::vector<ggml_tensor_extra_cl_iq4_nl *> temp_tensor_extras_iq4_nl; + std::vector<ggml_tensor_extra_cl_iq4_nl *> temp_tensor_extras_iq4_nl_in_use; + std::vector<ggml_tensor_extra_cl_q4_K *> temp_tensor_extras_q4_K; + std::vector<ggml_tensor_extra_cl_q4_K *> temp_tensor_extras_q4_K_in_use; + std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K; + std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K_in_use; + std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K; + std::vector<ggml_tensor_extra_cl_q3_K *> temp_tensor_extras_q3_K; + std::vector<ggml_tensor_extra_cl_iq2_s *> temp_tensor_extras_iq2_s; + std::vector<ggml_tensor_extra_cl_iq2_xs *> temp_tensor_extras_iq2_xs; + std::vector<ggml_tensor_extra_cl_iq2_xxs *> temp_tensor_extras_iq2_xxs; + std::vector<ggml_tensor_extra_cl_iq3_xxs *> temp_tensor_extras_iq3_xxs; + std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K_in_use; + std::vector<ggml_tensor_extra_cl_q3_K *> temp_tensor_extras_q3_K_in_use; + std::vector<ggml_tensor_extra_cl_iq2_s *> temp_tensor_extras_iq2_s_in_use; + std::vector<ggml_tensor_extra_cl_iq2_xs *> temp_tensor_extras_iq2_xs_in_use; + std::vector<ggml_tensor_extra_cl_iq2_xxs *> temp_tensor_extras_iq2_xxs_in_use; + std::vector<ggml_tensor_extra_cl_q2_0 *> temp_tensor_extras_q2_0; + std::vector<ggml_tensor_extra_cl_q2_0 *> temp_tensor_extras_q2_0_in_use; + std::vector<ggml_tensor_extra_cl_iq1_s *> temp_tensor_extras_iq1_s; + std::vector<ggml_tensor_extra_cl_iq1_m *> temp_tensor_extras_iq1_m; + std::vector<ggml_tensor_extra_cl_iq1_m *> temp_tensor_extras_iq1_m_in_use; + std::vector<ggml_tensor_extra_cl_iq1_s *> temp_tensor_extras_iq1_s_in_use; + std::vector<ggml_tensor_extra_cl_iq3_s *> temp_tensor_extras_iq3_s; + std::vector<ggml_tensor_extra_cl_iq3_s *> temp_tensor_extras_iq3_s_in_use; + std::vector<ggml_tensor_extra_cl_iq3_xxs *> temp_tensor_extras_iq3_xxs_in_use; + std::vector<ggml_tensor_extra_cl_iq4_xs *> temp_tensor_extras_iq4_xs; + std::vector<ggml_tensor_extra_cl_iq4_xs *> temp_tensor_extras_iq4_xs_in_use; + std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K; + std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K_in_use; + + // q8_0 tensors with AoS->SoA layout conversion installed by set_tensor. + // Two types of tensors get SOA'ed - normal weights and MoE weights. + // In Q8_0's case, we only have normal weights. If we ever have Q8_0 as MoE + // weights, they need to be added to this set in `set_tensors`. + std::unordered_set<const ggml_tensor *> q8_0_soa_tensors; + + // Same for q4_0. KV-cache q4_0 tensors are allocated but never pass + // through set_tensor, so they stay AoS and aren't in this set. + // In Q4_0's case, in addition to normal weights, we have MoE weights. + std::unordered_set<const ggml_tensor *> q4_0_soa_tensors; + + // The buffer_context is initially created by ggml_backend_buft_alloc_buffer + // before any tensor is initialized (at the beginning of alloc_tensor_range). + // Hence, there is always a buffer object in this vector. When each tensor is + // being initialized, this original buffer object will be released if both + // flattening and small allocation are enabled, and additional buffer + // objects will be created in init_tensor to represent flattened quantized + // weights. + std::vector<cl_mem> buffer; + // These are image1d_buffer_t objects that wrap around the quants and scales. + // For Q4_0 quantization, there should be two of them - one for quants and + // one for scales. They should be populated only when flattening and small + // allocation are enabled. + std::vector<cl_mem> img; + std::string name; +}; + +static void ggml_backend_opencl_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + delete ctx; +} + +static void * ggml_backend_opencl_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; + return (void *) (uintptr_t) dev_ctx->backend_ctx->alignment; +} + +static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + + if (tensor->view_src != nullptr) { + GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); + + ggml_tensor_extra_cl * view_extra = (ggml_tensor_extra_cl *) tensor->view_src->extra; + GGML_ASSERT(view_extra && "view_extra is nullptr?"); + + // Reuse extra of the parent tensor. The offset of this view tensor + // becomes `extra->offset + view_offs` and needs to be calculated when + // it is used. This changes is needed because of the change to + // ggml_alloc.c in https://github.com/ggml-org/llama.cpp/pull/7640. + // `buffer` passed in here will always be `tensor->buffer`. It is OK + // to allocate extras from the same buffer context for ordinary + // intermediate tensors. But for views into kv cache tensors, doing so + // would mess up the extras used by kv cache. + // Before #7640, `buffer` is for intermediate tensors, which is always + // different from that of kv cache tensors. + // + // NB: now extra->offset no longer accounts for view_offs. + // NB: this should not apply to weight tensors (for end-to-end runs, but + // may apply for test-backend-ops). + // FIXME: if any unexpected results are seen, double check the offset - + // there could be other places that need fix. + tensor->extra = view_extra; + } else { + { + size_t offset = (char *) tensor->data - (char *) ggml_backend_opencl_buffer_get_base(buffer); + + ggml_tensor_extra_cl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra(); + extra->offset = offset; + extra->data_device = ctx->buffer[0]; + extra->actual_size = ggml_nbytes(tensor); + + tensor->extra = extra; } - // that noshuffle layout is only produced at set_tensor time when - // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512). - // Smaller weights stay in the plain q4_K layout, which this kernel would - // misread -> defer them to the per-op path. Real FFN gate/up weights are - // far above the threshold, so production dispatch is unchanged. - if (!use_adreno_kernels(backend_ctx, gate->src[0]) || - !use_adreno_kernels(backend_ctx, up->src[0])) { - return false; + } + return GGML_STATUS_SUCCESS; +} + +// Allocate a temporary upload buffer of `nbytes` and populate it with `data` +// from host. On Adreno X1-85 the device-only pool intermittently fails to +// allocate at hundreds of MB once model weights fragment the heap (observed +// on Qwen3.5-9B output.weight Q6_K at 834 MB). Three-step retry: +// 1. CL_MEM_READ_WRITE alloc + clEnqueueWriteBuffer (normal fast path). +// 2. clFinish + retry (drains in-flight allocs that may be holding heap; +// mirrors the proven pattern at the FD-split partial buffer alloc). +// 3. CL_MEM_ALLOC_HOST_PTR + map(WRITE_INVALIDATE) + memcpy + unmap — +// different memory pool (host-pinned); true zero-copy on Adreno per +// QCOM guidance. (CL_MEM_USE_HOST_PTR is NOT zero-copy on Adreno: the +// driver triggers an internal copy because arbitrary host pages aren't +// guaranteed mappable/coherent, AND it draws from the same exhausted +// device pool — so it doesn't solve the problem.) +// Returns the ready-to-read buffer (caller must clReleaseMemObject) or NULL +// if all three strategies fail. The buffer is opaque to the caller — it can +// be passed as a kernel argument like any normal cl_mem. +static cl_mem ggml_cl_create_temp_upload_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const void * data, + const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + const cl_int werr = clEnqueueWriteBuffer(queue, buf, CL_TRUE, 0, nbytes, data, 0, NULL, NULL); + if (werr == CL_SUCCESS) { + return buf; } - return true; + clReleaseMemObject(buf); + } + buf = clCreateBuffer(context, + CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_WRITE_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + void * mapped = clEnqueueMapBuffer(queue, buf, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, nbytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + memcpy(mapped, data, nbytes); + const cl_int uerr = clEnqueueUnmapMemObject(queue, buf, mapped, 0, NULL, NULL); + if (uerr != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + +// Allocate a temporary download buffer of `nbytes`. The caller runs a kernel +// that writes into it, then reads it back to host via clEnqueueReadBuffer (or +// equivalent). Mirrors ggml_cl_create_temp_upload_buffer; the host-pinned +// fallback flags are flipped (CL_MEM_WRITE_ONLY | HOST_READ_ONLY) and the +// helper doesn't populate the buffer. +static cl_mem ggml_cl_create_temp_download_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + return buf; + } + buf = clCreateBuffer(context, + CL_MEM_WRITE_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_READ_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; } - - if (!ggml_can_fuse(cgraph, node_idx, ops)) { - return false; + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s download (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); } + return buf; +} - if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { - const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; - const ggml_tensor *mul = cgraph->nodes[node_idx+1]; +static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; + ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; - GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + cl_context context = backend_ctx->context; + cl_command_queue queue = backend_ctx->queue; - // rms_norm only supports f32 - if (mul->src[0]->type != GGML_TYPE_F32 || - mul->src[1]->type != GGML_TYPE_F32 || - mul->type != GGML_TYPE_F32) { - return false; - } +#ifdef GGML_OPENCL_SOA_Q + if (tensor->type == GGML_TYPE_Q2_K) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); - // if rms_norm is the B operand, then we don't handle broadcast - if (rms_norm == mul->src[1] && - !ggml_are_same_shape(mul->src[0], rms_norm)) { - return false; - } + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q2_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q2_K(); + extra->noshuffle = ggml_cl_q2_k_use_noshuffle(backend_ctx, tensor); - // rms_norm assumes contiguous rows - if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { - return false; - } - } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { - const ggml_tensor *norm = cgraph->nodes[node_idx]; - const ggml_tensor *mul = cgraph->nodes[node_idx+1]; - const ggml_tensor *add = cgraph->nodes[node_idx+2]; - const ggml_tensor *w = mul->src[0] == norm ? mul->src[1] : mul->src[0]; - const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; - // norm fusion only supports F32 - if (norm->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { - return false; - } + const size_t size_qs = n_blk * (blck/4); + const size_t size_sc = n_blk * (blck/16); + const size_t size_dm = n_blk * 2 * sizeof(ggml_fp16_t); + GGML_ASSERT(size_qs + size_sc + size_dm == ggml_nbytes(tensor) && "Incorrect tensor size"); - if (norm->src[0]->ne[0] % 4 != 0) { - return false; - } + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); - if (!ggml_is_contiguous(norm->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { - return false; - } - } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { - // rms_norm(x) * w + b, fused (residual). Mirrors the RMS_NORM+MUL gate - // plus the residual-add operand's constraints. - const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; - const ggml_tensor *mul = cgraph->nodes[node_idx+1]; - const ggml_tensor *add = cgraph->nodes[node_idx+2]; - const ggml_tensor *w = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0]; - const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + auto previous_origin = region.origin; - GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + region.origin = align_to(previous_origin + size_qs, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; - if (w->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 || - b->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32) { - return false; - } - if (rms_norm->src[0]->ne[0] % 4 != 0) { - return false; - } - // if rms_norm is the B operand of mul, broadcast is not handled - if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { - return false; - } - // the residual must match the normed output shape (no add broadcast) - if (!ggml_are_same_shape(b, add)) { - return false; - } - // rms_norm assumes contiguous rows - if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1]) || - !ggml_is_contiguous_rows(b)) { - return false; - } - } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_GROUP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { - const ggml_tensor *gn = cgraph->nodes[node_idx]; - const ggml_tensor *mul = cgraph->nodes[node_idx+1]; - const ggml_tensor *add = cgraph->nodes[node_idx+2]; - const ggml_tensor *w = mul->src[0] == gn ? mul->src[1] : mul->src[0]; - const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + region.origin = align_to(previous_origin + size_sc, backend_ctx->alignment); + region.size = size_dm; + CL_CHECK((extra->dm = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - if (gn->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { - return false; - } + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_q2_k_noshuffle + : backend_ctx->kernel_convert_block_q2_K; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); - if (!ggml_is_contiguous(gn->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { - return false; - } - } + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; - return true; -} + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); -static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor); -static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); -static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); -static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * gate_tensor, ggml_tensor * up_tensor, ggml_tensor * glu_tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - GGML_ASSERT(gate_tensor && up_tensor && glu_tensor); + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/16, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->dm, extra->dm, size_dm, k/(int)blck, m, true, true); + } +#endif - const ggml_tensor * Wg = gate_tensor->src[0]; - const ggml_tensor * Wu = up_tensor->src[0]; - const ggml_tensor * src1 = gate_tensor->src[1]; // == up_tensor->src[1] - const ggml_tensor * dst = glu_tensor; + extra->size_qs = size_qs; + extra->size_sc = size_sc; + extra->size_dm = size_dm; - GGML_ASSERT(Wg && Wg->extra); - GGML_ASSERT(Wu && Wu->extra); - GGML_ASSERT(src1 && src1->extra); - GGML_ASSERT(dst && dst->extra); + tensor->extra = extra; + return; + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + if (tensor->type == GGML_TYPE_Q3_K) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q4_K * extra_g = (ggml_tensor_extra_cl_q4_K *)Wg->extra; - ggml_tensor_extra_cl_q4_K * extra_u = (ggml_tensor_extra_cl_q4_K *)Wu->extra; + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q3_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q3_K(); + extra->noshuffle = ggml_cl_q3_k_use_noshuffle(backend_ctx, tensor); - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; - const int K = Wg->ne[0]; // ne00 - const int M = Wg->ne[1]; // ne01 (= ffn intermediate width) - const int N = 1; // decode GEMV + const size_t size_qs = n_blk * (blck/4); + const size_t size_hm = n_blk * (blck/8); + const size_t size_sc = n_blk * 12; + const size_t size_d = n_blk * sizeof(ggml_fp16_t); + GGML_ASSERT(size_qs + size_hm + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); - const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; - const int glu_op = (int)ggml_get_glu_op(dst); + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); - cl_context context = backend_ctx->context; - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + auto previous_origin = region.origin; - // q images for the two weight matrices (standard noshuffle layout) - img_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * K / 2 / 4; - img_desc.buffer = extra_g->q; - cl_mem qg_img = nullptr, qu_img = nullptr; - CL_CHECK((qg_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - img_desc.buffer = extra_u->q; - CL_CHECK((qu_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + region.origin = align_to(previous_origin + size_qs, backend_ctx->alignment); + region.size = size_hm; + CL_CHECK((extra->hm = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; - // shared activation image (one column at decode) - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - cl_mem b_sub_buf = nullptr, b_img = nullptr; - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - img_fmt = { CL_RGBA, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + region.origin = align_to(previous_origin + size_hm, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; - cl_kernel kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qg_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_g->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_g->dm)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra_g->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &qu_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra_u->d)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_u->dm)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extra_u->s)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &K)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_int), &M)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &glu_op)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_uchar), &mask_hi2)); + region.origin = align_to(previous_origin + size_sc, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // K-split = nsg_y subgroups. HARD-CAP at 8 (512 work-items): the fused - // kernel's cross-subgroup reduce uses a float4 reduceLM (gate+up packed) = - // 2x the LDS of the base GEMV's float2 reduce, so 16 co-resident subgroups - // exceed the per-CU LDS budget on X2 and the WG barrier DEADLOCKS -> GPU TDR - // (reproduced on upstream gemma-4 E4B decode, K=2560 M=10240). This used to - // be masked: get_kernel_workgroup_size reported 896 for this kernel (so the - // cap loop fell to 8), but it now returns 1024 and the Adreno per-kernel WG - // query is unreliable (over-reports), so cap explicitly instead of trusting - // it. nsg_y < 16 also means the cross-subgroup accumulation grouping differs - // from the standalone wide (nsg=16) GEMV, so the output is coherent but NOT - // byte-identical to the per-op path. Keep the maxwg query as a further floor - // for any driver that reports < 512. - size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); - size_t nsg_y = 8; - while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } - size_t local_work_size[3] = { 64, nsg_y, 1 }; - size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 }; + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_q3_k_noshuffle + : backend_ctx->kernel_convert_block_q3_K; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->hm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); - if (getenv("GGML_OPENCL_FUSE_DEBUG")) { - static int dbg = 0; - if (dbg < 3) { fprintf(stderr, "[FUSE_MM_GLU] fired #%d K=%d M=%d glu_op=%d nsg=%zu maxwg=%zu\n", ++dbg, K, M, glu_op, nsg_y, maxwg); fflush(stderr); } - } + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); - CL_CHECK(clReleaseMemObject(qg_img)); - CL_CHECK(clReleaseMemObject(qu_img)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); -#else - GGML_UNUSED(backend); - GGML_UNUSED(gate_tensor); - GGML_UNUSED(up_tensor); - GGML_UNUSED(glu_tensor); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->hm, extra->hm, size_hm, k/8, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sc, extra->sc, size_sc, 3*k/(int)blck, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + } #endif -} + extra->size_qs = size_qs; + extra->size_hm = size_hm; + extra->size_sc = size_sc; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + + if (tensor->type == GGML_TYPE_IQ3_S) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq3_s * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq3_s(); + extra->noshuffle = ggml_cl_iq3_s_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; -static void ggml_opencl_op_rms_norm_mul_add_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { - GGML_ASSERT(rms_norm_tensor && mul_tensor && add_tensor); + const size_t size_qs = n_blk * 64; + const size_t size_sg = n_blk * 32; + const size_t size_qh = n_blk * 8; + const size_t size_sc = n_blk * 4; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sg + size_qh + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); - const ggml_tensor * src0 = rms_norm_tensor->src[0]; - const ggml_tensor * src1 = mul_tensor->src[0] == rms_norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; - const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; - const ggml_tensor * dst = add_tensor; + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); - GGML_ASSERT(src0 && src0->extra); - GGML_ASSERT(src1 && src1->extra); - GGML_ASSERT(src2 && src2->extra); - GGML_ASSERT(dst && dst->extra); + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sg; + CL_CHECK((extra->sg = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offset2 = extra2->offset + src2->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + region.origin = align_to(region.origin + size_sg, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + region.origin = align_to(region.origin + size_qh, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - float eps; - memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); + region.origin = align_to(region.origin + size_sc, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; - const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; - const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; - const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; - const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; - const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; - const cl_ulong nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + cl_kernel kernel = backend_ctx->kernel_convert_block_iq3_s; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sg)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb_arg)); - GGML_ASSERT(ne00 % 4 == 0); + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; - size_t sgs; - if (backend_ctx->gpu_family == ADRENO) sgs = 64; - else if (backend_ctx->gpu_family == INTEL) sgs = 32; - else GGML_ASSERT(false && "Unsupported GPU"); + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); - cl_kernel kernel = backend_ctx->kernel_rms_norm_mul_add; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sg, extra->sg, size_sg, k/8, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, k/32, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/64, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/256, m, true, true); + } +#endif - int nth = sgs; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - while (nth < ne00 && nth < max_workgroup_size) nth *= 2; - nth = MIN(nth, max_workgroup_size); - nth = MIN(nth, ne00); + extra->size_qs = size_qs; + extra->size_sg = size_sg; + extra->size_qh = size_qh; + extra->size_sc = size_sc; + extra->size_d = size_d; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + tensor->extra = extra; + return; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 33, sizeof(float)*sgs, NULL)); + if (tensor->type == GGML_TYPE_IQ3_XXS) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq3_xxs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq3_xxs(); + extra->noshuffle = ggml_cl_iq3_xxs_use_noshuffle(backend_ctx, tensor); -static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; + const size_t size_qs = n_blk * 64; + const size_t size_sas = n_blk * 32; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sas + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); - // NOTE: this may oversynchronize by synchronizing with - // backends/devices which don't compute 'cgraph's - // dependencies. - sync_with_other_backends(backend); + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { - continue; - } + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { - continue; - } + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sas; + CL_CHECK((extra->sas = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { - ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); - i += 2; - continue; - } - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { - ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); - i += 2; - continue; - } - // Fuse the MoE combine: router-weight mul + cross-expert add chain -> - // one weighted-sum-across-experts kernel. - // Fold the gpt-oss MoE bias epilogue: add_id(gate_bias) + add_id(up_bias) + - // glu(swiglu_oai) -> one kernel, leaving the two matmuls as their own dispatches. - // Both add_ids are in-place passes over a tensor the GLU reads again, so this - // drops two full read+write passes per layer. Opt out GGML_OPENCL_FUSE_MOE_BIAS_GLU=0. - if (backend_ctx->fuse_moe_bias_glu && !backend_ctx->disable_fusion && - ggml_opencl_can_fuse_moe_bias_glu(cgraph, i)) { - ggml_cl_moe_bias_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2], - cgraph->nodes[i+3], cgraph->nodes[i+4]); - i += 4; - continue; - } + region.origin = align_to(region.origin + size_sas, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // Fold the MoE down-projection bias into the combine: add_id(down_bias) + the whole - // combine subgraph -> one kernel. Checked before the plain combine arm so the longer - // pattern wins. Opt out GGML_OPENCL_FUSE_MOE_BIAS_COMBINE=0. - if (backend_ctx->fuse_moe_bias_combine && backend_ctx->fuse_moe_combine && - !backend_ctx->disable_fusion) { - const ggml_tensor * bias_combine_out = nullptr; - if (ggml_opencl_can_fuse_moe_bias_combine(cgraph, i, &bias_combine_out)) { - ggml_cl_moe_bias_combine_fused(backend, node, cgraph->nodes[i+1], bias_combine_out); - i += 2 * (int)node->ne[1]; // ADD_ID + MUL + k VIEWs + (k-1) ADDs - continue; - } - } + cl_kernel kernel = backend_ctx->kernel_convert_block_iq3_xxs; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sas)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); - if (backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) { - const ggml_tensor * combine_out = nullptr; - if (ggml_opencl_can_fuse_moe_combine(cgraph, i, &combine_out)) { - ggml_cl_moe_combine_fused(backend, node, combine_out); - i += 2 * (int)node->ne[1] - 1; // skip the k VIEWs + (k-1) ADDs - continue; - } - } + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); - // Fuse rms_norm + mul(weight) + add(residual). Checked before the - // rms_norm+mul fuse so the 3-op pattern wins over its 2-op prefix. - // Default on, opt-out GGML_OPENCL_FUSE_RMS_ADD=0. - if (!backend_ctx->disable_fusion && backend_ctx->fuse_rms_add && - ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD })) { - ggml_opencl_op_rms_norm_mul_add_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); - i += 2; - continue; - } - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { - ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]); - i++; - continue; - } - // Fuse mul_mat(Wg,x) + mul_mat(Wu,x) + glu — fold the FFN's two decode - // GEMVs and the GLU into one dispatch. q4_K only (guarded below); the - // fused kernel uses the same accumulation/reduction order and the same - // scalar GLU formula -> coherent. Default on, opt-out GGML_OPENCL_FUSE_MM_GLU=0. #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // The fused executor (ggml_cl_mul_mat_q4_k_glu_fused) is image-path / - // Adreno-only (GGML_ABORT on the non-Adreno #else); gate the dispatch to - // match so the FFN GLU subgraph stays dormant on Intel/other drivers. - if (backend_ctx->fuse_mm_glu && !backend_ctx->disable_fusion && - ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU })) { - ggml_cl_mul_mat_q4_k_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); - i += 2; - continue; + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sas, extra->sas, size_sas, k/32, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/256, m, true, true); } #endif - bool ok = ggml_cl_compute_forward(backend, node); - if (!ok) { - GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); - } - GGML_ASSERT(ok); + extra->size_qs = size_qs; + extra->size_sas = size_sas; + extra->size_d = size_d; + + tensor->extra = extra; + return; } - return GGML_STATUS_SUCCESS; -} + if (tensor->type == GGML_TYPE_IQ1_M) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); -// The optimized gemm and gemv kernels are used for large matrices without batch. -// tensor is the quantized weights matrix. -inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - int64_t threshold_ne0 = 512; - int64_t threshold_ne1 = 512; - if (!backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) && - backend_ctx->adreno_cl_compiler_version.type != DX) { - threshold_ne0 = 128; - threshold_ne1 = 128; - } - bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && - tensor->ne[2] == 1 && tensor->ne[3] == 1; + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq1_m * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq1_m(); + extra->noshuffle = ggml_cl_iq1_m_use_noshuffle(backend_ctx, tensor); - // The noshuffle layout packs 2 rows per 32-bit texel and the GEMV reads it at an - // ne1/2 texel stride with an exact-cover dispatch, so it is only addressable when - // ne1 is a multiple of 64; an unaligned ne1 truncates the stride and the weight is - // read misaligned. That is a property of the layout, not of one quant -- q4_K, q5_K - // and q8_0 read the same packing as q6_K. The bound is 64, not 128: a q8_0 attention - // weight of ne1 = 2880 is a multiple of 64 but not 128 and is correct. - switch (tensor->type) { - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_Q8_0: - return threashold_ok && tensor->ne[1] % 64 == 0; - default: - break; - } - return threashold_ok; -} + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * 32; + const size_t size_qh = n_blk * 16; + const size_t size_sc = n_blk * 8; + GGML_ASSERT(size_qs + size_qh + size_sc == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); -static bool adreno_e17_compiler_quirks(const ggml_backend_opencl_context *backend_ctx) { - if (!backend_ctx || backend_ctx->gpu_family != GPU_FAMILY::ADRENO || - backend_ctx->adreno_cl_compiler_version.type != ADRENO_CL_COMPILER_TYPE::E17) { - return false; - } - const char * env = getenv("GGML_OPENCL_ADRENO_E17_QUIRKS"); - return !(env && env[0] == '0'); -} + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); -inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - // The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*. - // Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights. - // So, exclude A6x and A7x from using Adreno MoE kernels for now. - // The quants that have a general mul_mat_id kernel fallback to the general version; the - // rest fallback to CPU. - if (backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X || - backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X || - backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN)) { - return false; - } + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - if (adreno_e17_compiler_quirks(backend_ctx)) { - return false; - } + region.origin = align_to(region.origin + size_qh, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - int ne01 = tensor->ne[1]; - return (((strstr(tensor->name, "ffn") != NULL) && (strstr(tensor->name, "exps") != NULL)) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 32 == 0); -} + cl_kernel kernel = backend_ctx->kernel_convert_block_iq1_m; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); -// Device default for the tiled-wide lm_head/embed GEMV layout: ON for X2E and A8X. -// -// These kernels were previously off everywhere on the grounds that they compute -// wrong values at multi-superblock K. They do not: that NMSE ~2 came from the -// backend having no get_tensor restore path for the tiled layout, so -// test-backend-ops (which builds its CPU reference by copying the weights back -// out of the backend) compared a correct GPU result against a reference -// dequantized from tiled bytes. With the restore path added, MUL_MAT passes with -// the tiled kernels on, unmodified, on both devices. -// -// Perf, Qwen3-4B-Q4_K_M (q6_K lm_head 151936x2560), tg128, matched pairs with -// alternating lead, tiled vs o4: -// -// A8X +11.9% 6/6 pairs positive, order bias -0.06% (16.93 vs 15.14 tok/s) -// X2E +6.9% 4/4 pairs positive, order bias -0.03% (35.24 vs 32.87 tok/s) -// -// Measure this one on a COLD device. These kernels are far more clock-sensitive -// than the o4 route they replace: on a heat-soaked A8X (CPU cap at 1.5-1.9 GHz) -// tiled pins at ~14.2 tok/s while o4 still makes ~14.9, which reads as a 4-5% -// LOSS and inverts the ranking. The same box, after a reboot and a gate that -// waits for policy6 to return to 4396800, reports the +11.9% above with no -// order bias. A7X regresses hard on this layout and stays off. -// GGML_OPENCL_{Q4K,Q6K}_GEMV_TILED forces either way (=0 off, any other value on). -inline bool tiled_gemv_default_on(const ggml_backend_opencl_context *backend_ctx) { - return backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || - backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X); -} + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; -// Tiled-wide q6_K GEMV (default OFF; GGML_OPENCL_Q6K_GEMV_TILED forces either -// way: =0 off everywhere, any other value on everywhere). -// Both the convert (set_tensor) and the GEMV dispatch must agree on this so the -// buffer layout matches the kernel. -inline bool q6k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { - static const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_TILED"); - if (e && e[0] != '\0') { - return e[0] != '0'; - } - return tiled_gemv_default_on(backend_ctx); -} + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); -// Only the long-vocab lm_head/embed shapes use the tiled layout; ne01 % 64 == 0 -// is required by the 64-row tiling (no row padding in the buffers). -// use_adreno_kernels is required: only the Adreno GEMV path can read the tiled -// layout, so converting a weight it would decline (e.g. ne00 < 512) leaves the -// generic kernel reading tiled bytes as plain SOA. -inline bool use_q6k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - return q6k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q6_K && - tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && - use_adreno_kernels(backend_ctx, tensor); -} +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, k/16, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->sc, extra->sc, size_sc, k/64, m, true, true); + } +#endif -// q4_K analog of the tiled-wide lm_head/embed GEMV (default OFF; -// GGML_OPENCL_Q4K_GEMV_TILED forces either way: =0 off, else on). Same gate. -inline bool q4k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { - static const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_TILED"); - if (e && e[0] != '\0') { - return e[0] != '0'; + extra->size_qs = size_qs; + extra->size_qh = size_qh; + extra->size_sc = size_sc; + + tensor->extra = extra; + return; } - return tiled_gemv_default_on(backend_ctx); -} -inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - return q4k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q4_K && - tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && - use_adreno_kernels(backend_ctx, tensor); -} -inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + if (tensor->type == GGML_TYPE_IQ1_S) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); - bool adreno_kernel = use_adreno_kernels(backend_ctx, tensor); + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq1_s * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq1_s(); + extra->noshuffle = ggml_cl_iq1_s_use_noshuffle(backend_ctx, tensor); - size_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]; + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; - // The 2D weight transpose (transpose_2d_as_*) tiles rows by 4 over a 2D matrix, - // so it requires K(ne0)%32==0, M(ne1)%4==0 and ne2==ne3==1. - const bool shape_ok = (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 4 == 0) && - (tensor->ne[2] == 1) && (tensor->ne[3] == 1); + const size_t size_qs = n_blk * 32; + const size_t size_qh = n_blk * 16; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_qh + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); - return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27 -} + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); -inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - if (!use_adreno_kernels(backend_ctx, tensor)) { - return false; - } + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - const size_t elem_num = ggml_nelements(tensor); - const size_t q_img_width = elem_num / 8; - const size_t qh_img_width = elem_num / 16; - const bool shape_ok = tensor->ne[0] % 32 == 0 && tensor->ne[1] % 4 == 0 && - tensor->ne[2] == 1 && tensor->ne[3] == 1; + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - return shape_ok && q_img_width <= backend_ctx->image_max_buffer_size && - qh_img_width <= backend_ctx->image_max_buffer_size; -} + region.origin = align_to(region.origin + size_qh, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_convert_block_iq1_s; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); -inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || - !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { - return false; - } - return (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0); -#else - GGML_UNUSED(backend_ctx); - GGML_UNUSED(tensor); - return false; + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->qh, extra->qh, size_qh, k/32, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + } #endif -} -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v) { - if (backend_ctx->fa.kernel_flash_attn_f32_f16_bin == nullptr) { - return false; + extra->size_qs = size_qs; + extra->size_qh = size_qh; + extra->size_d = size_d; + + tensor->extra = extra; + return; } - const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; - const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + if (tensor->type == GGML_TYPE_IQ2_S) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); - const int n_q = q->ne[1]; - const int dk = q->ne[0]; - const int dv = v->ne[0]; + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq2_s * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq2_s(); + extra->noshuffle = ggml_cl_iq2_s_use_noshuffle(backend_ctx, tensor); - constexpr bool prefill_only = true; + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; - return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO && - (is_mixed || is_q8_0) && (dk == dv) - && (dk == 64 || dk == 128 || dk == 256 || dk == 512) - && (!prefill_only || n_q != 1)); -} -#endif + const size_t size_qs = n_blk * 32; + const size_t size_sg = n_blk * 32; + const size_t size_qh = n_blk * 8; + const size_t size_sc = n_blk * 8; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sg + size_qh + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); -// The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it -// is SLOWER than the route it replaces, not because it is unsafe. It was first -// parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that -// was a misattribution (the test-backend-ops dst sentinel was tripped by the o4 -// GEMV's unguarded tail store, fixed separately - and at the shape it was blamed -// for, k=1536, this predicate returns false anyway, so the flat route never ran). -// -// The escape's original rationale, "gemv_noshuffle perf drops for large M", -// predates the o4 kernel, which now covers the same long-vocab shapes and beats -// this route on every device measured (Qwen3-4B-Q4_K_M, q6_K lm_head -// 151936x2560, tg128, matched pairs vs o4): A8X -10.3% (0/3 pairs), X2E -3.7% -// (0/3). Keep it reachable for shapes o4 declines, but do not default it on. -static inline bool flat_large_m_enabled() { - static const char * e = getenv("GGML_OPENCL_FLAT_LARGE_M"); - static const bool en = e != nullptr && atoi(e) != 0; - return en; -} + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); -static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - if (tensor->ne[1] % 4 != 0 && tensor->ne[2] == 1 && tensor->ne[3] == 1) { - return true; - } + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - if (!flat_large_m_enabled()) { - return false; - } - // gemv_noshuffle variant perf drops for large M, use flat variant for large M. - // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. - // note that this forces large M weights to use LM GEMM. - // EXCEPT when this branch's tiled-canonical lm_head/embed layout is active: the - // weight is converted to the 64-row tiled layout, which the flat gemv would - // misread as garbage. use_q4k_tiled owns these large-M weights, so defer to it. - return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1 - && !use_q4k_tiled(backend_ctx, tensor); -} + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sg; + CL_CHECK((extra->sg = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); -static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - // NOTE on ordering: the ne01 % 128 escape below is a CORRECTNESS guard, not a - // performance one, so it must be reachable regardless of flat_large_m_enabled(). - // The opt-in gate therefore sits after it, and after the tiled deferral. - // gemv_noshuffle variant perf drops for large M, use flat variant for large M. - // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. - // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold. - // note that this forces large M weights to use LM GEMM. - // When this branch's tiled-canonical lm_head/embed layout is active, the weight is - // converted to the 64-row tiled layout, which the flat gemv would misread as - // garbage. use_q6k_tiled owns these large-M weights (it requires ne01 % 64 == 0, - // so it never claims an odd-vocab weight), so defer to it first. - if (use_q6k_tiled(backend_ctx, tensor)) { - return false; - } - // The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the - // gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of - // ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles. - // It is therefore only correct for ne01 % 128 == 0: an odd ne01 (e.g. granitemoe - // lm_head [1536, 49155] -- odd vocab) truncates the texel stride, misaligning every - // odd column of the transposed layout (gross garbage) and dropping the last row; - // other non-multiples over-dispatch and write past the end of dst. Route such - // tensors to the flat GEMV + regular convert; the matching GEMM (ne1>1) falls back - // to CPU (see supports_op). All standard even-vocab/hidden dims are multiples of - // 128 and keep the noshuffle path. - if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { - return true; - } + region.origin = align_to(region.origin + size_sg, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - if (!flat_large_m_enabled()) { - return false; - } + region.origin = align_to(region.origin + size_qh, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sc, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a - // proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head. - // Add a direct size escape so such weights also take the flat path, without changing - // which weights ne0 >= 2048 already routes there. - // The size escape is not taken on the A7X since its compiler miscompiles the flat K-quant GEMV - return tensor->ne[1] >= 32768 - && (tensor->ne[0] >= 2048 || (backend_ctx->adreno_gen != ADRENO_GPU_GEN::A7X && ggml_nbytes(tensor) >= (256ull << 20))) - && tensor->ne[2] == 1 && tensor->ne[3] == 1; -} + cl_kernel kernel = backend_ctx->kernel_convert_block_iq2_s; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sg)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb_arg)); -inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (!backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans || - !backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin) { - return false; - } - return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && - !use_q6k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor); -#else - GGML_UNUSED(backend_ctx); - GGML_UNUSED(tensor); - return false; -#endif -} + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; -inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans || - !backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) { - return false; - } - return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && - !use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor); -#else - GGML_UNUSED(backend_ctx); - GGML_UNUSED(tensor); - return false; -#endif -} + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); -inline bool use_q5_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (!backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans || - !backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin) { - return false; - } - return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0); -#else - GGML_UNUSED(backend_ctx); - GGML_UNUSED(tensor); - return false; + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sg, extra->sg, size_sg, k/8, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, k/32, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/32, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + } #endif -} -static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context; - ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; + extra->size_qs = size_qs; + extra->size_sg = size_sg; + extra->size_qh = size_qh; + extra->size_sc = size_sc; + extra->size_d = size_d; - // reject ops that match the opfilter regex - if (dev_ctx->opfilter && std::regex_match(std::string(ggml_op_desc(op)), *dev_ctx->opfilter)) { - return false; + tensor->extra = extra; + return; } - switch (op->op) { - case GGML_OP_NONE: - return true; - case GGML_OP_GET_ROWS: - switch (op->src[0]->type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - return true; - case GGML_TYPE_Q4_0: -#ifdef GGML_OPENCL_SOA_Q - // We do not support flattened Q4_0 (and possibly other Q's) - return false; -#else // GGML_OPENCL_SOA_Q - return true; -#endif // GGML_OPENCL_SOA_Q - default: - return false; - } - case GGML_OP_SET_ROWS: - { - // TODO: add support - // ref: https://github.com/ggml-org/llama.cpp/pull/14274 -#pragma message("TODO: implement BF16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, IQ4_NL support (https://github.com/ggml-org/llama.cpp/pull/14661)") - if (op->src[0]->type != GGML_TYPE_F32) { - return false; - } - switch (op->type) { - case GGML_TYPE_F16: - case GGML_TYPE_F32: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q4_0: - return (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); - default: - return false; - } - } - case GGML_OP_CPY: - case GGML_OP_DUP: - case GGML_OP_CONT: - switch (op->src[0]->type) { - case GGML_TYPE_F32: - switch (op->type) { - case GGML_TYPE_F16: - case GGML_TYPE_F32: - return true; - default: - return false; - } - case GGML_TYPE_F16: - switch (op->type) { - case GGML_TYPE_F16: - case GGML_TYPE_F32: - return true; - default: - return false; - } - case GGML_TYPE_I32: - switch (op->type) { - case GGML_TYPE_I32: - return true; - default: - return false; - } - default: - return false; - } - case GGML_OP_SET: { - return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32) && - op->type == op->src[0]->type && - op->type == op->src[1]->type; - } - case GGML_OP_SCALE: - return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); - case GGML_OP_ADD: - if (op->type == GGML_TYPE_F16) { - const bool src0_ok = op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32; - const bool src1_ok = op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32; - if (src0_ok && src1_ok) { - return true; - } - } - case GGML_OP_MUL: - case GGML_OP_DIV: - case GGML_OP_SUB: - return (op->src[0]->type == op->src[1]->type) && - (op->src[0]->type == op->type) && - (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); - case GGML_OP_ADD_ID: - return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_SQR: - case GGML_OP_SQRT: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && - ggml_is_contiguous(op->src[0]); - case GGML_OP_UNARY: - switch (ggml_get_unary_op(op)) { - case GGML_UNARY_OP_GELU: - case GGML_UNARY_OP_SILU: - case GGML_UNARY_OP_RELU: - case GGML_UNARY_OP_GELU_ERF: - case GGML_UNARY_OP_GELU_QUICK: - return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; - case GGML_UNARY_OP_SIGMOID: - return ggml_is_contiguous(op->src[0]); - case GGML_UNARY_OP_TANH: - case GGML_UNARY_OP_NEG: - case GGML_UNARY_OP_EXP: - // Adreno F16 exp/expm1 overflow even post-half->float convert. - return op->src[0]->type == GGML_TYPE_F32; - case GGML_UNARY_OP_EXPM1: - return op->src[0]->type == GGML_TYPE_F32; - case GGML_UNARY_OP_ABS: - case GGML_UNARY_OP_SGN: - case GGML_UNARY_OP_STEP: - case GGML_UNARY_OP_ELU: - case GGML_UNARY_OP_HARDSWISH: - case GGML_UNARY_OP_HARDSIGMOID: - case GGML_UNARY_OP_FLOOR: - case GGML_UNARY_OP_CEIL: - case GGML_UNARY_OP_ROUND: - case GGML_UNARY_OP_TRUNC: - return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; - case GGML_UNARY_OP_SOFTPLUS: - return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; - default: - return false; - } - case GGML_OP_GLU: - switch (ggml_get_glu_op(op)) { - case GGML_GLU_OP_GEGLU: - case GGML_GLU_OP_REGLU: - case GGML_GLU_OP_SWIGLU: - case GGML_GLU_OP_SWIGLU_OAI: - case GGML_GLU_OP_GEGLU_ERF: - case GGML_GLU_OP_GEGLU_QUICK: - case GGML_GLU_OP_SWIGLU_CLAMP: - return ggml_is_contiguous_1(op->src[0]) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); - default: - return false; - } - case GGML_OP_TRI: - return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); - case GGML_OP_FILL: - return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); - case GGML_OP_CLAMP: - return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_SOFT_MAX: - case GGML_OP_NORM: - return true; - case GGML_OP_RMS_NORM: - return op->ne[0] % 4 == 0 && ggml_is_contiguous_rows(op->src[0]); - case GGML_OP_L2_NORM: - return ggml_is_contiguous_rows(op->src[0]); - case GGML_OP_REPEAT: - return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; // Assuming F32 for now, can be expanded - case GGML_OP_PAD: - // TODO: add circular padding support for opencl, see https://github.com/ggml-org/llama.cpp/pull/16985 - if (ggml_get_op_params_i32(op, 8) != 0) { - return false; - } - return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; - case GGML_OP_UPSCALE: { - ggml_scale_mode mode = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & 0xFF); - const bool antialias = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & GGML_SCALE_FLAG_ANTIALIAS); - return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && - (mode == GGML_SCALE_MODE_NEAREST || mode == GGML_SCALE_MODE_BILINEAR) && !antialias; - } - case GGML_OP_CONV_2D: - return (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16) || - (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) || - (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); - case GGML_OP_SSM_CONV: - return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); - case GGML_OP_SSM_SCAN: { - if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || - op->src[1]->type != GGML_TYPE_F32 || op->src[2]->type != GGML_TYPE_F32 || - op->src[3]->type != GGML_TYPE_F32 || op->src[4]->type != GGML_TYPE_F32 || - op->src[5]->type != GGML_TYPE_F32 || op->src[6]->type != GGML_TYPE_I32) { - return false; - } + if (tensor->type == GGML_TYPE_IQ2_XS) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq2_xs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq2_xs(); + extra->noshuffle = ggml_cl_iq2_xs_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * 64; + const size_t size_sc = n_blk * 8; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sc, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_convert_block_iq2_xs; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); - const int64_t d_state = op->src[0]->ne[0]; - return d_state >= 1 && d_state <= 256 && (d_state & (d_state - 1)) == 0; - } - case GGML_OP_GATED_DELTA_NET: - { - // Match the Vulkan backend: only F32 -> F32, S_v in {16, 32, 64, 128}. - if (op->src[0]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) { - return false; - } - const int64_t S_v = op->src[2]->ne[0]; - return S_v == 16 || S_v == 32 || S_v == 64 || S_v == 128; - } - case GGML_OP_CONCAT: - { - const ggml_type t = op->src[0]->type; - return op->src[1]->type == t && op->type == t && - !ggml_is_quantized(t) && ggml_blck_size(t) == 1 && - (ggml_type_size(t) == 1 || ggml_type_size(t) == 2 || - ggml_type_size(t) == 4 || ggml_type_size(t) == 8); - } - case GGML_OP_TIMESTEP_EMBEDDING: - return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; - case GGML_OP_GROUP_NORM: - return ggml_is_contiguous(op->src[0]); - case GGML_OP_MUL_MAT: - if (op->src[0]->type == GGML_TYPE_F16) { - return true; - } else if (op->src[0]->type == GGML_TYPE_BF16) { - return true; - } else if (op->src[0]->type == GGML_TYPE_F32) { - return op->src[1]->type == GGML_TYPE_F32; - } else if (op->src[0]->type == GGML_TYPE_Q1_0) { - return op->src[1]->type == GGML_TYPE_F32; - } else if (op->src[0]->type == GGML_TYPE_Q4_0) { - // Non-contig src0 routes through on-device dequant-to-f16. - return op->src[1]->type == GGML_TYPE_F32; - } else if (op->src[0]->type == GGML_TYPE_Q4_1 || - op->src[0]->type == GGML_TYPE_Q5_0 || op->src[0]->type == GGML_TYPE_Q5_1 || - op->src[0]->type == GGML_TYPE_MXFP4 || - op->src[0]->type == GGML_TYPE_IQ4_NL || - op->src[0]->type == GGML_TYPE_Q4_K || - op->src[0]->type == GGML_TYPE_Q5_K || - op->src[0]->type == GGML_TYPE_Q6_K) { - // The E031.41 compiler (usually with A7x) miscompiles the flat K-quant - // GEMV kernels (kernel_mul_mv_q*_K_f32_flat) and makes lm_head run much - // slower than it should. So, make it fallback to CPU to preserve performance - // for this compiler series. - static const char * a7x_lmhead_env = getenv("GGML_OPENCL_A7X_LMHEAD_CPU"); - static const bool a7x_lmhead_cpu = (a7x_lmhead_env == nullptr || a7x_lmhead_env[0] != '0'); - if (a7x_lmhead_cpu && - backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && - (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || - op->src[0]->type == GGML_TYPE_Q6_K) && - op->src[0]->ne[1] >= 32768) { // vocab-scale weight; no FFN/attn weight is this tall - return false; - } - // The generic mul_mv (GEMV) kernels are wrong for large-batch prefill on - // Adreno. A quant mul_mat only avoids the GEMV when it reaches the Adreno - // trans-weight GEMM, which needs both a GEMM kernel for the type and - // use_adreno_kernels(). Decline the large-N shapes that would otherwise - // fall through to the GEMV. - { - const ggml_type t = op->src[0]->type; - const bool type_has_gemm = (t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q4_1 || - t == GGML_TYPE_IQ4_NL || t == GGML_TYPE_Q8_0 || - t == GGML_TYPE_Q4_K || t == GGML_TYPE_Q5_K || - t == GGML_TYPE_Q6_K); - const bool uses_gemm = type_has_gemm && use_adreno_kernels(backend_ctx, op->src[0]); - if (!uses_gemm && op->src[1]->ne[1] >= 512) { - return false; - } - } - return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); - } else if (op->src[0]->type == GGML_TYPE_Q8_0) { - // ggml_cl_mul_mat_q8_0_f32_adreno now honors src1/dst view_offs (the - // activation sub-buffer starts at offset1 and the kernels take offsetd), - // so a broadcast q8_0 matmul (src1 batch > src0 batch, e.g. Qwen3.5-9B-UD - // / Qwen3.6-35B q8_0 GDN ssm_out) runs on GPU via the per-slice broadcast - // iteration in ggml_cl_mul_mat. No special-casing needed. - return op->src[1]->type == GGML_TYPE_F32; - } - return false; - case GGML_OP_MUL_MAT_ID: - if (op->src[0]->type == GGML_TYPE_Q4_0 || - op->src[0]->type == GGML_TYPE_Q8_0 || - op->src[0]->type == GGML_TYPE_MXFP4) { - if (op->src[1]->type == GGML_TYPE_F32) { - return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); - } - } - // q4_0, q8_0 and mxfp4 have general MUL_MAT_ID support, - // the quantizations here currently do not - they are only supported by Adreno with certain shapes - if (op->src[0]->type == GGML_TYPE_Q4_1 || - op->src[0]->type == GGML_TYPE_Q5_0 || - op->src[0]->type == GGML_TYPE_Q5_1 || - op->src[0]->type == GGML_TYPE_Q4_K || - op->src[0]->type == GGML_TYPE_Q5_K || - op->src[0]->type == GGML_TYPE_Q6_K) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (op->src[1]->type == GGML_TYPE_F32) { - return use_adreno_moe_kernels(backend_ctx, op->src[0]) - && ggml_is_contiguous(op->src[0]) - && ggml_is_contiguous(op->src[1]); - } + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/32, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + } #endif - return false; - } - return false; - case GGML_OP_RESHAPE: - case GGML_OP_VIEW: - case GGML_OP_PERMUTE: - case GGML_OP_TRANSPOSE: - return true; - case GGML_OP_DIAG: - return true; - case GGML_OP_DIAG_MASK_INF: - return op->ne[3] == 1; - case GGML_OP_ROPE: { - const int mode = ((const int32_t *) op->op_params)[2]; - const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; - const bool is_vision = mode == GGML_ROPE_TYPE_VISION; - if (is_mrope && !is_vision) { - if (op->src[0]->type == GGML_TYPE_F32 || - op->src[0]->type == GGML_TYPE_F16) { - return true; - } - return false; - } - if (is_vision) { - if (op->src[0]->type == GGML_TYPE_F32 || - op->src[0]->type == GGML_TYPE_F16) { - return true; - } - return false; - } - return true; + + extra->size_qs = size_qs; + extra->size_sc = size_sc; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + + if (tensor->type == GGML_TYPE_IQ2_XXS) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq2_xxs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq2_xxs(); + extra->noshuffle = ggml_cl_iq2_xxs_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * 32; + const size_t size_sas = n_blk * 32; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sas + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sas; + CL_CHECK((extra->sas = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sas, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_convert_block_iq2_xxs; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sas)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sas, extra->sas, size_sas, k/32, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); } - case GGML_OP_SOLVE_TRI: - return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); - case GGML_OP_IM2COL: - return true; - case GGML_OP_ARGSORT: { - load_cl_kernels_argsort(backend_ctx); +#endif + + extra->size_qs = size_qs; + extra->size_sas = size_sas; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + + if (tensor->type == GGML_TYPE_Q1_0) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q1_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q1_0(); + + // q1_0 block = ggml_half d + (QK1_0/8) quant bytes = 2 + 16 = 18 bytes + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, then quants. + cl_buffer_region region; + + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_convert_block_q1_0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + + // q is uint32 (32 sign bits each); d is one half per 128-block. +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + int M = tensor->ne[1]; // ne01 + int K = tensor->ne[0]; // ne00 + + GGML_ASSERT(K % 128 == 0); + GGML_ASSERT(M % 4 == 0); + GGML_ASSERT(tensor->ne[2] == 1); + GGML_ASSERT(tensor->ne[3] == 1); + + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/32, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/128, M); + } // end transpose +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_argsort_f32_i32; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + return; + } - int cols = 1; - while (cols < op->ne[0]) { - cols *= 2; - } + if (tensor->type == GGML_TYPE_Q2_0) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q2_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q2_0(); + extra->noshuffle = ggml_cl_q2_0_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * 16; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_convert_block_q2_0; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); - return cols <= max_workgroup_size && op->src[0]->type == GGML_TYPE_F32; - } - case GGML_OP_SUM_ROWS: - case GGML_OP_CUMSUM: - return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); - case GGML_OP_MEAN: - return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_FLASH_ATTN_EXT: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_fa_bin_kernels_prefill(backend_ctx, op->src[0], op->src[1], op->src[2])) { - return true; - } + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/64, m, true, true); + } #endif - // The E17 compilers segfault while building FA kernels, skip E17 for now - if (adreno_e17_compiler_quirks(backend_ctx)) { - return false; - } - const ggml_tensor * q = op->src[0]; - const ggml_tensor * k = op->src[1]; - const ggml_tensor * v = op->src[2]; - const int dk = q->ne[0]; - const int dv = v->ne[0]; + extra->size_qs = size_qs; + extra->size_d = size_d; - const struct { int dk; int dv; } supported_dims[] = { - { 40, 40}, { 64, 64}, { 80, 80}, { 96, 96}, - {112, 112}, {128, 128}, {192, 128}, - {192, 192}, {256, 256}, - {512, 512}, - }; + tensor->extra = extra; + return; + } - bool dims_supported = false; - for (size_t i = 0; i < sizeof(supported_dims)/sizeof(supported_dims[0]); ++i) { - if (supported_dims[i].dk == dk && supported_dims[i].dv == dv) { - dims_supported = true; - break; - } - } - if (!dims_supported) { - return false; + // We separate the quantized bits and scale from block_q4_0 by using an + // additional kernel, where each thread handles a block. We first read the + // original weights into a temporary buffer, then create two separate + // buffers for quantized bits and scales, which are then populated by the + // conversion kernel. + if (tensor->type == GGML_TYPE_Q4_0) { + // Views can't SoA-ify here — parent owns the layout (see q8_0 guard). + // A view of an AoS tensor (the KV cache, written by a state restore) + // is written in place. + if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { + if (!ggml_cl_is_q4_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueWriteBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); } + return; + } + // Tensors should have been preallocated, therefore they should + // already have ggml_tensor_extra_cl as extra. + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - const bool is_f32_f32 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && - v->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; - const bool is_f16_f16 = q->type == GGML_TYPE_F16 && k->type == GGML_TYPE_F16 && - v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; - const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && - v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q4_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_0(); - const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && - v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 && - dk % 32 == 0 && dv % 32 == 0; - const bool is_f32_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && - v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 && - dk % 32 == 0 && dv % 32 == 0; + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); - // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram - // building the flash_attn programs whose KV path is mixed-type or - // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it - // is DK-independent). It is a driver crash, not codegen-wrong-output, so - // it cannot be caught in-process (fatal=false only handles clean compile - // errors). The uniform f16_f16 / f32_f32 programs compile fine on this - // compiler, so decline only the KV-convert variants; ggml then runs - // those (f16-KV / quant-KV) attention layers on the CPU backend. - // Negative compiler carve-out, same idiom as the Intel DK=512 decline - // below and the X1E driver-quirk guards. - if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && - (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { - return false; - } + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); - // Asymmetric KV: host-dequants both sides to F32, uses f32 kernel. - auto is_kv_type_ok = [](ggml_type t) { - return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 || - t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q8_0; - }; - const bool is_f32_asym = q->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && - k->type != v->type && - is_kv_type_ok(k->type) && is_kv_type_ok(v->type); + // We consider the specified offset arg as always, although For weights + // the offset arg should be 0 (we do not assert this). + //GGML_ASSERT(offset == 0); - const bool kv_combo_ok = is_f32_f32 || is_f16_f16 || is_f32_f16 || - is_f32_q8_0 || is_f32_q4_0 || is_f32_asym; - if (!kv_combo_ok) { - return false; - } + // We create subbuffers from the original tensor buffer for scales and + // quants - i.e., scales and quants are aliases into the buffer object + // that backs the original tensor. This is a cleaner way to adapt to the + // new memory management. + // In the old code, we allocate new buffers for scales and quants + // respectively, which could still be done but would result in double + // allocation; properly deallocating the preallocated buffer that backs + // the tensors is tricky and would leak the backend specific information + // into the general backend code. + // Does this create misaligned subbuffers (alignment is 1024) in certain + // cases ? + cl_buffer_region region; - // Some compilers for A7x (Adreno 740, compiler E031.41) crashes when - // building FA kernels with mixed or quant types (f32_f16, f32_q8_0, f32_q4_0) - // Here we skip all A7x for these kernels to avoid crash - if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && - (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { - return false; - } + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, then quants. + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; - if (dk == 512) { - if (backend_ctx->gpu_family == INTEL) { - return false; - } - if (!is_f32_f16) { - return false; - } - if (q->ne[1] == 1) { - // DK=512 decode is bandwidth-bound and slower on the GPU - // than on the CPU; decline it here so it runs on the CPU. - // Prefill (n_q > 1) stays on the GPU. - return false; - } else { - // prefill, BM-tile in its own FA_PREFILL_ONLY program - if (!ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false)) { - return false; - } - } - } - return true; - } - default: - return false; - } -} + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); -// Forward declaration - implementation appears later in the file. -static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe q4_0 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0_trans4_ns; -static ggml_guid_t ggml_backend_opencl_guid() { - static ggml_guid guid = { 0xde, 0xe0, 0x70, 0xa2, 0x73, 0x4e, 0x4d, 0xbc, 0xb0, 0xc7, 0x4f, 0xd4, 0x6d, 0x4e, 0x90, 0xfe }; - return &guid; -} + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); -static ggml_backend_i ggml_backend_opencl_i = { - /* .get_name = */ ggml_backend_opencl_name, - /* .free = */ ggml_backend_opencl_free, - /* .set_tensor_async = */ NULL, /* ggml_backend_opencl_set_tensor_async */ - /* .get_tensor_async = */ NULL, /* ggml_backend_opencl_get_tensor_async */ - /* .set_tensor_2d_async = */ NULL, - /* .get_tensor_2d_async = */ NULL, - /* .cpy_tensor_async = */ NULL, /* ggml_backend_opencl_cpy_tensor_async */ - /* .synchronize = */ ggml_backend_opencl_synchronize, - /* .graph_plan_create = */ NULL, - /* .graph_plan_free = */ NULL, - /* .graph_plan_update = */ NULL, - /* .graph_plan_compute = */ NULL, - /* .graph_compute = */ ggml_backend_opencl_graph_compute, - /* .event_record = */ NULL, - /* .event_wait = */ NULL, - /* .graph_optimize = */ NULL, -}; + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + // Create image for Q + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast<size_t>(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + tensor->extra = extra; + // MoE tensors are also SOA'ed + ctx->q4_0_soa_tensors.insert(tensor); -ggml_backend_t ggml_backend_opencl_init(void) { - ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_opencl_reg(), 0); - ggml_backend_opencl_context *backend_ctx = ggml_cl_init(dev); - backend_ctx->ref_count++; + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - ggml_backend_t backend = new ggml_backend { - /* .guid = */ ggml_backend_opencl_guid(), - /* .iface = */ ggml_backend_opencl_i, - /* .device = */ dev, - /* .context = */ backend_ctx - }; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; - return backend; -} + // The optimized kernels need weights in natural order, so unshuffle. + if (use_adreno_kernels(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q4_0_noshuffle; + } +#else + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); -bool ggml_backend_is_opencl(ggml_backend_t backend) { - return backend && backend->iface.get_name == ggml_backend_opencl_name; -} + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; -// -// buffer -// -struct ggml_backend_opencl_buffer_context { - // A buffer context can hold multiple cl_mem objects. This is for flattening - // quantized weights and should be used with GGML_OPENCL_SMALL_ALLOC where - // each tensor is allocated a separate buffer. When flattening is enabled - // with small allocation, each tensor is backed by two cl_mem objects (for - // quants and scales) packed into a backend_opencl_buffer. - ggml_backend_opencl_buffer_context(cl_mem buf) - : name("OpenCL") { - buffer.push_back(buf); - } + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); - ~ggml_backend_opencl_buffer_context() { - for (cl_mem buf : buffer) { - CL_CHECK(clReleaseMemObject(buf)); - } - for (cl_mem im : img) { - CL_CHECK(clReleaseMemObject(im)); - } + tensor->extra = extra; + ctx->q4_0_soa_tensors.insert(tensor); - // Delete all extras to trigger their destructors - for (ggml_tensor_extra_cl * e : temp_tensor_extras) { - delete e; - } - for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0) { - delete e; - } - for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1) { - delete e; - } - for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0) { - delete e; - } - for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1) { - delete e; - } - for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4) { - delete e; - } - for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0) { - delete e; - } - for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0) { - delete e; - } - for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl) { - delete e; - } - for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K) { - delete e; - } - for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K) { - delete e; - } - for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { - delete e; - } - for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { - delete e; - } - } + // transpose the weights and scales +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Only do transpose for large, non batched matrix + // TODO: use preallocated images instead of sub-buffer then image + if (use_adreno_kernels(backend_ctx, tensor)) { + int M = tensor->ne[1]; + int K = tensor->ne[0]; - ggml_tensor_extra_cl * ggml_opencl_alloc_temp_tensor_extra() { - ggml_tensor_extra_cl * extra; - if (temp_tensor_extras.empty()) { - extra = new ggml_tensor_extra_cl(); - } else { - extra = temp_tensor_extras.back(); - temp_tensor_extras.pop_back(); - } + GGML_ASSERT(K % 32 == 0); - temp_tensor_extras_in_use.push_back(extra); + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; - extra->reset(); - return extra; - } + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K / 8, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K / 32, M); - ggml_tensor_extra_cl_q1_0 * ggml_opencl_alloc_temp_tensor_extra_q1_0() { - ggml_tensor_extra_cl_q1_0 * extra; - if (temp_tensor_extras_q1_0.empty()) { - extra = new ggml_tensor_extra_cl_q1_0(); - } else { - extra = temp_tensor_extras_q1_0.back(); - temp_tensor_extras_q1_0.pop_back(); + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + wimg_fmt = { CL_R, CL_HALF_FLOAT }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 32; + wimg_desc.buffer = extra->d; + CL_CHECK((extra->d_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q and d as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; + } + if (tensor->type == GGML_TYPE_Q4_1) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - temp_tensor_extras_q1_0_in_use.push_back(extra); + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q4_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_1(); - extra->reset(); - return extra; - } + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); - ggml_tensor_extra_cl_q4_0 * ggml_opencl_alloc_temp_tensor_extra_q4_0() { - ggml_tensor_extra_cl_q4_0 * extra; - if (temp_tensor_extras_q4_0.empty()) { - extra = new ggml_tensor_extra_cl_q4_0(); - } else { - extra = temp_tensor_extras_q4_0.back(); - temp_tensor_extras_q4_0.pop_back(); - } + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); - temp_tensor_extras_q4_0_in_use.push_back(extra); + cl_buffer_region region; - extra->reset(); - return extra; - } + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, mins, then quants. + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; - ggml_tensor_extra_cl_q4_1 * ggml_opencl_alloc_temp_tensor_extra_q4_1() { - ggml_tensor_extra_cl_q4_1 * extra; - if (temp_tensor_extras_q4_1.empty()) { - extra = new ggml_tensor_extra_cl_q4_1(); - } else { - extra = temp_tensor_extras_q4_1.back(); - temp_tensor_extras_q4_1.pop_back(); - } + // Create subbuffer for mins. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_m; + extra->m = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; - temp_tensor_extras_q4_1_in_use.push_back(extra); + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe q4_1 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1_trans4_ns; - extra->reset(); - return extra; - } + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - ggml_tensor_extra_cl_q5_0 * ggml_opencl_alloc_temp_tensor_extra_q5_0() { - ggml_tensor_extra_cl_q5_0 * extra; - if (temp_tensor_extras_q5_0.empty()) { - extra = new ggml_tensor_extra_cl_q5_0(); - } else { - extra = temp_tensor_extras_q5_0.back(); - temp_tensor_extras_q5_0.pop_back(); - } + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; - temp_tensor_extras_q5_0_in_use.push_back(extra); + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); - extra->reset(); - return extra; - } + // Create image for Q + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast<size_t>(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + tensor->extra = extra; - ggml_tensor_extra_cl_q5_1 * ggml_opencl_alloc_temp_tensor_extra_q5_1() { - ggml_tensor_extra_cl_q5_1 * extra; - if (temp_tensor_extras_q5_1.empty()) { - extra = new ggml_tensor_extra_cl_q5_1(); - } else { - extra = temp_tensor_extras_q5_1.back(); - temp_tensor_extras_q5_1.pop_back(); + return; } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - temp_tensor_extras_q5_1_in_use.push_back(extra); - - extra->reset(); - return extra; - } + // normal q4_1 repack +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; - ggml_tensor_extra_cl_mxfp4 * ggml_opencl_alloc_temp_tensor_extra_mxfp4() { - ggml_tensor_extra_cl_mxfp4 * extra; - if (temp_tensor_extras_mxfp4.empty()) { - extra = new ggml_tensor_extra_cl_mxfp4(); - } else { - extra = temp_tensor_extras_mxfp4.back(); - temp_tensor_extras_mxfp4.pop_back(); + if (use_adreno_kernels(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q4_1_noshuffle; } +#else + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); - temp_tensor_extras_mxfp4_in_use.push_back(extra); + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - extra->reset(); - return extra; - } + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); - ggml_tensor_extra_cl_q8_0 * ggml_opencl_alloc_temp_tensor_extra_q8_0() { - ggml_tensor_extra_cl_q8_0 * extra; - if (temp_tensor_extras_q8_0.empty()) { - extra = new ggml_tensor_extra_cl_q8_0(); - } else { - extra = temp_tensor_extras_q8_0.back(); - temp_tensor_extras_q8_0.pop_back(); - } + tensor->extra = extra; - temp_tensor_extras_q8_0_in_use.push_back(extra); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { - extra->reset(); - return extra; - } + int M = tensor->ne[1]; + int K = tensor->ne[0]; - ggml_tensor_extra_cl_iq4_nl * ggml_opencl_alloc_temp_tensor_extra_iq4_nl() { - ggml_tensor_extra_cl_iq4_nl * extra; - if (temp_tensor_extras_iq4_nl.empty()) { - extra = new ggml_tensor_extra_cl_iq4_nl(); - } else { - extra = temp_tensor_extras_iq4_nl.back(); - temp_tensor_extras_iq4_nl.pop_back(); + GGML_ASSERT(K % 32 == 0); + + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + // Transpose m as ushort + transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; + } + if (tensor->type == GGML_TYPE_Q5_0) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - temp_tensor_extras_iq4_nl_in_use.push_back(extra); + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q5_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_0(); - extra->reset(); - return extra; - } + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_qs = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(int32_t); + GGML_ASSERT(size_d + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); - ggml_tensor_extra_cl_q4_K * ggml_opencl_alloc_temp_tensor_extra_q4_K() { - ggml_tensor_extra_cl_q4_K * extra; - if (temp_tensor_extras_q4_K.empty()) { - extra = new ggml_tensor_extra_cl_q4_K(); - } else { - extra = temp_tensor_extras_q4_K.back(); - temp_tensor_extras_q4_K.pop_back(); - } + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); - temp_tensor_extras_q4_K_in_use.push_back(extra); + cl_buffer_region region; - extra->reset(); - return extra; - } + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; - ggml_tensor_extra_cl_q5_K * ggml_opencl_alloc_temp_tensor_extra_q5_K() { - ggml_tensor_extra_cl_q5_K * extra; - if (temp_tensor_extras_q5_K.empty()) { - extra = new ggml_tensor_extra_cl_q5_K(); - } else { - extra = temp_tensor_extras_q5_K.back(); - temp_tensor_extras_q5_K.pop_back(); - } + // Create subbuffer for qh. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_qh; + extra->qh = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; - temp_tensor_extras_q5_K_in_use.push_back(extra); + // Create subbuffer for qs. + region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); + region.size = size_qs; + extra->qs = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - extra->reset(); - return extra; - } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe q5_0 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_trans4_ns; - ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() { - ggml_tensor_extra_cl_q6_K * extra; - if (temp_tensor_extras_q6_K.empty()) { - extra = new ggml_tensor_extra_cl_q6_K(); - } else { - extra = temp_tensor_extras_q6_K.back(); - temp_tensor_extras_q6_K.pop_back(); - } + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - temp_tensor_extras_q6_K_in_use.push_back(extra); + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; - extra->reset(); - return extra; - } + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); - void reset() { - for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { - temp_tensor_extras.push_back(e); - } - temp_tensor_extras_in_use.clear(); + // Create image for Q + cl_image_format img_format_qs = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_qs = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast<size_t>(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->qs } + }; + extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); + tensor->extra = extra; - for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { - temp_tensor_extras_q1_0.push_back(e); - } - temp_tensor_extras_q1_0_in_use.clear(); + // Generic dp4a MoE path + { + static const char * q5dp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A"); + const bool q5dp4a = q5dp4a_env ? (atoi(q5dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q5dp4a && ne02 > 1 && (ne00 % 32 == 0)) { + size_t nb32 = (size_t)ne00 / 32; + size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; + size_t mn_elems = (size_t)ne02 * ne01 * nb32; + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_0; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->min)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 4, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } - for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { - temp_tensor_extras_q4_0.push_back(e); + return; } - temp_tensor_extras_q4_0_in_use.clear(); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { - temp_tensor_extras_q4_1.push_back(e); - } - temp_tensor_extras_q4_1_in_use.clear(); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0_in_use) { - temp_tensor_extras_q5_0.push_back(e); - } - temp_tensor_extras_q5_0_in_use.clear(); + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1_in_use) { - temp_tensor_extras_q5_1.push_back(e); - } - temp_tensor_extras_q5_1_in_use.clear(); + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); - for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { - temp_tensor_extras_mxfp4.push_back(e); - } - temp_tensor_extras_mxfp4_in_use.clear(); + tensor->extra = extra; - for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { - temp_tensor_extras_q8_0.push_back(e); - } - temp_tensor_extras_q8_0_in_use.clear(); + int M = tensor->ne[1]; + int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); - for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl_in_use) { - temp_tensor_extras_iq4_nl.push_back(e); - } - temp_tensor_extras_iq4_nl_in_use.clear(); + // Transpose qs as ushort + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M); + // Transpose qh as uchar + transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); - for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { - temp_tensor_extras_q4_K.push_back(e); + return; } - temp_tensor_extras_q4_K_in_use.clear(); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &n_blk)); - for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { - temp_tensor_extras_q5_K.push_back(e); - } - temp_tensor_extras_q5_K_in_use.clear(); + size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { - temp_tensor_extras_q6_K.push_back(e); - } - temp_tensor_extras_q6_K_in_use.clear(); + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); - q8_0_soa_tensors.clear(); - q4_0_soa_tensors.clear(); + tensor->extra = extra; + return; } + if (tensor->type == GGML_TYPE_Q5_1) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - // Pools for extras. Available extras are in `temp_tensor_extras`. Extras - // being used are in `temp_tensor_extras_in_use`. At the first run, new - // extras get created and put in `in_use`. When the buffer is reset via - // the `reset` callback, all extras in `in_use` get moved to available extras - // for reuse. - std::vector<ggml_tensor_extra_cl *> temp_tensor_extras; - std::vector<ggml_tensor_extra_cl *> temp_tensor_extras_in_use; - std::vector<ggml_tensor_extra_cl_q1_0 *> temp_tensor_extras_q1_0; - std::vector<ggml_tensor_extra_cl_q1_0 *> temp_tensor_extras_q1_0_in_use; - std::vector<ggml_tensor_extra_cl_q4_0 *> temp_tensor_extras_q4_0; - std::vector<ggml_tensor_extra_cl_q4_0 *> temp_tensor_extras_q4_0_in_use; - std::vector<ggml_tensor_extra_cl_q4_1 *> temp_tensor_extras_q4_1; - std::vector<ggml_tensor_extra_cl_q4_1 *> temp_tensor_extras_q4_1_in_use; - std::vector<ggml_tensor_extra_cl_q5_0 *> temp_tensor_extras_q5_0; - std::vector<ggml_tensor_extra_cl_q5_0 *> temp_tensor_extras_q5_0_in_use; - std::vector<ggml_tensor_extra_cl_q5_1 *> temp_tensor_extras_q5_1; - std::vector<ggml_tensor_extra_cl_q5_1 *> temp_tensor_extras_q5_1_in_use; - std::vector<ggml_tensor_extra_cl_mxfp4 *> temp_tensor_extras_mxfp4; - std::vector<ggml_tensor_extra_cl_mxfp4 *> temp_tensor_extras_mxfp4_in_use; - std::vector<ggml_tensor_extra_cl_q8_0 *> temp_tensor_extras_q8_0; - std::vector<ggml_tensor_extra_cl_q8_0 *> temp_tensor_extras_q8_0_in_use; - std::vector<ggml_tensor_extra_cl_iq4_nl *> temp_tensor_extras_iq4_nl; - std::vector<ggml_tensor_extra_cl_iq4_nl *> temp_tensor_extras_iq4_nl_in_use; - std::vector<ggml_tensor_extra_cl_q4_K *> temp_tensor_extras_q4_K; - std::vector<ggml_tensor_extra_cl_q4_K *> temp_tensor_extras_q4_K_in_use; - std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K; - std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K_in_use; - std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K; - std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K_in_use; + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q5_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_1(); - // q8_0 tensors with AoS->SoA layout conversion installed by set_tensor. - // Two types of tensors get SOA'ed - normal weights and MoE weights. - // In Q8_0's case, we only have normal weights. If we ever have Q8_0 as MoE - // weights, they need to be added to this set in `set_tensors`. - std::unordered_set<const ggml_tensor *> q8_0_soa_tensors; + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_qs = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(int32_t); + GGML_ASSERT(size_d + size_m + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); - // Same for q4_0. KV-cache q4_0 tensors are allocated but never pass - // through set_tensor, so they stay AoS and aren't in this set. - // In Q4_0's case, in addition to normal weights, we have MoE weights. - std::unordered_set<const ggml_tensor *> q4_0_soa_tensors; + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); - // The buffer_context is initially created by ggml_backend_buft_alloc_buffer - // before any tensor is initialized (at the beginning of alloc_tensor_range). - // Hence, there is always a buffer object in this vector. When each tensor is - // being initialized, this original buffer object will be released if both - // flattening and small allocation are enabled, and additional buffer - // objects will be created in init_tensor to represent flattened quantized - // weights. - std::vector<cl_mem> buffer; - // These are image1d_buffer_t objects that wrap around the quants and scales. - // For Q4_0 quantization, there should be two of them - one for quants and - // one for scales. They should be populated only when flattening and small - // allocation are enabled. - std::vector<cl_mem> img; - std::string name; -}; + cl_buffer_region region; -static void ggml_backend_opencl_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - delete ctx; -} + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, mins, then quants. + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; -static void * ggml_backend_opencl_buffer_get_base(ggml_backend_buffer_t buffer) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; - return (void *) (uintptr_t) dev_ctx->backend_ctx->alignment; -} + // Create subbuffer for mins. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_m; + extra->m = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; -static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + // Create subbuffer for qh. + region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); + region.size = size_qh; + extra->qh = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; - if (tensor->view_src != nullptr) { - GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); + // Create subbuffer for qs. + region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); + region.size = size_qs; + extra->qs = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - ggml_tensor_extra_cl * view_extra = (ggml_tensor_extra_cl *) tensor->view_src->extra; - GGML_ASSERT(view_extra && "view_extra is nullptr?"); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe q5_1 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_trans4_ns; - // Reuse extra of the parent tensor. The offset of this view tensor - // becomes `extra->offset + view_offs` and needs to be calculated when - // it is used. This changes is needed because of the change to - // ggml_alloc.c in https://github.com/ggml-org/llama.cpp/pull/7640. - // `buffer` passed in here will always be `tensor->buffer`. It is OK - // to allocate extras from the same buffer context for ordinary - // intermediate tensors. But for views into kv cache tensors, doing so - // would mess up the extras used by kv cache. - // Before #7640, `buffer` is for intermediate tensors, which is always - // different from that of kv cache tensors. - // - // NB: now extra->offset no longer accounts for view_offs. - // NB: this should not apply to weight tensors (for end-to-end runs, but - // may apply for test-backend-ops). - // FIXME: if any unexpected results are seen, double check the offset - - // there could be other places that need fix. - tensor->extra = view_extra; - } else { - { - size_t offset = (char *) tensor->data - (char *) ggml_backend_opencl_buffer_get_base(buffer); + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + // Create image for Q + cl_image_format img_format_qs = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_qs = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast<size_t>(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->qs } + }; + extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); + tensor->extra = extra; + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - ggml_tensor_extra_cl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra(); - extra->offset = offset; - extra->data_device = ctx->buffer[0]; - extra->actual_size = ggml_nbytes(tensor); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; - } - } - return GGML_STATUS_SUCCESS; -} -// Allocate a temporary upload buffer of `nbytes` and populate it with `data` -// from host. On Adreno X1-85 the device-only pool intermittently fails to -// allocate at hundreds of MB once model weights fragment the heap (observed -// on Qwen3.5-9B output.weight Q6_K at 834 MB). Three-step retry: -// 1. CL_MEM_READ_WRITE alloc + clEnqueueWriteBuffer (normal fast path). -// 2. clFinish + retry (drains in-flight allocs that may be holding heap; -// mirrors the proven pattern at the FD-split partial buffer alloc). -// 3. CL_MEM_ALLOC_HOST_PTR + map(WRITE_INVALIDATE) + memcpy + unmap — -// different memory pool (host-pinned); true zero-copy on Adreno per -// QCOM guidance. (CL_MEM_USE_HOST_PTR is NOT zero-copy on Adreno: the -// driver triggers an internal copy because arbitrary host pages aren't -// guaranteed mappable/coherent, AND it draws from the same exhausted -// device pool — so it doesn't solve the problem.) -// Returns the ready-to-read buffer (caller must clReleaseMemObject) or NULL -// if all three strategies fail. The buffer is opaque to the caller — it can -// be passed as a kernel argument like any normal cl_mem. -static cl_mem ggml_cl_create_temp_upload_buffer( - cl_context context, cl_command_queue queue, - size_t nbytes, const void * data, - const char * tensor_name_for_log) -{ - cl_int err; - cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); - if (err != CL_SUCCESS) { - clFinish(queue); - buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); - } - if (err == CL_SUCCESS) { - const cl_int werr = clEnqueueWriteBuffer(queue, buf, CL_TRUE, 0, nbytes, data, 0, NULL, NULL); - if (werr == CL_SUCCESS) { - return buf; - } - clReleaseMemObject(buf); - } - buf = clCreateBuffer(context, - CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_WRITE_ONLY, - nbytes, NULL, &err); - if (err != CL_SUCCESS) { - return NULL; - } - void * mapped = clEnqueueMapBuffer(queue, buf, CL_TRUE, - CL_MAP_WRITE_INVALIDATE_REGION, 0, nbytes, 0, NULL, NULL, &err); - if (err != CL_SUCCESS) { - clReleaseMemObject(buf); - return NULL; - } - memcpy(mapped, data, nbytes); - const cl_int uerr = clEnqueueUnmapMemObject(queue, buf, mapped, 0, NULL, NULL); - if (uerr != CL_SUCCESS) { - clReleaseMemObject(buf); - return NULL; - } - if (tensor_name_for_log) { - GGML_LOG_INFO("ggml_opencl: %s (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", - tensor_name_for_log, nbytes / 1024.0 / 1024.0); - } - return buf; -} + int M = tensor->ne[1]; + int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); -// Allocate a temporary download buffer of `nbytes`. The caller runs a kernel -// that writes into it, then reads it back to host via clEnqueueReadBuffer (or -// equivalent). Mirrors ggml_cl_create_temp_upload_buffer; the host-pinned -// fallback flags are flipped (CL_MEM_WRITE_ONLY | HOST_READ_ONLY) and the -// helper doesn't populate the buffer. -static cl_mem ggml_cl_create_temp_download_buffer( - cl_context context, cl_command_queue queue, - size_t nbytes, const char * tensor_name_for_log) -{ - cl_int err; - cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); - if (err != CL_SUCCESS) { - clFinish(queue); - buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); - } - if (err == CL_SUCCESS) { - return buf; - } - buf = clCreateBuffer(context, - CL_MEM_WRITE_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_READ_ONLY, - nbytes, NULL, &err); - if (err != CL_SUCCESS) { - return NULL; - } - if (tensor_name_for_log) { - GGML_LOG_INFO("ggml_opencl: %s download (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", - tensor_name_for_log, nbytes / 1024.0 / 1024.0); - } - return buf; -} + // Transpose qs as ushort + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M); + // Transpose qh as uchar + transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + // Transpose m as ushort + transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); -static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; - ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); - cl_context context = backend_ctx->context; - cl_command_queue queue = backend_ctx->queue; + size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; -#ifdef GGML_OPENCL_SOA_Q - if (tensor->type == GGML_TYPE_Q1_0) { + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q1_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q1_0(); + ggml_tensor_extra_cl_mxfp4 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_mxfp4(); - // q1_0 block = ggml_half d + (QK1_0/8) quant bytes = 2 + 16 = 18 bytes - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); - GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + size_t size_e = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(char); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. @@ -9916,101 +14780,120 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( + region.size = size_e; + extra->e = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.origin = align_to(previous_origin + size_e, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); - cl_kernel kernel = backend_ctx->kernel_convert_block_q1_0; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe mxfp4 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); + + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + tensor->extra = extra; + + // Create image for Q + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast<size_t>(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + tensor->extra = extra; + + return; + } + +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + size_t global_work_size[3] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[3] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); + // Create image for Q + cl_image_format img_format_q = {CL_RG, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast<size_t>(ggml_nelements(tensor)/32*2), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); tensor->extra = extra; - // q is uint32 (32 sign bits each); d is one half per 128-block. -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (enable_adreno_trans_weight(backend_ctx, tensor)) { - int M = tensor->ne[1]; // ne01 - int K = tensor->ne[0]; // ne00 - - GGML_ASSERT(K % 128 == 0); - GGML_ASSERT(M % 4 == 0); - GGML_ASSERT(tensor->ne[2] == 1); - GGML_ASSERT(tensor->ne[3] == 1); - - transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/32, M); - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/128, M); - } // end transpose -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - return; } - // We separate the quantized bits and scale from block_q4_0 by using an - // additional kernel, where each thread handles a block. We first read the - // original weights into a temporary buffer, then create two separate - // buffers for quantized bits and scales, which are then populated by the - // conversion kernel. - if (tensor->type == GGML_TYPE_Q4_0) { - // Views can't SoA-ify here — parent owns the layout (see q8_0 guard). + if (tensor->type == GGML_TYPE_Q8_0) { + // Views share the parent's buffer; parent owns SoA conversion. + // A view of an AoS tensor (the KV cache, written by a state restore) + // is written in place. if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { + if (!ggml_cl_is_q8_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueWriteBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + } return; } - // Tensors should have been preallocated, therefore they should - // already have ggml_tensor_extra_cl as extra. + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q4_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_0(); + ggml_tensor_extra_cl_q8_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q8_0(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)*sizeof(char)); GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); - // We consider the specified offset arg as always, although For weights - // the offset arg should be 0 (we do not assert this). - //GGML_ASSERT(offset == 0); - - // We create subbuffers from the original tensor buffer for scales and - // quants - i.e., scales and quants are aliases into the buffer object - // that backs the original tensor. This is a cleaner way to adapt to the - // new memory management. - // In the old code, we allocate new buffers for scales and quants - // respectively, which could still be done but would result in double - // allocation; properly deallocating the preallocated buffer that backs - // the tensors is tricky and would leak the backend specific information - // into the general backend code. - // Does this create misaligned subbuffers (alignment is 1024) in certain - // cases ? - cl_buffer_region region; - // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. + cl_buffer_region region; + // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; @@ -10028,55 +14911,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Adreno moe q4_0 kernel needs special transpose and unshuffling - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0_trans4_ns; - - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); - - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; - - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); - - // Create image for Q - cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_q = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast<size_t>(ggml_nelements(tensor) / 8), - 0, 0, 0, 0, 0, 0, 0, - { extra->q } - }; - extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); - tensor->extra = extra; - // MoE tensors are also SOA'ed - ctx->q4_0_soa_tensors.insert(tensor); - - return; - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; + cl_kernel kernel = backend_ctx->kernel_convert_block_q8_0; - // The optimized kernels need weights in natural order, so unshuffle. - if (use_adreno_kernels(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q4_0_noshuffle; - } -#else - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; -#endif // GGML_OPENCL_USE_ADRENO_KERNELS CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); @@ -10090,62 +14926,65 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; - ctx->q4_0_soa_tensors.insert(tensor); + ctx->q8_0_soa_tensors.insert(tensor); - // transpose the weights and scales + // Generic dp4a MoE path (opt-in GGML_OPENCL_Q8_MOE_DP4A) #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Only do transpose for large, non batched matrix - // TODO: use preallocated images instead of sub-buffer then image - if (use_adreno_kernels(backend_ctx, tensor)) { - int M = tensor->ne[1]; - int K = tensor->ne[0]; - - GGML_ASSERT(K % 32 == 0); + { + static const char * q8dp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A"); + const bool q8dp4a = q8dp4a_env ? (atoi(q8dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q8dp4a && tensor->ne[2] > 1 && (tensor->ne[0] % 32 == 0)) { + int ne00 = (int)tensor->ne[0]; + int ne01 = (int)tensor->ne[1]; + int ne02 = (int)tensor->ne[2]; + size_t nb32 = (size_t)ne00 / 32; + size_t scale_elems = (size_t)ne02 * ne01 * nb32 * 2; // 2 per-16-seg scales / 32-block + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, scale_elems * sizeof(cl_half), NULL, &err); + CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q8_0; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } +#endif - if (use_q4_0_bin_kernels(backend_ctx, tensor)) { - cl_int err; - cl_image_format wimg_fmt; - cl_image_desc wimg_desc; + // Transpose the weights and scales +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { - // transpose quants as 32-bit words (M-first) - GGML_ASSERT(M % 64 == 0); - transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K / 8, M); - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K / 32, M); + int M = tensor->ne[1]; // ne01 + int K = tensor->ne[0]; // ne00 - wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&wimg_desc, 0, sizeof(wimg_desc)); - wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - wimg_desc.image_width = (size_t)M * K / 8; - wimg_desc.buffer = extra->q; - CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + GGML_ASSERT(K % 32 == 0); + GGML_ASSERT(M % 4 == 0); + GGML_ASSERT(tensor->ne[2] == 1); + GGML_ASSERT(tensor->ne[3] == 1); - wimg_fmt = { CL_R, CL_HALF_FLOAT }; - memset(&wimg_desc, 0, sizeof(wimg_desc)); - wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - wimg_desc.image_width = (size_t)M * K / 32; - wimg_desc.buffer = extra->d; - CL_CHECK((extra->d_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); - } else { - // Transpose q and d as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); - } - } + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } // end transpose #endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; } - if (tensor->type == GGML_TYPE_Q4_1) { + if (tensor->type == GGML_TYPE_IQ4_NL) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); - // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q4_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_1(); + ggml_tensor_extra_cl_iq4_nl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq4_nl(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/2); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); @@ -10153,8 +14992,6 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, cl_buffer_region region; - // The original tensor memory is divided into scales and quants, i.e., - // we first store scales, mins, then quants. // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; @@ -10164,124 +15001,159 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(err); auto previous_origin = region.origin; - // Create subbuffer for mins. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_m; - extra->m = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); + #ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_iq4_nl; + if (use_adreno_kernels(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_iq4_nl_noshuffle; + } + #else + cl_kernel kernel = backend_ctx->kernel_convert_block_iq4_nl; + #endif + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Adreno moe q4_1 kernel needs special transpose and unshuffling - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1_trans4_ns; + if (use_adreno_kernels(backend_ctx, tensor)) { + int M = tensor->ne[1]; + int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } +#endif + return; + } + if (tensor->type == GGML_TYPE_IQ4_XS) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq4_xs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq4_xs(); + extra->noshuffle = ggml_cl_iq4_xs_use_noshuffle(backend_ctx, tensor); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; - // Create image for Q - cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_q = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast<size_t>(ggml_nelements(tensor) / 8), - 0, 0, 0, 0, 0, 0, 0, - { extra->q } - }; - extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); - tensor->extra = extra; + const size_t size_qs = n_blk * 128; + const size_t size_sl = n_blk * 4; + const size_t size_sh = n_blk * 2; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sl + size_sh + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - return; - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sl; + CL_CHECK((extra->sl = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // normal q4_1 repack -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; + region.origin = align_to(region.origin + size_sl, backend_ctx->alignment); + region.size = size_sh; + CL_CHECK((extra->sh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - if (use_adreno_kernels(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q4_1_noshuffle; - } -#else - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + region.origin = align_to(region.origin + size_sh, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_iq4_xs_noshuffle : backend_ctx->kernel_convert_block_iq4_xs; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sl)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sh)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); CL_CHECK(clReleaseMemObject(data_device)); - tensor->extra = extra; - #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - - int M = tensor->ne[1]; - int K = tensor->ne[0]; + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sl, extra->sl, size_sl, k/256, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->sh, extra->sh, size_sh, k/256, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/256, m, true, true); + } +#endif - GGML_ASSERT(K % 32 == 0); + extra->size_qs = size_qs; + extra->size_sl = size_sl; + extra->size_sh = size_sh; + extra->size_d = size_d; - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); - // Transpose m as ushort - transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + tensor->extra = extra; return; } - if (tensor->type == GGML_TYPE_Q5_0) { + + if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q5_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_0(); + ggml_tensor_extra_cl_q4_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_K(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_qs = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(int32_t); - GGML_ASSERT(size_d + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); + size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3 * ggml_blck_size(tensor->type) / 64); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + GGML_ASSERT(size_d + size_dm + size_s + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); - GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + GGML_ASSERT(data_device != NULL && "q4_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; - // Create subbuffer for scales. + // Create subbuffer for d. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( @@ -10290,127 +15162,129 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(err); auto previous_origin = region.origin; - // Create subbuffer for qh. + // Create subbuffer for mins. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_qh; - extra->qh = clCreateSubBuffer( + region.size = size_dm; + extra->dm = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; - // Create subbuffer for qs. - region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); - region.size = size_qs; - extra->qs = clCreateSubBuffer( + // Create subbuffer for s. + region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); + region.size = size_s; + extra->s = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Adreno moe q5_0 kernel needs special transpose and unshuffling if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_trans4_ns; + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_k_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; + size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); - // Create image for Q - cl_image_format img_format_qs = {CL_R, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_qs = { + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, - { extra->qs } + { extra->q } }; - extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + CL_CHECK(err); tensor->extra = extra; - // Generic dp4a MoE path - { - static const char * q5dp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A"); - const bool q5dp4a = q5dp4a_env ? (atoi(q5dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); - if (q5dp4a && ne02 > 1 && (ne00 % 32 == 0)) { - size_t nb32 = (size_t)ne00 / 32; - size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; - size_t mn_elems = (size_t)ne02 * ne01 * nb32; - extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); - extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); - cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_0; - CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); - CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->min)); - CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(ek, 4, sizeof(int), &ne01)); - size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; - size_t el[3] = { 64, 1, 1 }; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - } - } - return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed/ + // output q4_K weight (token_embd.weight, ne1=vocab) is NOT matched by + // use_adreno_moe_kernels, so it lands here in the general branch. Produce + // the final 64-row-tiled canonical layout directly into q/d/dm/s (buffer + // sizes already match), read back by kernel_gemv_noshuffle_q4_k_f32_tiled. + if (use_q4k_tiled(backend_ctx, tensor)) { + cl_kernel tk = backend_ctx->kernel_convert_block_q4_k_tiled_ns; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; - tensor->extra = extra; + CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(tk, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(tk, 6, sizeof(int), &ne01)); - int M = tensor->ne[1]; - int K = tensor->ne[0]; - GGML_ASSERT(K % 32 == 0); + size_t gws[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; + size_t lws[] = {64, 1, 1}; - // Transpose qs as ushort - transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M); - // Transpose qh as uchar - transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + cl_event tevt; + CL_CHECK(clEnqueueNDRangeKernel(queue, tk, 3, NULL, gws, lws, 0, NULL, &tevt)); + CL_CHECK(clWaitForEvents(1, &tevt)); + CL_CHECK(clReleaseMemObject(data_device)); + extra->q_img = nullptr; + tensor->extra = extra; return; } + + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q4_K_noshuffle; + } +#else + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; #endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &n_blk)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); - size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1}; + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; @@ -10418,32 +15292,66 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); - tensor->extra = extra; + tensor->extra = extra; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { + + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + GGML_ASSERT(K % 32 == 0); + + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + } + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); + transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); + + // Transpose s as uchar + transpose_2d_as_8b(backend_ctx, extra->s, extra->s, size_s, K/256*12, M, true, true); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS return; } - if (tensor->type == GGML_TYPE_Q5_1) { + if (tensor->type == GGML_TYPE_Q5_K) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q5_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_1(); - - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_qs = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(int32_t); - GGML_ASSERT(size_d + size_m + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); + ggml_tensor_extra_cl_q5_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_K(); + + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/8; + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3*ggml_blck_size(tensor->type)/64); + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + GGML_ASSERT(size_q + size_qh + size_s + size_d + size_dm == ggml_nbytes(tensor) && + "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); - GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + GGML_ASSERT(data_device != NULL && "q5_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; - // The original tensor memory is divided into scales and quants, i.e., - // we first store scales, mins, then quants. - // Create subbuffer for scales. + // Create subbuffer for d. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( @@ -10452,1179 +15360,1528 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(err); auto previous_origin = region.origin; - // Create subbuffer for mins. + // Create subbuffer for dm. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_m; - extra->m = clCreateSubBuffer( + region.size = size_dm; + extra->dm = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; - // Create subbuffer for qh. - region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); - region.size = size_qh; - extra->qh = clCreateSubBuffer( + // Create subbuffer for s. + region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); + region.size = size_s; + extra->s = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; - // Create subbuffer for qs. - region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); - region.size = size_qs; - extra->qs = clCreateSubBuffer( + // Create subbuffer for q (lower 4 bits) + region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for qh (upper 1 bit) + region.origin = align_to(previous_origin + size_q, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Adreno moe q5_1 kernel needs special transpose and unshuffling if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_trans4_ns; + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_k_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_F0)); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; + size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); - // Create image for Q - cl_image_format img_format_qs = {CL_R, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_qs = { + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, - { extra->qs } + { extra->q } }; - extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + CL_CHECK(err); tensor->extra = extra; + // Generic dp4a MoE path + { + static const char * q5kdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A"); + const bool q5kdp4a = q5kdp4a_env ? (atoi(q5kdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q5kdp4a && ne02 > 1 && (ne00 % 256 == 0)) { + size_t nb32 = (size_t)ne00 / 32; + size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; + size_t mn_elems = (size_t)ne02 * ne01 * nb32; + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_K; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 4, sizeof(cl_mem), &extra->min)); + CL_CHECK(clSetKernelArg(ek, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 6, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), (size_t)(ne00 / 256), (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } + return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; + if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle; + } +#else + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; +#endif - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); - tensor->extra = extra; + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + extra->size_q = size_q; + extra->size_qh = size_qh; + extra->size_s = size_s; + extra->size_d = size_d; + extra->size_dm = size_dm; + + tensor->extra = extra; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); - // Transpose qs as ushort - transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M); - // Transpose qh as uchar - transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); - // Transpose m as ushort - transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); + if (use_q5_k_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; - return; + // transpose q as 32-bit words (M-first); qh/d/dm stay in their existing layout + // (both new ILA kernels read qh via the existing [K/8][M] uchar plane directly). + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + // Transpose s as uchar + transpose_2d_as_8b(backend_ctx, extra->s, extra->s, size_s, K/256*12, M, true, true); + } else { + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + } + transpose_2d_as_8b (backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); + transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); + return; + } + if (tensor->type == GGML_TYPE_Q6_K) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q6_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q6_K(); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && + "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q6_K set_tensor: temp upload buffer alloc failed"); + + cl_buffer_region region; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno MoE Q6_K kernel needs special transposed layout + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + size_t moe_size_ql = (size_t)(ggml_nelements(tensor) / 8) * sizeof(uint32_t); // 4 bits per element + size_t moe_size_qh = (size_t)(ggml_nelements(tensor) / 16) * sizeof(uint32_t); // 2 bits per element + size_t moe_size_s = size_s; + size_t moe_size_d = size_d; + + // Subbuffer for ql + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = moe_size_ql; + CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + auto previous_origin = region.origin; + + // Subbuffer for qh + region.origin = align_to(previous_origin + moe_size_ql, backend_ctx->alignment); + region.size = moe_size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + // Subbuffer for scales + region.origin = align_to(previous_origin + moe_size_qh, backend_ctx->alignment); + region.size = moe_size_s; + CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + // Subbuffer for d + region.origin = align_to(previous_origin + moe_size_s, backend_ctx->alignment); + region.size = moe_size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_trans4_ns; + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; - tensor->extra = extra; - return; - } - if (tensor->type == GGML_TYPE_MXFP4) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_mxfp4 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_mxfp4(); + size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; + size_t local_work_size[] = {64, 1, 1}; - size_t size_e = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(char); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); - cl_int err; - cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); - GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + // Create image for ql + cl_image_format img_format_ql = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_ql = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast<size_t>(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->ql } + }; + extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_ql, &img_desc_ql, NULL, &err); + tensor->extra = extra; - // The original tensor memory is divided into scales and quants, i.e., - // we first store scales, then quants. - cl_buffer_region region; + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - // Create subbuffer for scales. + // Subbuffer for ql region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_e; - extra->e = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + region.size = size_ql; + CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); auto previous_origin = region.origin; - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_e, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + // Subbuffer for qh + region.origin = align_to(previous_origin + size_ql, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + // Subbuffer for scales + region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); + region.size = size_s; + CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + // Create subbuffer for d. + region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Adreno moe mxfp4 kernel needs special transpose and unshuffling - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4_trans4_ns; + // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed + // /output q6_K weight (e.g. token_embd.weight, ne1=vocab) is NOT matched by + // use_adreno_moe_kernels, so it lands here in the general branch. Produce + // the final 64-row-tiled canonical layout directly into ql/qh/s/d (buffer + // sizes already match), read back by kernel_gemv_noshuffle_q6_K_f32_tiled. + // Bypasses the plain-SOA convert + per-array transpose below. + if (use_q6k_tiled(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_tiled_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; + size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); - tensor->extra = extra; - - // Create image for Q - cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_q = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast<size_t>(ggml_nelements(tensor) / 8), - 0, 0, 0, 0, 0, 0, 0, - { extra->q } - }; - extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); - tensor->extra = extra; + extra->size_ql = size_ql; + extra->size_qh = size_qh; + extra->size_s = size_s; + extra->size_d = size_d; + tensor->extra = extra; return; } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + // Flatten the weights + cl_kernel kernel; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + kernel = backend_ctx->kernel_convert_block_q6_K; + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q6_K_noshuffle; + } +#else + kernel = backend_ctx->kernel_convert_block_q6_K; #endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); + cl_uchar mask = 0xff; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); - size_t global_work_size[3] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[3] = {64, 1, 1}; + size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); - // Create image for Q - cl_image_format img_format_q = {CL_RG, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_q = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast<size_t>(ggml_nelements(tensor)/32*2), - 0, 0, 0, 0, 0, 0, 0, - { extra->q } - }; - extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); - tensor->extra = extra; + extra->size_ql = size_ql; + extra->size_qh = size_qh; + extra->size_s = size_s; + extra->size_d = size_d; - return; - } - if (tensor->type == GGML_TYPE_Q8_0) { - // Views share the parent's buffer; parent owns SoA conversion. - if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { - return; - } + tensor->extra = extra; - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { + cl_int M = tensor->ne[1]; // ne01 + cl_int K = tensor->ne[0]; // ne00 - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q8_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q8_0(); + if (use_q6_k_bin_kernels(backend_ctx, tensor)) { + GGML_ASSERT(K % 256 == 0); + GGML_ASSERT(M % 64 == 0); - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)*sizeof(char)); - GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + transpose_2d_as_32b(backend_ctx, extra->ql, extra->ql, size_ql, K/8, M); + transpose_2d_as_32b(backend_ctx, extra->qh, extra->qh, size_qh, K/16, M); - cl_int err; - cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); - GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + cl_image_format wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc wimg_desc; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = static_cast<size_t>(ggml_nelements(tensor) / 8); + wimg_desc.buffer = extra->ql; + CL_CHECK((extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); - // The original tensor memory is divided into scales and quants, i.e., - // we first store scales, then quants. - cl_buffer_region region; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = static_cast<size_t>(ggml_nelements(tensor) / 16); + wimg_desc.buffer = extra->qh; + CL_CHECK((extra->qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose ql as ushort + transpose_2d_as_16b(backend_ctx, + extra->ql, extra->ql, size_ql, K/4, M); - // Create subbuffer for scales. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + // Transpose qh as uchar + transpose_2d_as_8b(backend_ctx, + extra->qh, extra->qh, size_qh, K/4, M); - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + // Transpose s as ushort + transpose_2d_as_16b(backend_ctx, + extra->s, extra->s, size_s, K/16/2, M); + } + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, + extra->d, extra->d, size_d, K/256, M); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; + } +#endif // GGML_OPENCL_SOA_Q + + // convert bf16 to f16 and store as f16 in device buffer + if (tensor->type == GGML_TYPE_BF16) { + GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0 + && "Offset and size must be multiples of 2 for bf16 tensors"); + + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); + + cl_ulong n_elements = size / sizeof(ggml_fp16_t); + cl_ulong off_dst = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, + size, const_cast<void *>(data), &err); CL_CHECK(err); - cl_kernel kernel = backend_ctx->kernel_convert_block_q8_0; - + cl_kernel kernel = backend_ctx->kernel_convert_bf16_to_f16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); - - tensor->extra = extra; - ctx->q8_0_soa_tensors.insert(tensor); - - // Generic dp4a MoE path (opt-in GGML_OPENCL_Q8_MOE_DP4A) -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - { - static const char * q8dp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A"); - const bool q8dp4a = q8dp4a_env ? (atoi(q8dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); - if (q8dp4a && tensor->ne[2] > 1 && (tensor->ne[0] % 32 == 0)) { - int ne00 = (int)tensor->ne[0]; - int ne01 = (int)tensor->ne[1]; - int ne02 = (int)tensor->ne[2]; - size_t nb32 = (size_t)ne00 / 32; - size_t scale_elems = (size_t)ne02 * ne01 * nb32 * 2; // 2 per-16-seg scales / 32-block - extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, scale_elems * sizeof(cl_half), NULL, &err); - CL_CHECK(err); - cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q8_0; - CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); - CL_CHECK(clSetKernelArg(ek, 2, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne01)); - size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; - size_t el[3] = { 64, 1, 1 }; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - } - } -#endif - - // Transpose the weights and scales -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (enable_adreno_trans_weight(backend_ctx, tensor)) { - - int M = tensor->ne[1]; // ne01 - int K = tensor->ne[0]; // ne00 - - GGML_ASSERT(K % 32 == 0); - GGML_ASSERT(M % 4 == 0); - GGML_ASSERT(tensor->ne[2] == 1); - GGML_ASSERT(tensor->ne[3] == 1); - - transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); - } // end transpose -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK(clReleaseEvent(evt)); return; } - if (tensor->type == GGML_TYPE_IQ4_NL) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_iq4_nl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq4_nl(); + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/2); - GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + // a view reuses its parent's extra, whose offset does not include view_offs + CL_CHECK(clEnqueueWriteBuffer( + queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); - cl_int err; - cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); - GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + GGML_UNUSED(buffer); +} - cl_buffer_region region; +static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + GGML_ASSERT(tensor->extra); - // Create subbuffer for scales. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; + ggml_backend_opencl_context *backend_ctx = dev_ctx->backend_ctx; - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + cl_context context = backend_ctx->context; + cl_command_queue queue = backend_ctx->queue; - #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_iq4_nl; - if (use_adreno_kernels(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_iq4_nl_noshuffle; - } - #else - cl_kernel kernel = backend_ctx->kernel_convert_block_iq4_nl; - #endif - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + // Make sure all previously submitted commands in other devices are finished. + sync_with_other_backends(backend_ctx); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); +#ifdef GGML_OPENCL_SOA_Q + if (tensor->type == GGML_TYPE_Q2_K) { + ggml_tensor_extra_cl_q2_K * extra = (ggml_tensor_extra_cl_q2_K *)tensor->extra; - size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + const size_t blck = (size_t)ggml_blck_size(tensor->type); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - tensor->extra = extra; + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + cl_mem qs = extra->qs, sc = extra->sc, dm = extra->dm; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - int M = tensor->ne[1]; - int K = tensor->ne[0]; - GGML_ASSERT(K % 32 == 0); - - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + ggml_cl_buffer buf_qs, buf_sc, buf_dm; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_sc.allocate(context, extra->size_sc); + buf_dm.allocate(context, extra->size_dm); + qs = buf_qs.buffer; + sc = buf_sc.buffer; + dm = buf_dm.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/16, true, true); + transpose_2d_as_32b(backend_ctx, extra->dm, dm, extra->size_dm, m, k/(int)blck, true, true); } #endif + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q2_k_noshuffle + : backend_ctx->kernel_restore_block_q2_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } - if (tensor->type == GGML_TYPE_Q4_K) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q4_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_K(); + if (tensor->type == GGML_TYPE_Q3_K) { + ggml_tensor_extra_cl_q3_K * extra = (ggml_tensor_extra_cl_q3_K *)tensor->extra; - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3 * ggml_blck_size(tensor->type) / 64); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - GGML_ASSERT(size_d + size_dm + size_s + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + const size_t blck = (size_t)ggml_blck_size(tensor->type); - cl_int err; - cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); - GGML_ASSERT(data_device != NULL && "q4_K set_tensor: temp upload buffer alloc failed"); + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - cl_buffer_region region; + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; - // Create subbuffer for d. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + cl_mem qs = extra->qs, hm = extra->hm, sc = extra->sc, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_hm, buf_sc, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_hm.allocate(context, extra->size_hm); + buf_sc.allocate(context, extra->size_sc); + buf_d.allocate(context, extra->size_d); + qs = buf_qs.buffer; + hm = buf_hm.buffer; + sc = buf_sc.buffer; + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_8b(backend_ctx, extra->hm, hm, extra->size_hm, m, k/8, true, true); + transpose_2d_as_32b(backend_ctx, extra->sc, sc, extra->size_sc, m, 3*k/(int)blck, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + } +#endif + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q3_k_noshuffle + : backend_ctx->kernel_restore_block_q3_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } - // Create subbuffer for mins. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_dm; - extra->dm = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + if (tensor->type == GGML_TYPE_IQ3_S) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq3_s * extra = (ggml_tensor_extra_cl_iq3_s *)tensor->extra; - // Create subbuffer for s. - region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); - region.size = size_s; - extra->s = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + const size_t blck = (size_t)ggml_blck_size(tensor->type); - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + cl_mem qs = extra->qs, sg = extra->sg, qh = extra->qh, sc = extra->sc, d = extra->d; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_k_trans4_ns; + ggml_cl_buffer buf_qs, buf_sg, buf_qh, buf_sc, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_sg.allocate(context, extra->size_sg); + buf_qh.allocate(context, extra->size_qh); + buf_sc.allocate(context, extra->size_sc); + buf_d.allocate(context, extra->size_d); + + qs = buf_qs.buffer; + sg = buf_sg.buffer; + qh = buf_qh.buffer; + sc = buf_sc.buffer; + d = buf_d.buffer; + + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_8b(backend_ctx, extra->sg, sg, extra->size_sg, m, k/8, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/32, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/64, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/256, true, true); + } +#endif - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; + cl_kernel kernel = backend_ctx->kernel_restore_block_iq3_s; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); + if (tensor->type == GGML_TYPE_IQ3_XXS) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq3_xxs * extra = (ggml_tensor_extra_cl_iq3_xxs *)tensor->extra; - size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; - size_t local_work_size[] = {64, 1, 1}; + const size_t blck = (size_t)ggml_blck_size(tensor->type); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_q = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast<size_t>(ggml_nelements(tensor) / 8), - 0, 0, 0, 0, 0, 0, 0, - { extra->q } - }; - extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); - CL_CHECK(err); - tensor->extra = extra; + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; - return; + cl_mem qs = extra->qs, sas = extra->sas, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sas, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_sas.allocate(context, extra->size_sas); + buf_d.allocate(context, extra->size_d); + + qs = buf_qs.buffer; + sas = buf_sas.buffer; + d = buf_d.buffer; + + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_32b(backend_ctx, extra->sas, sas, extra->size_sas, m, k/32, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/256, true, true); } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS +#endif -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed/ - // output q4_K weight (token_embd.weight, ne1=vocab) is NOT matched by - // use_adreno_moe_kernels, so it lands here in the general branch. Produce - // the final 64-row-tiled canonical layout directly into q/d/dm/s (buffer - // sizes already match), read back by kernel_gemv_noshuffle_q4_k_f32_tiled. - if (use_q4k_tiled(backend_ctx, tensor)) { - cl_kernel tk = backend_ctx->kernel_convert_block_q4_k_tiled_ns; + cl_kernel kernel = backend_ctx->kernel_restore_block_iq3_xxs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } - CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(tk, 5, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(tk, 6, sizeof(int), &ne01)); + if (tensor->type == GGML_TYPE_IQ1_M) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq1_m * extra = (ggml_tensor_extra_cl_iq1_m *)tensor->extra; - size_t gws[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; - size_t lws[] = {64, 1, 1}; + const size_t blck = (size_t)ggml_blck_size(tensor->type); - cl_event tevt; - CL_CHECK(clEnqueueNDRangeKernel(queue, tk, 3, NULL, gws, lws, 0, NULL, &tevt)); - CL_CHECK(clWaitForEvents(1, &tevt)); - CL_CHECK(clReleaseMemObject(data_device)); + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - extra->q_img = nullptr; - tensor->extra = extra; - return; + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, qh = extra->qh, sc = extra->sc; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_qh, buf_sc; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + qs = buf_qs.buffer; + buf_qh.allocate(context, extra->size_qh); + qh = buf_qh.buffer; + buf_sc.allocate(context, extra->size_sc); + sc = buf_sc.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/16, true, true); + transpose_2d_as_16b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/64, true, true); } +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_iq1_m; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q4_K_noshuffle; + if (tensor->type == GGML_TYPE_IQ1_S) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; } -#else - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + ggml_tensor_extra_cl_iq1_s * extra = (ggml_tensor_extra_cl_iq1_s *)tensor->extra; - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + const size_t blck = (size_t)ggml_blck_size(tensor->type); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + cl_mem qs = extra->qs, qh = extra->qh, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_qh, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + qs = buf_qs.buffer; + buf_qh.allocate(context, extra->size_qh); + qh = buf_qh.buffer; + buf_d.allocate(context, extra->size_d); + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); + transpose_2d_as_16b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/32, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + } +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_iq1_s; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } - tensor->extra = extra; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { + if (tensor->type == GGML_TYPE_IQ2_S) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq2_s * extra = (ggml_tensor_extra_cl_iq2_s *)tensor->extra; - int M = tensor->ne[1]; - int K = tensor->ne[0]; + const size_t blck = (size_t)ggml_blck_size(tensor->type); - GGML_ASSERT(K % 32 == 0); + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - if (use_q4_k_bin_kernels(backend_ctx, tensor)) { - cl_int err; - cl_image_format wimg_fmt; - cl_image_desc wimg_desc; + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; - // transpose quants as 32-bit words (M-first) - GGML_ASSERT(M % 64 == 0); - transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + cl_mem qs = extra->qs, sg = extra->sg, qh = extra->qh, sc = extra->sc, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sg, buf_qh, buf_sc, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + qs = buf_qs.buffer; + buf_sg.allocate(context, extra->size_sg); + sg = buf_sg.buffer; + buf_qh.allocate(context, extra->size_qh); + qh = buf_qh.buffer; + buf_sc.allocate(context, extra->size_sc); + sc = buf_sc.buffer; + buf_d.allocate(context, extra->size_d); + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); + transpose_2d_as_8b(backend_ctx, extra->sg, sg, extra->size_sg, m, k/8, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/32, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/32, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + } +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_s; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } - wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&wimg_desc, 0, sizeof(wimg_desc)); - wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - wimg_desc.image_width = (size_t)M * K / 8; - wimg_desc.buffer = extra->q; - CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); - } else { - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - } - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); - transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); + if (tensor->type == GGML_TYPE_IQ2_XS) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq2_xs * extra = (ggml_tensor_extra_cl_iq2_xs *)tensor->extra; - // Transpose s as uchar - transpose_2d_as_8b(backend_ctx, extra->s, extra->s, size_s, K/256*12, M, true, true); + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, sc = extra->sc, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sc, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + qs = buf_qs.buffer; + buf_sc.allocate(context, extra->size_sc); + sc = buf_sc.buffer; + buf_d.allocate(context, extra->size_d); + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_16b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/32, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_xs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } - if (tensor->type == GGML_TYPE_Q5_K) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q5_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_K(); + if (tensor->type == GGML_TYPE_IQ2_XXS) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq2_xxs * extra = (ggml_tensor_extra_cl_iq2_xxs *)tensor->extra; - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/8; - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3*ggml_blck_size(tensor->type)/64); - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - GGML_ASSERT(size_q + size_qh + size_s + size_d + size_dm == ggml_nbytes(tensor) && - "Incorrect tensor size"); + const size_t blck = (size_t)ggml_blck_size(tensor->type); - cl_int err; - cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); - GGML_ASSERT(data_device != NULL && "q5_K set_tensor: temp upload buffer alloc failed"); + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - cl_buffer_region region; + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; - // Create subbuffer for d. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + cl_mem qs = extra->qs, sas = extra->sas, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sas, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + qs = buf_qs.buffer; + buf_sas.allocate(context, extra->size_sas); + sas = buf_sas.buffer; + buf_d.allocate(context, extra->size_d); + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); + transpose_2d_as_32b(backend_ctx, extra->sas, sas, extra->size_sas, m, k/32, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + } +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_xxs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } - // Create subbuffer for dm. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_dm; - extra->dm = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + if (tensor->type == GGML_TYPE_Q1_0) { + ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra; - // Create subbuffer for s. - region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); - region.size = size_s; - extra->s = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_q; + ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_unpacked; - // Create subbuffer for q (lower 4 bits) - region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + int M = tensor->ne[1]; + int K = tensor->ne[0]; - // Create subbuffer for qh (upper 1 bit) - region.origin = align_to(previous_origin + size_q, backend_ctx->alignment); - region.size = size_qh; - CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - CL_CHECK(err); + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_k_trans4_ns; + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/32); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/128); - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_F0)); + cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); - size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; - size_t local_work_size[] = {64, 1, 1}; + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } +#endif - cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_q = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast<size_t>(ggml_nelements(tensor) / 8), - 0, 0, 0, 0, 0, 0, 0, - { extra->q } - }; - extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); - CL_CHECK(err); - tensor->extra = extra; + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); - // Generic dp4a MoE path - { - static const char * q5kdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A"); - const bool q5kdp4a = q5kdp4a_env ? (atoi(q5kdp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); - if (q5kdp4a && ne02 > 1 && (ne00 % 256 == 0)) { - size_t nb32 = (size_t)ne00 / 32; - size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; - size_t mn_elems = (size_t)ne02 * ne01 * nb32; - extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); - extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); - cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_K; - CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(ek, 3, sizeof(cl_mem), &extra->scale)); - CL_CHECK(clSetKernelArg(ek, 4, sizeof(cl_mem), &extra->min)); - CL_CHECK(clSetKernelArg(ek, 5, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(ek, 6, sizeof(int), &ne01)); - size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), (size_t)(ne00 / 256), (size_t)ne02 }; - size_t el[3] = { 64, 1, 1 }; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - } - } + cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } - return; + if (tensor->type == GGML_TYPE_Q2_0) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + ggml_tensor_extra_cl_q2_0 * extra = (ggml_tensor_extra_cl_q2_0 *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, d = extra->d; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; - if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle; + ggml_cl_buffer buf_qs, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + qs = buf_qs.buffer; + buf_d.allocate(context, extra->size_d); + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/64, true, true); } -#else - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; #endif - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); - - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + cl_kernel kernel = backend_ctx->kernel_restore_block_q2_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &nb_arg)); + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } - extra->size_q = size_q; - extra->size_qh = size_qh; - extra->size_s = size_s; - extra->size_d = size_d; - extra->size_dm = size_dm; + // In end-to-end runs, get_tensor is usually used to get back the logits, + // where we can simply do clEnqueueReadBuffer since they are f32. + // However, in test-backend-ops, the GPU graph is copied to the CPU backend, + // which requires reading back quantized weight tensors. + // To properly support this, we need to restore block_q4_0 struct arrays + // from the flattened buffers. + if (tensor->type == GGML_TYPE_Q4_0) { + // KV-cache q4_0 stays AoS — direct readback, no SoA restore. + if (!ggml_cl_is_q4_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueReadBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + return; + } + // SoA extra lives on the parent tensor — follow view_src. + const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; + ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *)extra_src->extra; - tensor->extra = extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_trans4_ns; - int M = tensor->ne[1]; - int K = tensor->ne[0]; + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - GGML_ASSERT(K % 32 == 0); + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); - if (use_q5_k_bin_kernels(backend_ctx, tensor)) { - cl_int err; - cl_image_format wimg_fmt; - cl_image_desc wimg_desc; + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; - // transpose q as 32-bit words (M-first); qh/d/dm stay in their existing layout - // (both new ILA kernels read qh via the existing [K/8][M] uchar plane directly). - GGML_ASSERT(M % 64 == 0); - transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (use_adreno_kernels(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_q; + ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_unpacked; - wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&wimg_desc, 0, sizeof(wimg_desc)); - wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - wimg_desc.image_width = (size_t)M * K / 8; - wimg_desc.buffer = extra->q; - CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + cl_int M = tensor->ne[1]; // ne01 + cl_int K = tensor->ne[0]; // ne00 - // Transpose s as uchar - transpose_2d_as_8b(backend_ctx, extra->s, extra->s, size_s, K/256*12, M, true, true); + GGML_ASSERT(K % 32 == 0); + GGML_ASSERT(M % 4 == 0); + + size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); } else { - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); } - transpose_2d_as_8b (backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); - transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - return; - } - if (tensor->type == GGML_TYPE_Q6_K) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); +#endif - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q6_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q6_K(); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && - "Incorrect tensor size"); + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - cl_int err; - cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); - GGML_ASSERT(data_device != NULL && "q6_K set_tensor: temp upload buffer alloc failed"); + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; - cl_buffer_region region; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q4_1) { + ggml_tensor_extra_cl_q4_1 * extra = (ggml_tensor_extra_cl_q4_1 *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Adreno MoE Q6_K kernel needs special transposed layout if (use_adreno_moe_kernels(backend_ctx, tensor)) { - size_t moe_size_ql = (size_t)(ggml_nelements(tensor) / 8) * sizeof(uint32_t); // 4 bits per element - size_t moe_size_qh = (size_t)(ggml_nelements(tensor) / 16) * sizeof(uint32_t); // 2 bits per element - size_t moe_size_s = size_s; - size_t moe_size_d = size_d; + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_trans4_ns; - // Subbuffer for ql - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = moe_size_ql; - CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - auto previous_origin = region.origin; + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); - // Subbuffer for qh - region.origin = align_to(previous_origin + moe_size_ql, backend_ctx->alignment); - region.size = moe_size_qh; - CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - previous_origin = region.origin; + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; - // Subbuffer for scales - region.origin = align_to(previous_origin + moe_size_qh, backend_ctx->alignment); - region.size = moe_size_s; - CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - previous_origin = region.origin; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (use_adreno_kernels(backend_ctx, tensor)) { + static ggml_cl_buffer buf_trans_q; + static ggml_cl_buffer buf_trans_m; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_unpacked; - // Subbuffer for d - region.origin = align_to(previous_origin + moe_size_s, backend_ctx->alignment); - region.size = moe_size_d; - CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + cl_int M = tensor->ne[1]; + cl_int K = tensor->ne[0]; - cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_trans4_ns; + GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); + + size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + GGML_ASSERT(size_d + size_q + size_m == ggml_nbytes(tensor) && "Incorrect tensor size"); + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_m.allocate(backend_ctx->context, size_m); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + // transpose q, d, m back + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); - - size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; - size_t local_work_size[] = {64, 1, 1}; - - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; - // Create image for ql - cl_image_format img_format_ql = {CL_R, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_ql = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast<size_t>(ggml_nelements(tensor) / 8), - 0, 0, 0, 0, 0, 0, 0, - { extra->ql } - }; - extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_ql, &img_desc_ql, NULL, &err); - tensor->extra = extra; + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_m.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS +#endif - // Subbuffer for ql - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_ql; - CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - auto previous_origin = region.origin; + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - // Subbuffer for qh - region.origin = align_to(previous_origin + size_ql, backend_ctx->alignment); - region.size = size_qh; - CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - previous_origin = region.origin; + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); - // Subbuffer for scales - region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); - region.size = size_s; - CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - previous_origin = region.origin; + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; - // Create subbuffer for d. - region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); - region.size = size_d; - CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - previous_origin = region.origin; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q5_0) { + ggml_tensor_extra_cl_q5_0 * extra = (ggml_tensor_extra_cl_q5_0 *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed - // /output q6_K weight (e.g. token_embd.weight, ne1=vocab) is NOT matched by - // use_adreno_moe_kernels, so it lands here in the general branch. Produce - // the final 64-row-tiled canonical layout directly into ql/qh/s/d (buffer - // sizes already match), read back by kernel_gemv_noshuffle_q6_K_f32_tiled. - // Bypasses the plain-SOA convert + per-array transpose below. - if (use_q6k_tiled(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_tiled_ns; + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + // TODO: use ggml_cl_buffer to manage this temporary buffer + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); - - size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; - size_t local_work_size[] = {64, 1, 1}; + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); - - extra->size_ql = size_ql; - extra->size_qh = size_qh; - extra->size_s = size_s; - extra->size_d = size_d; - tensor->extra = extra; return; } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - - // Flatten the weights - cl_kernel kernel; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - kernel = backend_ctx->kernel_convert_block_q6_K; - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q6_K_noshuffle; - } -#else - kernel = backend_ctx->kernel_convert_block_q6_K; -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - - cl_uchar mask = 0xff; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); - - size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + if (use_adreno_kernels(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_qs; + ggml_cl_buffer buf_trans_qh; + ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_unpacked; - extra->size_ql = size_ql; - extra->size_qh = size_qh; - extra->size_s = size_s; - extra->size_d = size_d; + cl_int M = tensor->ne[1]; + cl_int K = tensor->ne[0]; - tensor->extra = extra; + GGML_ASSERT(K % 32 == 0); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { - cl_int M = tensor->ne[1]; // ne01 - cl_int K = tensor->ne[0]; // ne00 + size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t); + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - if (use_q6_k_bin_kernels(backend_ctx, tensor)) { - GGML_ASSERT(K % 256 == 0); - GGML_ASSERT(M % 64 == 0); + buf_trans_qs.allocate(backend_ctx->context, size_qs); + buf_trans_qh.allocate(backend_ctx->context, size_qh); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - transpose_2d_as_32b(backend_ctx, extra->ql, extra->ql, size_ql, K/8, M); - transpose_2d_as_32b(backend_ctx, extra->qh, extra->qh, size_qh, K/16, M); + transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4); + transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); - cl_image_format wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; - cl_image_desc wimg_desc; - memset(&wimg_desc, 0, sizeof(wimg_desc)); - wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - wimg_desc.image_width = static_cast<size_t>(ggml_nelements(tensor) / 8); - wimg_desc.buffer = extra->ql; - CL_CHECK((extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; - memset(&wimg_desc, 0, sizeof(wimg_desc)); - wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - wimg_desc.image_width = static_cast<size_t>(ggml_nelements(tensor) / 16); - wimg_desc.buffer = extra->qh; - CL_CHECK((extra->qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); - } else { - // Transpose ql as ushort - transpose_2d_as_16b(backend_ctx, - extra->ql, extra->ql, size_ql, K/4, M); + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; - // Transpose qh as uchar - transpose_2d_as_8b(backend_ctx, - extra->qh, extra->qh, size_qh, K/4, M); + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); - // Transpose s as ushort - transpose_2d_as_16b(backend_ctx, - extra->s, extra->s, size_s, K/16/2, M); - } - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, - extra->d, extra->d, size_d, K/256, M); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS - return; - } -#endif // GGML_OPENCL_SOA_Q - - // convert bf16 to f16 and store as f16 in device buffer - if (tensor->type == GGML_TYPE_BF16) { - GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0 - && "Offset and size must be multiples of 2 for bf16 tensors"); - - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra); - - cl_ulong n_elements = size / sizeof(ggml_fp16_t); - cl_ulong off_dst = (extra->offset + offset) / sizeof(ggml_fp16_t); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, - size, const_cast<void *>(data), &err); + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); CL_CHECK(err); - cl_kernel kernel = backend_ctx->kernel_convert_bf16_to_f16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->data_device)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &off_dst)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements)); + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); - size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 }; - size_t local_work_size[] = { 64, 1, 1 }; + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); - CL_CHECK(clReleaseEvent(evt)); - return; } + if (tensor->type == GGML_TYPE_Q5_1) { + ggml_tensor_extra_cl_q5_1 * extra = (ggml_tensor_extra_cl_q5_1 *)tensor->extra; - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra); - - CL_CHECK(clEnqueueWriteBuffer( - queue, extra->data_device, CL_TRUE, extra->offset + offset, - size, data, 0, NULL, NULL)); - - GGML_UNUSED(buffer); -} - -static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - GGML_ASSERT(tensor->extra); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + // TODO: use ggml_cl_buffer to manage this temporary buffer + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; - ggml_backend_opencl_context *backend_ctx = dev_ctx->backend_ctx; + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_trans4_ns; - cl_context context = backend_ctx->context; - cl_command_queue queue = backend_ctx->queue; + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); - // Make sure all previously submitted commands in other devices are finished. - sync_with_other_backends(backend_ctx); + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; -#ifdef GGML_OPENCL_SOA_Q - if (tensor->type == GGML_TYPE_Q1_0) { - ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (enable_adreno_trans_weight(backend_ctx, tensor)) { - ggml_cl_buffer buf_trans_q; + if (use_adreno_kernels(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_qs; + ggml_cl_buffer buf_trans_qh; ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_trans_m; ggml_cl_buffer buf_unpacked; - int M = tensor->ne[1]; - int K = tensor->ne[0]; + cl_int M = tensor->ne[1]; + cl_int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); + size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t); + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_qs.allocate(backend_ctx->context, size_qs); + buf_trans_qh.allocate(backend_ctx->context, size_qh); buf_trans_d.allocate(backend_ctx->context, size_d); + buf_trans_m.allocate(backend_ctx->context, size_m); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/32); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/128); + // Transpose back: from col-major to row-major + transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4); + transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); - cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_m.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } -#endif - +#endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err); + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); CL_CHECK(err); - cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { + ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } - // In end-to-end runs, get_tensor is usually used to get back the logits, - // where we can simply do clEnqueueReadBuffer since they are f32. - // However, in test-backend-ops, the GPU graph is copied to the CPU backend, - // which requires reading back quantized weight tensors. - // To properly support this, we need to restore block_q4_0 struct arrays - // from the flattened buffers. - if (tensor->type == GGML_TYPE_Q4_0) { - // KV-cache q4_0 stays AoS — direct readback, no SoA restore. - if (!ggml_cl_is_q4_0_soa(tensor)) { + if (tensor->type == GGML_TYPE_Q8_0) { + // KV-cache q8_0 stays AoS (see Q4_0 branch). + if (!ggml_cl_is_q8_0_soa(tensor)) { ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; CL_CHECK(clEnqueueReadBuffer( queue, extra_aos->data_device, CL_TRUE, @@ -11632,51 +16889,78 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, size, data, 0, NULL, NULL)); return; } - // SoA extra lives on the parent tensor — follow view_src. + // SoA extra lives on the parent — follow view_src. const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; - ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *)extra_src->extra; + ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)extra_src->extra; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_trans4_ns; + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; + GGML_ASSERT(tensor->ne[2] == 1); + GGML_ASSERT(tensor->ne[3] == 1); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), 1, 1}; + size_t local_work_size[3] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } - if (use_adreno_kernels(backend_ctx, tensor)) { - ggml_cl_buffer buf_trans_q; - ggml_cl_buffer buf_trans_d; - ggml_cl_buffer buf_unpacked; +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - cl_int M = tensor->ne[1]; // ne01 - cl_int K = tensor->ne[0]; // ne00 + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_IQ4_NL) { + ggml_tensor_extra_cl_iq4_nl * extra = (ggml_tensor_extra_cl_iq4_nl *)tensor->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { + static ggml_cl_buffer buf_trans_q; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_unpacked; + + cl_int M = tensor->ne[1]; + cl_int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); - GGML_ASSERT(M % 4 == 0); - size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*(ggml_blck_size(tensor->type)/2); size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); @@ -11684,41 +16968,40 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - if (use_q4_0_bin_kernels(backend_ctx, tensor)) { - transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); - } else { - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); - } + // transpose q, d back + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + cl_kernel kernel = backend_ctx->kernel_restore_block_iq4_nl_noshuffle; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)n_blk, 1, 1}; + size_t local_work_size[] = {1, 1, 1}; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } #endif + cl_kernel kernel = backend_ctx->kernel_restore_block_iq4_nl; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_blk)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t global_work_size[] = {(size_t)n_blk, 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; @@ -11731,27 +17014,140 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } - if (tensor->type == GGML_TYPE_Q4_1) { - ggml_tensor_extra_cl_q4_1 * extra = (ggml_tensor_extra_cl_q4_1 *)tensor->extra; + if (tensor->type == GGML_TYPE_IQ4_XS) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq4_xs * extra = (ggml_tensor_extra_cl_iq4_xs *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, sl = extra->sl, sh = extra->sh, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sl, buf_sh, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_sl.allocate(context, extra->size_sl); + buf_sh.allocate(context, extra->size_sh); + buf_d.allocate(context, extra->size_d); + + qs = buf_qs.buffer; + sl = buf_sl.buffer; + sh = buf_sh.buffer; + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_16b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_32b(backend_ctx, extra->sl, sl, extra->size_sl, m, k/256, true, true); + transpose_2d_as_16b(backend_ctx, extra->sh, sh, extra->size_sh, m, k/256, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/256, true, true); + } +#endif + + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_iq4_xs_noshuffle : backend_ctx->kernel_restore_block_iq4_xs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } + + if (tensor->type == GGML_TYPE_Q4_K) { + ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Undo the 64-row-tiled canonical pack (kernel_convert_block_q4_k_tiled_ns). + // Without this, a read-back of a tiled weight returns the tiled bytes + // reinterpreted as block_q4_K -- which is how test-backend-ops builds its + // CPU reference (ggml_backend_graph_copy -> tensor_get), so the tiled path + // "failed" the suite while computing the correct product. + if (use_q4k_tiled(backend_ctx, tensor)) { + const int ne00v = tensor->ne[0]; + const int ne01v = tensor->ne[1]; + const int nbv = ne00v / 256; + const size_t n_blk = (size_t)nbv * ne01v; + + std::vector<uint32_t> tq(n_blk*32); + std::vector<uint16_t> td(n_blk), tdm(n_blk); + std::vector<uint8_t> ts(n_blk*12); + CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, tq.size()*4, tq.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->dm, CL_TRUE, 0, tdm.size()*2, tdm.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); + + std::vector<uint8_t> rebuilt(ggml_nbytes(tensor), 0); + for (int i01 = 0; i01 < ne01v; ++i01) { + const int rt = i01/64, rit = i01%64; + for (int i00 = 0; i00 < nbv; ++i00) { + uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*144; + const int tb = rt*nbv + i00; + const size_t si = (size_t)tb*64 + rit; + + memcpy(b + 0, &td [si], 2); + memcpy(b + 2, &tdm[si], 2); + memcpy(b + 4, &ts[si*12], 12); + + uint32_t qw[32]; + for (int gr = 0; gr < 8; ++gr) { + const size_t base = ((size_t)tb*8 + gr)*64 + rit; + for (int j = 0; j < 4; ++j) qw[gr*4 + j] = tq[base*4 + j]; + } + uint8_t * q = b + 16; + for (int e = 0; e < 256; ++e) { + const int g = e>>6, w = e&63, h = w>>5, l = w&31; + const uint32_t code = (qw[e>>3] >> ((e&7)*4)) & 0xF; + q[g*32 + l] |= (uint8_t)(h ? (code << 4) : code); + } + } + } + memcpy(data, rebuilt.data() + offset, size); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_trans4_ns; + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_k_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; + size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, @@ -11763,60 +17159,64 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } - if (use_adreno_kernels(backend_ctx, tensor)) { - static ggml_cl_buffer buf_trans_q; - static ggml_cl_buffer buf_trans_m; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_unpacked; - - cl_int M = tensor->ne[1]; - cl_int K = tensor->ne[0]; + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { + int M = tensor->ne[1]; + int K = tensor->ne[0]; - GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*12; - size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; - size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - GGML_ASSERT(size_d + size_q + size_m == ggml_nbytes(tensor) && "Incorrect tensor size"); + static ggml_cl_buffer buf_trans_q; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_trans_dm; + static ggml_cl_buffer buf_trans_s; buf_trans_q.allocate(backend_ctx->context, size_q); - buf_trans_m.allocate(backend_ctx->context, size_m); buf_trans_d.allocate(backend_ctx->context, size_d); - buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + buf_trans_dm.allocate(backend_ctx->context, size_dm); + buf_trans_s.allocate(backend_ctx->context, size_s); - // transpose q, d, m back - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); - transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); + // Transpose q, d, dm, s back + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + } + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); + transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); + transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_s.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_dm.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_m.buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); - - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); return; } -#endif - - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1; + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; @@ -11831,29 +17231,37 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } - if (tensor->type == GGML_TYPE_Q5_0) { - ggml_tensor_extra_cl_q5_0 * extra = (ggml_tensor_extra_cl_q5_0 *)tensor->extra; + if (tensor->type == GGML_TYPE_Q5_K) { + ggml_tensor_extra_cl_q5_K * extra = (ggml_tensor_extra_cl_q5_K *)tensor->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - // TODO: use ggml_cl_buffer to manage this temporary buffer cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_trans4_ns; + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_k_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_F0)); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; + size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, @@ -11865,60 +17273,73 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } - if (use_adreno_kernels(backend_ctx, tensor)) { - ggml_cl_buffer buf_trans_qs; - ggml_cl_buffer buf_trans_qh; - ggml_cl_buffer buf_trans_d; - ggml_cl_buffer buf_unpacked; - - cl_int M = tensor->ne[1]; - cl_int K = tensor->ne[0]; + if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { + int M = tensor->ne[1]; + int K = tensor->ne[0]; - GGML_ASSERT(K % 32 == 0); + size_t size_q = extra->size_q; + size_t size_qh = extra->size_qh; + size_t size_s = extra->size_s; + size_t size_d = extra->size_d; + size_t size_dm = extra->size_dm; - size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; - size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t); - size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + static ggml_cl_buffer buf_trans_q; + static ggml_cl_buffer buf_trans_qh; + static ggml_cl_buffer buf_trans_s; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_trans_dm; - buf_trans_qs.allocate(backend_ctx->context, size_qs); + buf_trans_q.allocate(backend_ctx->context, size_q); buf_trans_qh.allocate(backend_ctx->context, size_qh); + buf_trans_s.allocate(backend_ctx->context, size_s); buf_trans_d.allocate(backend_ctx->context, size_d); - buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + buf_trans_dm.allocate(backend_ctx->context, size_dm); - transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4); - transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + // bin kernel transposes s but src kernel does not + cl_mem buf_s = extra->s; - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + if (use_q5_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); + transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); + buf_s = buf_trans_s.buffer; + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + } + transpose_2d_as_8b (backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); + transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_trans_dm.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); - - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; @@ -11933,30 +17354,89 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } - if (tensor->type == GGML_TYPE_Q5_1) { - ggml_tensor_extra_cl_q5_1 * extra = (ggml_tensor_extra_cl_q5_1 *)tensor->extra; + if (tensor->type == GGML_TYPE_Q6_K) { + ggml_tensor_extra_cl_q6_K * extra = (ggml_tensor_extra_cl_q6_K *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Undo the 64-row-tiled canonical pack (kernel_convert_block_q6_k_tiled_ns). + // See the q4_K tiled restore above for why a read-back path is required. + if (use_q6k_tiled(backend_ctx, tensor)) { + const int ne00v = tensor->ne[0]; + const int ne01v = tensor->ne[1]; + const int nbv = ne00v / 256; + const size_t n_blk = (size_t)nbv * ne01v; + + std::vector<uint32_t> tql(n_blk*32), tqh(n_blk*16); + std::vector<uint8_t> ts(n_blk*16); + std::vector<uint16_t> td(n_blk); + CL_CHECK(clEnqueueReadBuffer(queue, extra->ql, CL_TRUE, 0, tql.size()*4, tql.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->qh, CL_TRUE, 0, tqh.size()*4, tqh.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); + + std::vector<uint8_t> rebuilt(ggml_nbytes(tensor), 0); + for (int i01 = 0; i01 < ne01v; ++i01) { + const int rt = i01/64, rit = i01%64; + for (int i00 = 0; i00 < nbv; ++i00) { + uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*210; + const int tb = rt*nbv + i00; + const size_t si = (size_t)tb*64 + rit; + + uint32_t qlw[32], qhw[16]; + for (int g = 0; g < 8; ++g) { + const size_t base = ((size_t)tb*8 + g)*64 + rit; + for (int j = 0; j < 4; ++j) qlw[g*4 + j] = tql[base*4 + j]; + } + for (int g = 0; g < 4; ++g) { + const size_t base = ((size_t)tb*4 + g)*64 + rit; + for (int j = 0; j < 4; ++j) qhw[g*4 + j] = tqh[base*4 + j]; + } + + uint8_t * ql = b; + uint8_t * qh = b + 128; + for (int e = 0; e < 256; ++e) { + const int n = (e >= 128) ? 1 : 0; + const int within = e - n*128, q = within/32, l = within%32; + const int off_ql = n*64, off_qh = n*32; + const uint8_t low4 = (qlw[e>>3] >> ((e&7)*4)) & 0xF; + const uint8_t hi2 = (qhw[e>>4] >> ((e&15)*2)) & 0x3; + if (q == 0) ql[off_ql + l] |= low4; + else if (q == 1) ql[off_ql + l + 32] |= low4; + else if (q == 2) ql[off_ql + l] |= (uint8_t)(low4 << 4); + else ql[off_ql + l + 32] |= (uint8_t)(low4 << 4); + qh[off_qh + l] |= (uint8_t)(hi2 << (q*2)); + } + memcpy(b + 192, &ts[si*16], 16); + memcpy(b + 208, &td[si], 2); + } + } + memcpy(data, rebuilt.data() + offset, size); + return; + } if (use_adreno_moe_kernels(backend_ctx, tensor)) { - // TODO: use ggml_cl_buffer to manage this temporary buffer cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_trans4_ns; + cl_kernel kernel = backend_ctx->kernel_restore_block_q6_k_trans4_ns; + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; + size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, @@ -11968,120 +17448,84 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { + static ggml_cl_buffer buf_trans_ql; + static ggml_cl_buffer buf_trans_qh; + static ggml_cl_buffer buf_trans_s; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_unpacked; - if (use_adreno_kernels(backend_ctx, tensor)) { - ggml_cl_buffer buf_trans_qs; - ggml_cl_buffer buf_trans_qh; - ggml_cl_buffer buf_trans_d; - ggml_cl_buffer buf_trans_m; - ggml_cl_buffer buf_unpacked; + cl_int M = tensor->ne[1]; // ne01 + cl_int K = tensor->ne[0]; // ne00 - cl_int M = tensor->ne[1]; - cl_int K = tensor->ne[0]; - GGML_ASSERT(K % 32 == 0); + GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); - size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; - size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t); - size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); - buf_trans_qs.allocate(backend_ctx->context, size_qs); + buf_trans_ql.allocate(backend_ctx->context, size_ql); buf_trans_qh.allocate(backend_ctx->context, size_qh); buf_trans_d.allocate(backend_ctx->context, size_d); - buf_trans_m.allocate(backend_ctx->context, size_m); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - // Transpose back: from col-major to row-major - transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4); - transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); - transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); - - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; - - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; - - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_m.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); - - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); - return; - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; - - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (tensor->type == GGML_TYPE_MXFP4) { - ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra; - - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4_trans4_ns; - - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + cl_mem s_buffer; + if (use_q6_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/8); + transpose_2d_as_32b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/16); + // s is left row-major, untransposed, for the binary layout. + s_buffer = extra->s; + } else { + // transpose ql, qh, s and d back + buf_trans_s.allocate(backend_ctx->context, size_s); + transpose_2d_as_16b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/4); + transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/4); + transpose_2d_as_16b(backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/16/2); + s_buffer = buf_trans_s.buffer; + } + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 32), static_cast<size_t>(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + // unpack + cl_uchar mask = 0xFF; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_ql.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s_buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)n_blk, 1, 1}; + size_t local_work_size[] = {1, 1, 1}; cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; } - #endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_uchar mask = 0xFF; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)n_blk, 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; @@ -12094,1238 +17538,1297 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } - if (tensor->type == GGML_TYPE_Q8_0) { - // KV-cache q8_0 stays AoS (see Q4_0 branch). - if (!ggml_cl_is_q8_0_soa(tensor)) { - ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; - CL_CHECK(clEnqueueReadBuffer( - queue, extra_aos->data_device, CL_TRUE, - extra_aos->offset + tensor->view_offs + offset, - size, data, 0, NULL, NULL)); - return; - } - // SoA extra lives on the parent — follow view_src. - const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; - ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)extra_src->extra; - - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (enable_adreno_trans_weight(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; +#endif // GGML_OPENCL_SOA_Q - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - GGML_ASSERT(tensor->ne[2] == 1); - GGML_ASSERT(tensor->ne[3] == 1); + if (tensor->type == GGML_TYPE_BF16) { + GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0 + && "Offset and size must be multiples of 2 for bf16 tensors"); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); - size_t global_work_size[3] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), 1, 1}; - size_t local_work_size[3] = {64, 1, 1}; + cl_ulong n_elements = size / sizeof(ggml_fp16_t); + cl_ulong off_src = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err); + CL_CHECK(err); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } -#endif - cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + cl_kernel kernel = backend_ctx->kernel_convert_f16_to_bf16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); + queue, data_device, CL_TRUE, 0, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); + return; } - if (tensor->type == GGML_TYPE_IQ4_NL) { - ggml_tensor_extra_cl_iq4_nl * extra = (ggml_tensor_extra_cl_iq4_nl *)tensor->extra; - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - static ggml_cl_buffer buf_trans_q; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_unpacked; + CL_CHECK(clEnqueueReadBuffer( + queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); - cl_int M = tensor->ne[1]; - cl_int K = tensor->ne[0]; - GGML_ASSERT(K % 32 == 0); + GGML_UNUSED(buffer); +} - size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*(ggml_blck_size(tensor->type)/2); - size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); +static void ggml_backend_opencl_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; + ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; - buf_trans_q.allocate(backend_ctx->context, size_q); - buf_trans_d.allocate(backend_ctx->context, size_d); - buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + cl_command_queue queue = backend_ctx->queue; - // transpose q, d back - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + for (cl_mem buf : ctx->buffer) { + CL_CHECK(clEnqueueFillBuffer(queue, buf, &value, sizeof(value), 0, buffer->size, 0, NULL, NULL)); + } + CL_CHECK(clFinish(queue)); +} - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; +static void ggml_backend_opencl_buffer_reset(ggml_backend_buffer_t buffer) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ctx->reset(); +} - cl_kernel kernel = backend_ctx->kernel_restore_block_iq4_nl_noshuffle; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); +static ggml_backend_buffer_i ggml_backend_opencl_buffer_interface = { + /* .free_buffer = */ ggml_backend_opencl_buffer_free_buffer, + /* .get_base = */ ggml_backend_opencl_buffer_get_base, + /* .init_tensor = */ ggml_backend_opencl_buffer_init_tensor, + /* .memset_tensor = */ NULL, + /* .set_tensor = */ ggml_backend_opencl_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_opencl_buffer_get_tensor, + /* .set_tensor_2d = */ NULL, + /* .get_tensor_2d = */ NULL, + /* .cpy_tensor = */ NULL, + /* .clear = */ ggml_backend_opencl_buffer_clear, + /* .reset = */ ggml_backend_opencl_buffer_reset, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, +}; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); +// +// buffer type +// - size_t global_work_size[] = {(size_t)n_blk, 1, 1}; - size_t local_work_size[] = {1, 1, 1}; +static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type) { + return "OpenCL"; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); - return; - } -#endif - cl_kernel kernel = backend_ctx->kernel_restore_block_iq4_nl; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + GGML_UNUSED(buffer_type); +} - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_blk)); +static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buffer_type, size_t size) { + ggml_backend_opencl_context *backend_ctx = ggml_cl_init(buffer_type->device); - size_t global_work_size[] = {(size_t)n_blk, 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + if (!backend_ctx->program_cache_initialized) { + backend_ctx->program_cache = cl_program_cache_init(backend_ctx->device); + backend_ctx->program_cache_initialized = true; + } + load_cl_kernels(backend_ctx); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + // clCreateBuffer returns -61 for size 0 + size = std::max(size, (size_t)1); + + cl_int err; + cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + // On Adreno X1-85 the device pool intermittently fails at hundreds of MB + // once the heap fragments (e.g. graph-allocator compute-buffer reserve + // after model load). Four-step retry: + // 1. normal alloc (fast path) + // 2. clFinish + retry (drains in-flight allocs) + // 3. cl_qcom_large_buffer (X2-class driver only, OpenCL 3.0 only) + // 4. ALLOC_HOST_PTR (host-pinned pool) — last-resort fallback. This + // buffer backs compute scratch read/written by every kernel in the + // graph, so kernel accesses fall to host memory and runtime perf + // degrades meaningfully. Better than failing to load, but the user + // should see the warning and consider -ngl reduction. + if (err != CL_SUCCESS) { + clFinish(backend_ctx->queue); + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + } +#if GGML_OPENCL_TARGET_VERSION >= 300 + // clCreateBufferWithProperties and cl_mem_properties are OpenCL 3.0. Drivers older than + // that do not export the symbol, so a build targeting them fails to link. The large + // buffer extension is only ever enabled on drivers that are well past 3.0, so this path + // is dead there anyway. + if (err != CL_SUCCESS && backend_ctx->adreno_use_large_buffer) { + cl_mem_properties props[] = { 0x41A6 /* CL_LARGE_BUFFER_QCOM */, 1, 0 }; + mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); + } +#endif + if (err != CL_SUCCESS) { + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, size, NULL, &err); + if (err == CL_SUCCESS) { + GGML_LOG_WARN("%s: %.2f MiB allocated via CL_MEM_ALLOC_HOST_PTR fallback — " + "device pool exhausted; runtime perf will be degraded. " + "Consider lowering -ngl or context size.\n", + __func__, size / 1024.0 / 1024.0); + } } - if (tensor->type == GGML_TYPE_Q4_K) { - ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; - - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; - -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Undo the 64-row-tiled canonical pack (kernel_convert_block_q4_k_tiled_ns). - // Without this, a read-back of a tiled weight returns the tiled bytes - // reinterpreted as block_q4_K -- which is how test-backend-ops builds its - // CPU reference (ggml_backend_graph_copy -> tensor_get), so the tiled path - // "failed" the suite while computing the correct product. - if (use_q4k_tiled(backend_ctx, tensor)) { - const int ne00v = tensor->ne[0]; - const int ne01v = tensor->ne[1]; - const int nbv = ne00v / 256; - const size_t n_blk = (size_t)nbv * ne01v; - std::vector<uint32_t> tq(n_blk*32); - std::vector<uint16_t> td(n_blk), tdm(n_blk); - std::vector<uint8_t> ts(n_blk*12); - CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, tq.size()*4, tq.data(), 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->dm, CL_TRUE, 0, tdm.size()*2, tdm.data(), 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); + if (err != CL_SUCCESS) { + GGML_LOG_ERROR("%s: failed to allocate %.2f MiB (err=%d). " + "Consider reducing -ngl, lowering -c / -ub, or using quantized KV cache.\n", + __func__, size / 1024.0 / 1024.0, err); + return nullptr; + } - std::vector<uint8_t> rebuilt(ggml_nbytes(tensor), 0); - for (int i01 = 0; i01 < ne01v; ++i01) { - const int rt = i01/64, rit = i01%64; - for (int i00 = 0; i00 < nbv; ++i00) { - uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*144; - const int tb = rt*nbv + i00; - const size_t si = (size_t)tb*64 + rit; + ggml_backend_opencl_buffer_context * ctx = new ggml_backend_opencl_buffer_context(mem); - memcpy(b + 0, &td [si], 2); - memcpy(b + 2, &tdm[si], 2); - memcpy(b + 4, &ts[si*12], 12); + return ggml_backend_buffer_init(buffer_type, ggml_backend_opencl_buffer_interface, ctx, size); +} - uint32_t qw[32]; - for (int gr = 0; gr < 8; ++gr) { - const size_t base = ((size_t)tb*8 + gr)*64 + rit; - for (int j = 0; j < 4; ++j) qw[gr*4 + j] = tq[base*4 + j]; - } - uint8_t * q = b + 16; - for (int e = 0; e < 256; ++e) { - const int g = e>>6, w = e&63, h = w>>5, l = w&31; - const uint32_t code = (qw[e>>3] >> ((e&7)*4)) & 0xF; - q[g*32 + l] |= (uint8_t)(h ? (code << 4) : code); - } - } - } - memcpy(data, rebuilt.data() + offset, size); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); +static size_t ggml_backend_opencl_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer_type->device->context; + return dev_ctx->backend_ctx->alignment; +} - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_k_trans4_ns; +static size_t ggml_backend_opencl_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) { + static size_t max_size = -1; + if (max_size == (size_t)-1) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer_type->device->context; + max_size = dev_ctx->backend_ctx->max_alloc_size; + } + return max_size; +} - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); +static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { + return ggml_backend_is_opencl(backend); - size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; - size_t local_work_size[] = {64, 1, 1}; + UNUSED(buft); +} - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { - int M = tensor->ne[1]; - int K = tensor->ne[0]; +static size_t ggml_backend_opencl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + size_t size = ggml_nbytes(tensor); +#ifdef GGML_OPENCL_SOA_Q + // set_tensor carves quantized weights into per-component subbuffers (d/q, + // ql/qh/s/d, ...) whose origins are each rounded up to the device base + // alignment. When a component's size is not a multiple of the alignment + // (e.g. q6_K [1536,49155]: size_s = 49155*96 leaves a 96-byte gap at 128-byte + // alignment), the aligned carve extends past ggml_nbytes and the last + // subbuffer would overlap the next tensor in the pool. Reserve the worst-case + // carve slack: at most 5 components (q5_K), i.e. 4 aligned gaps. + if (ggml_is_quantized(tensor->type)) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buft->device->context; + size += 4 * dev_ctx->backend_ctx->alignment; + } +#endif // GGML_OPENCL_SOA_Q + return size; +} - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*12; +static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = { + /* .get_name = */ ggml_backend_opencl_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_opencl_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ NULL, +}; - static ggml_cl_buffer buf_trans_q; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_trans_dm; - static ggml_cl_buffer buf_trans_s; +// +// backend device +// - buf_trans_q.allocate(backend_ctx->context, size_q); - buf_trans_d.allocate(backend_ctx->context, size_d); - buf_trans_dm.allocate(backend_ctx->context, size_dm); - buf_trans_s.allocate(backend_ctx->context, size_s); +static const char * ggml_backend_opencl_device_get_name(ggml_backend_dev_t dev) { + return "GPUOpenCL"; - // Transpose q, d, dm, s back - if (use_q4_k_bin_kernels(backend_ctx, tensor)) { - transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); - } else { - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); - } - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); - transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); - transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); + GGML_UNUSED(dev); +} - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_s.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_dm.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); +static const char * ggml_backend_opencl_device_get_description(ggml_backend_dev_t dev) { + ggml_backend_opencl_device_context *dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + return dev_ctx->device_name.c_str(); +} - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; +static void ggml_backend_opencl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static const size_t opencl_extra_margin = 1024ull*1024ull*1024ull; - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + // OpenCL does not provide reliable currently-free device memory. + // Use total/global memory as a best-effort upper bound. + // Improved safety: Reduce by a 1GiB extra margin for common --fit + *total = dev_ctx->global_mem_size; + *free = *total > opencl_extra_margin ? *total - opencl_extra_margin : 0; +} - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; +static enum ggml_backend_dev_type ggml_backend_opencl_device_get_type(ggml_backend_dev_t dev) { + return GGML_BACKEND_DEVICE_TYPE_GPU; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (tensor->type == GGML_TYPE_Q5_K) { - ggml_tensor_extra_cl_q5_K * extra = (ggml_tensor_extra_cl_q5_K *)tensor->extra; + GGML_UNUSED(dev); +} - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); +static void ggml_backend_opencl_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { + props->name = ggml_backend_opencl_device_get_name(dev); + props->description = ggml_backend_opencl_device_get_description(dev); + props->type = ggml_backend_opencl_device_get_type(dev); + ggml_backend_opencl_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = ggml_backend_dev_caps { + /* .async = */ false, + /* .host_buffer = */ false, + /* .buffer_from_host_ptr = */ false, + /* .events = */ false, + /* .mmap_support = */ false, + }; +} - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; +static ggml_backend_t ggml_backend_opencl_device_init(ggml_backend_dev_t dev, const char * params) { + ggml_backend_opencl_context * backend_ctx = ggml_cl_init(dev); + // Getting a new reference to the backend, increase ref_count + backend_ctx->ref_count++; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_k_trans4_ns; + ggml_backend_t backend = new ggml_backend { + /* .guid = */ ggml_backend_opencl_guid(), + /* .interface = */ ggml_backend_opencl_i, + /* .device = */ dev, + /* .context = */ backend_ctx, + }; - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_F0)); + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + ggml_opencl_print_backend_info(dev_ctx); + return backend; - size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; - size_t local_work_size[] = {64, 1, 1}; + GGML_UNUSED(params); +} - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { - int M = tensor->ne[1]; - int K = tensor->ne[0]; +static ggml_backend_buffer_type_t ggml_backend_opencl_device_get_buffer_type(ggml_backend_dev_t dev) { + auto * dev_ctx = static_cast<ggml_backend_opencl_device_context *>(dev->context); - size_t size_q = extra->size_q; - size_t size_qh = extra->size_qh; - size_t size_s = extra->size_s; - size_t size_d = extra->size_d; - size_t size_dm = extra->size_dm; + dev_ctx->buffer_type = ggml_backend_buffer_type{ + /* .iface = */ ggml_backend_opencl_buffer_type_interface, + /* .device = */ dev, + /* .context = */ nullptr, + }; - static ggml_cl_buffer buf_trans_q; - static ggml_cl_buffer buf_trans_qh; - static ggml_cl_buffer buf_trans_s; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_trans_dm; + return &dev_ctx->buffer_type; +} - buf_trans_q.allocate(backend_ctx->context, size_q); - buf_trans_qh.allocate(backend_ctx->context, size_qh); - buf_trans_s.allocate(backend_ctx->context, size_s); - buf_trans_d.allocate(backend_ctx->context, size_d); - buf_trans_dm.allocate(backend_ctx->context, size_dm); +static ggml_backend_buffer_t ggml_backend_opencl_device_buffer_from_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { + GGML_UNUSED(dev); + GGML_UNUSED(ptr); + GGML_UNUSED(size); + GGML_UNUSED(max_tensor_size); + return nullptr; +} - // bin kernel transposes s but src kernel does not - cl_mem buf_s = extra->s; +static bool ggml_backend_opencl_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + ggml_cl_init(dev); + return ggml_opencl_supports_op(dev, op); +} - if (use_q5_k_bin_kernels(backend_ctx, tensor)) { - transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); - transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); - buf_s = buf_trans_s.buffer; - } else { - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); - } - transpose_2d_as_8b (backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); - transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); +static bool ggml_backend_opencl_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { + // Check 'dev' and 'buffer_type' are not objects belonging to this backend. + if (dev->iface.get_name != ggml_backend_opencl_device_get_name || + buft->iface.get_name != ggml_backend_opencl_buffer_type_get_name) { + return false; + } - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_trans_dm.buffer)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); + // Check cl_context is the same. clEnqueue* commands may not use + // buffers from another cl_context. + ggml_backend_opencl_context * backend_ctx0 = ggml_cl_init(dev); + ggml_backend_opencl_context * backend_ctx1 = ggml_cl_init(buft->device); + return backend_ctx0->context == backend_ctx1->context; +} - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; +namespace /* anonymous */ { +struct ggml_backend_device_i ggml_backend_opencl_device_i = { + /* .get_name = */ ggml_backend_opencl_device_get_name, + /* .get_description = */ ggml_backend_opencl_device_get_description, + /* .get_memory = */ ggml_backend_opencl_device_get_memory, + /* .get_type = */ ggml_backend_opencl_device_get_type, + /* .get_props = */ ggml_backend_opencl_device_get_props, + /* .init_backend = */ ggml_backend_opencl_device_init, + /* .get_buffer_type = */ ggml_backend_opencl_device_get_buffer_type, + /* .get_host_buffer_type = */ NULL, + /* .buffer_from_host_ptr = */ ggml_backend_opencl_device_buffer_from_ptr, + /* .supports_op = */ ggml_backend_opencl_device_supports_op, + /* .supports_buft = */ ggml_backend_opencl_device_supports_buft, + /* .offload_op = */ NULL, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_synchronize = */ NULL, +}; +} - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS +// Backend registry - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); +static const char * ggml_backend_opencl_reg_get_name(ggml_backend_reg_t reg) { + return "OpenCL"; - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + GGML_UNUSED(reg); +} - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (tensor->type == GGML_TYPE_Q6_K) { - ggml_tensor_extra_cl_q6_K * extra = (ggml_tensor_extra_cl_q6_K *)tensor->extra; +static size_t ggml_backend_opencl_reg_device_count(ggml_backend_reg_t reg) { + return g_ggml_backend_opencl_devices.size(); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Undo the 64-row-tiled canonical pack (kernel_convert_block_q6_k_tiled_ns). - // See the q4_K tiled restore above for why a read-back path is required. - if (use_q6k_tiled(backend_ctx, tensor)) { - const int ne00v = tensor->ne[0]; - const int ne01v = tensor->ne[1]; - const int nbv = ne00v / 256; - const size_t n_blk = (size_t)nbv * ne01v; + GGML_UNUSED(reg); +} - std::vector<uint32_t> tql(n_blk*32), tqh(n_blk*16); - std::vector<uint8_t> ts(n_blk*16); - std::vector<uint16_t> td(n_blk); - CL_CHECK(clEnqueueReadBuffer(queue, extra->ql, CL_TRUE, 0, tql.size()*4, tql.data(), 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->qh, CL_TRUE, 0, tqh.size()*4, tqh.data(), 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); +static ggml_backend_dev_t ggml_backend_opencl_reg_device_get(ggml_backend_reg_t reg, size_t index) { + GGML_ASSERT(index < ggml_backend_opencl_reg_device_count(reg)); - std::vector<uint8_t> rebuilt(ggml_nbytes(tensor), 0); - for (int i01 = 0; i01 < ne01v; ++i01) { - const int rt = i01/64, rit = i01%64; - for (int i00 = 0; i00 < nbv; ++i00) { - uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*210; - const int tb = rt*nbv + i00; - const size_t si = (size_t)tb*64 + rit; + return &g_ggml_backend_opencl_devices[index]; - uint32_t qlw[32], qhw[16]; - for (int g = 0; g < 8; ++g) { - const size_t base = ((size_t)tb*8 + g)*64 + rit; - for (int j = 0; j < 4; ++j) qlw[g*4 + j] = tql[base*4 + j]; - } - for (int g = 0; g < 4; ++g) { - const size_t base = ((size_t)tb*4 + g)*64 + rit; - for (int j = 0; j < 4; ++j) qhw[g*4 + j] = tqh[base*4 + j]; - } + GGML_UNUSED(reg); + GGML_UNUSED(index); +} - uint8_t * ql = b; - uint8_t * qh = b + 128; - for (int e = 0; e < 256; ++e) { - const int n = (e >= 128) ? 1 : 0; - const int within = e - n*128, q = within/32, l = within%32; - const int off_ql = n*64, off_qh = n*32; - const uint8_t low4 = (qlw[e>>3] >> ((e&7)*4)) & 0xF; - const uint8_t hi2 = (qhw[e>>4] >> ((e&15)*2)) & 0x3; - if (q == 0) ql[off_ql + l] |= low4; - else if (q == 1) ql[off_ql + l + 32] |= low4; - else if (q == 2) ql[off_ql + l] |= (uint8_t)(low4 << 4); - else ql[off_ql + l + 32] |= (uint8_t)(low4 << 4); - qh[off_qh + l] |= (uint8_t)(hi2 << (q*2)); - } - memcpy(b + 192, &ts[si*16], 16); - memcpy(b + 208, &td[si], 2); - } - } - memcpy(data, rebuilt.data() + offset, size); - return; - } - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); +static struct ggml_backend_reg_i ggml_backend_opencl_reg_i = { + /* .get_name = */ ggml_backend_opencl_reg_get_name, + /* .device_count = */ ggml_backend_opencl_reg_device_count, + /* .device_get = */ ggml_backend_opencl_reg_device_get, + /* .get_proc_address = */ NULL, +}; - cl_kernel kernel = backend_ctx->kernel_restore_block_q6_k_trans4_ns; +ggml_backend_reg_t ggml_backend_opencl_reg(void) { + static std::mutex mutex; + static ggml_backend_reg reg; + static bool initialized = false; + std::lock_guard<std::mutex> lock(mutex); - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + if (initialized) { + return ® + } + initialized = true; - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); + g_ggml_backend_opencl_devices = ggml_opencl_probe_devices(®); + + reg = ggml_backend_reg{ + /* .api_version = */ GGML_BACKEND_API_VERSION, + /* .iface = */ ggml_backend_opencl_reg_i, + /* .context = */ NULL, + }; - size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)}; - size_t local_work_size[] = {64, 1, 1}; + return ® +} - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { - static ggml_cl_buffer buf_trans_ql; - static ggml_cl_buffer buf_trans_qh; - static ggml_cl_buffer buf_trans_s; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_unpacked; +GGML_BACKEND_DL_IMPL(ggml_backend_opencl_reg) - cl_int M = tensor->ne[1]; // ne01 - cl_int K = tensor->ne[0]; // ne00 +//------------------------------------------------------------------------------ +// Debugging utils +//------------------------------------------------------------------------------ +#if 0 +#define QK4_0 32 +typedef struct { + ggml_fp16_t d; // delta + uint8_t qs[QK4_0 / 2]; // nibbles / quants +} block_q4_0; +static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, + "wrong q4_0 block size/padding"); - GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); +#define QK_MXFP4 32 - size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); +#include <math.h> +#ifdef __cplusplus +#include "half.hpp" +#endif - buf_trans_ql.allocate(backend_ctx->context, size_ql); - buf_trans_qh.allocate(backend_ctx->context, size_qh); - buf_trans_d.allocate(backend_ctx->context, size_d); - buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); +static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tensor) { + void * buf = malloc(ggml_nbytes(tensor)); - cl_mem s_buffer; - if (use_q6_k_bin_kernels(backend_ctx, tensor)) { - transpose_2d_as_32b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/8); - transpose_2d_as_32b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/16); - // s is left row-major, untransposed, for the binary layout. - s_buffer = extra->s; - } else { - // transpose ql, qh, s and d back - buf_trans_s.allocate(backend_ctx->context, size_s); - transpose_2d_as_16b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/4); - transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/4); - transpose_2d_as_16b(backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/16/2); - s_buffer = buf_trans_s.buffer; - } - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_command_queue queue = backend_ctx->queue; +#ifdef GGML_OPENCL_SOA_Q + void * buf_q; + void * buf_d; +#endif - // unpack - cl_uchar mask = 0xFF; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_ql.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s_buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + // Make sure everything is done. + CL_CHECK(clFinish(queue)); - size_t global_work_size[] = {(size_t)n_blk, 1, 1}; - size_t local_work_size[] = {1, 1, 1}; +#ifdef GGML_OPENCL_SOA_Q + if (tensor->type == GGML_TYPE_Q4_0) { + ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *) tensor->extra; + GGML_ASSERT(extra); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + size_t size_q = ggml_nelements(tensor)/QK4_0 * QK4_0/2; + size_t size_d = ggml_nelements(tensor)/QK4_0 * sizeof(ggml_fp16_t); + GGML_ASSERT(size_q + size_d == ggml_nbytes(tensor)); + buf_q = malloc(size_q); + buf_d = malloc(size_d); - return; - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_d, buf_d, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); + } else if (tensor->type == GGML_TYPE_MXFP4) { + ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *) tensor->extra; + GGML_ASSERT(extra); - cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); - GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + size_t size_q = ggml_nelements(tensor)/QK_MXFP4 * QK_MXFP4/2; + size_t size_e = ggml_nelements(tensor)/QK_MXFP4 * sizeof(char); + GGML_ASSERT(size_q + size_e == ggml_nbytes(tensor)); + buf_q = malloc(size_q); + buf_d = malloc(size_e); - cl_uchar mask = 0xFF; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->e, CL_TRUE, 0, size_e, buf_d, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); + } else { + // Read out the tensor from GPU memory. + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); - size_t global_work_size[] = {(size_t)n_blk, 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, + extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); + } +#else + // Read out the tensor from GPU memory. + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, + extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); +#endif // GGML_OPENCL_SOA_Q + + // Open file and dump. + char fname[512]; + snprintf(fname, sizeof(fname), "./tensor-dumps/%s.txt", tensor->name); + FILE * f = fopen(fname, "w"); + if (!f) { + printf("Failed to open %s\n", fname); return; } + + if (tensor->type == GGML_TYPE_F32) { + float * data = (float *) buf; + for (int i = 0; i < ggml_nelements(tensor); ++i) { + if (isnan(data[i])) { + printf("NaN found: %s\n", tensor->name); + break; + } + fprintf(f, "%f\n", data[i]); + } + } else if (tensor->type == GGML_TYPE_I32) { + int * data = (int *) buf; + for (int i = 0; i < ggml_nelements(tensor); ++i) { + if (isnan(data[i])) { + printf("NaN found: %s\n", tensor->name); + break; + } + fprintf(f, "%d\n", data[i]); + } + } else if (tensor->type == GGML_TYPE_F16) { +#ifdef __cplusplus + half_float::half * data = (half_float::half *) buf; + for (int i = 0; i < ggml_nelements(tensor); ++i) { + if (std::isnan(data[i])) { + printf("NaN found: %s\n", tensor->name); + break; + } + fprintf(f, "%f\n", float(data[i])); + } +#endif + } else if (tensor->type == GGML_TYPE_Q4_0) { +#ifdef GGML_OPENCL_SOA_Q + ggml_fp16_t * data_d = (ggml_fp16_t *)buf_d; + unsigned char * data_q = (unsigned char *)buf_q; + + for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { + fprintf(f, "%04x, ", data_d[i]); + for (int k = 0; k < QK4_0/2; ++k) { + fprintf(f, "%02x, ", data_q[k]); + } + fprintf(f, "\n"); + data_q += QK4_0/2; + } + free(buf_d); + free(buf_q); +#else + block_q4_0 * data = (block_q4_0 *) buf; + for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { + fprintf(f, "%04x, ", data[i].d); + for (int k = 0; k < QK4_0/2; ++k) { + fprintf(f, "%02x, ", data[i].qs[k]); + } + fprintf(f, "\n"); + } #endif // GGML_OPENCL_SOA_Q + } + free(buf); + fflush(f); + fclose(f); +} +#else +#define dump_tensor(tensor) +#endif - if (tensor->type == GGML_TYPE_BF16) { - GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0 - && "Offset and size must be multiples of 2 for bf16 tensors"); +//------------------------------------------------------------------------------ +// Ops +//------------------------------------------------------------------------------ - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra); +static bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { + const int64_t ne10 = src1->ne[0]; - cl_ulong n_elements = size / sizeof(ggml_fp16_t); - cl_ulong off_src = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t); + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + + // TODO: find the optimal values for these + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && + src1->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F32 && + (ne0 >= 32 && ne1 >= 32 && ne10 >= 32); +} + +// Copy a noncontiguous tensor to contiguous tensor. ne[] remains the same but +// nb[] is recalculated such that tensor is contiguous. +static void ggml_cl_copy_to_contiguous(ggml_backend_t backend, const ggml_tensor * src, cl_mem dst, + cl_ulong &nb0, cl_ulong &nb1, cl_ulong &nb2, cl_ulong &nb3) { + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err); - CL_CHECK(err); + const int tensor_type_size = ggml_type_size(src->type); - cl_kernel kernel = backend_ctx->kernel_convert_f16_to_bf16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements)); + const int ne00 = src->ne[0]; + const int ne01 = src->ne[1]; + const int ne02 = src->ne[2]; + const int ne03 = src->ne[3]; - size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 }; - size_t local_work_size[] = { 64, 1, 1 }; + const cl_ulong nb00 = src->nb[0]; + const cl_ulong nb01 = src->nb[1]; + const cl_ulong nb02 = src->nb[2]; + const cl_ulong nb03 = src->nb[3]; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseEvent(evt)); + const int ne0 = src->ne[0]; + const int ne1 = src->ne[1]; + const int ne2 = src->ne[2]; + const int ne3 = src->ne[3]; - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, 0, size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); + nb0 = tensor_type_size; + nb1 = tensor_type_size*ne00; + nb2 = tensor_type_size*ne00*ne01; + nb3 = tensor_type_size*ne00*ne01*ne02; - return; - } + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *)src->extra; - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + cl_ulong offset0 = extra->offset + src->view_offs; + cl_ulong offsetd = 0; - CL_CHECK(clEnqueueReadBuffer( - queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset, - size, data, 0, NULL, NULL)); + cl_kernel kernel; - GGML_UNUSED(buffer); -} + switch (src->type) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_cpy_f32_f32; + break; + case GGML_TYPE_F16: + case GGML_TYPE_BF16: // stored as f16 on device + kernel = backend_ctx->kernel_cpy_f16_f16; + break; + default: + GGML_ASSERT(false && "not implemented"); + } -static void ggml_backend_opencl_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; - ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dst)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb3)); - cl_command_queue queue = backend_ctx->queue; + const int nth = MIN(64, ne00); - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - for (cl_mem buf : ctx->buffer) { - CL_CHECK(clEnqueueFillBuffer(queue, buf, &value, sizeof(value), 0, buffer->size, 0, NULL, NULL)); - } - CL_CHECK(clFinish(queue)); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src); } -static void ggml_backend_opencl_buffer_reset(ggml_backend_buffer_t buffer) { - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ctx->reset(); +static void ggml_cl_nop(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + UNUSED(backend); + UNUSED(src0); + UNUSED(src1); + UNUSED(dst); } -static ggml_backend_buffer_i ggml_backend_opencl_buffer_interface = { - /* .free_buffer = */ ggml_backend_opencl_buffer_free_buffer, - /* .get_base = */ ggml_backend_opencl_buffer_get_base, - /* .init_tensor = */ ggml_backend_opencl_buffer_init_tensor, - /* .memset_tensor = */ NULL, - /* .set_tensor = */ ggml_backend_opencl_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_opencl_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, - /* .cpy_tensor = */ NULL, - /* .clear = */ ggml_backend_opencl_buffer_clear, - /* .reset = */ ggml_backend_opencl_buffer_reset, -}; +static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); -// -// buffer type -// + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(int, ne1, src1, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); -static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type) { - return "OpenCL"; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - GGML_UNUSED(buffer_type); -} + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; -static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buffer_type, size_t size) { - ggml_backend_opencl_context *backend_ctx = ggml_cl_init(buffer_type->device); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - if (!backend_ctx->program_cache_initialized) { - backend_ctx->program_cache = cl_program_cache_init(backend_ctx->device); - backend_ctx->program_cache_initialized = true; + cl_kernel kernel; + + switch (src0->type) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_get_rows_f32; + break; + case GGML_TYPE_I32: + kernel = backend_ctx->kernel_get_rows_i32; + break; + case GGML_TYPE_F16: + kernel = backend_ctx->kernel_get_rows_f16; + break; + case GGML_TYPE_Q4_0: + kernel = backend_ctx->kernel_get_rows_q4_0; + break; + default: + GGML_ASSERT(false && "not implemented"); } - load_cl_kernels(backend_ctx); - // clCreateBuffer returns -61 for size 0 - size = std::max(size, (size_t)1); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); - cl_int err; - cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); - // On Adreno X1-85 the device pool intermittently fails at hundreds of MB - // once the heap fragments (e.g. graph-allocator compute-buffer reserve - // after model load). Four-step retry: - // 1. normal alloc (fast path) - // 2. clFinish + retry (drains in-flight allocs) - // 3. cl_qcom_large_buffer (X2-class driver only, OpenCL 3.0 only) - // 4. ALLOC_HOST_PTR (host-pinned pool) — last-resort fallback. This - // buffer backs compute scratch read/written by every kernel in the - // graph, so kernel accesses fall to host memory and runtime perf - // degrades meaningfully. Better than failing to load, but the user - // should see the warning and consider -ngl reduction. - if (err != CL_SUCCESS) { - clFinish(backend_ctx->queue); - mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); - } -#if GGML_OPENCL_TARGET_VERSION >= 300 - // clCreateBufferWithProperties and cl_mem_properties are OpenCL 3.0. Drivers older than - // that do not export the symbol, so a build targeting them fails to link. The large - // buffer extension is only ever enabled on drivers that are well past 3.0, so this path - // is dead there anyway. - if (err != CL_SUCCESS && backend_ctx->adreno_use_large_buffer) { - cl_mem_properties props[] = { 0x41A6 /* CL_LARGE_BUFFER_QCOM */, 1, 0 }; - mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); - } -#endif - if (err != CL_SUCCESS) { - mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, size, NULL, &err); - if (err == CL_SUCCESS) { - GGML_LOG_WARN("%s: %.2f MiB allocated via CL_MEM_ALLOC_HOST_PTR fallback — " - "device pool exhausted; runtime perf will be degraded. " - "Consider lowering -ngl or context size.\n", - __func__, size / 1024.0 / 1024.0); - } + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + int nth = 1; + while (nth < ne00 && 2*nth <= max_workgroup_size) { + nth *= 2; } - if (err != CL_SUCCESS) { - GGML_LOG_ERROR("%s: failed to allocate %.2f MiB (err=%d). " - "Consider reducing -ngl, lowering -c / -ub, or using quantized KV cache.\n", - __func__, size / 1024.0 / 1024.0, err); - return nullptr; + int nchunks = 1; + if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) { + const int chunk_target = nth * 4; + nchunks = (ne00 + chunk_target - 1) / chunk_target; + nchunks = MAX(1, MIN(nchunks, 64)); } - ggml_backend_opencl_buffer_context * ctx = new ggml_backend_opencl_buffer_context(mem); - - return ggml_backend_buffer_init(buffer_type, ggml_backend_opencl_buffer_interface, ctx, size); -} + size_t global_work_size[] = {(size_t)ne10*nth*nchunks, (size_t)ne11, (size_t)ne12}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; -static size_t ggml_backend_opencl_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer_type->device->context; - return dev_ctx->backend_ctx->alignment; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static size_t ggml_backend_opencl_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) { - static size_t max_size = -1; - if (max_size == (size_t)-1) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer_type->device->context; - max_size = dev_ctx->backend_ctx->max_alloc_size; +// check if a Q8_0 tensor has been SOA'ed in set_tensor +// we store SOA'ed tensors in a map in set_tensor, check against that map +static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->type != GGML_TYPE_Q8_0 || tensor->buffer == nullptr) { + return false; } - return max_size; -} - -static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { - return ggml_backend_is_opencl(backend); - - UNUSED(buft); + auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; + if (ctx == nullptr) { + return false; + } + const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; + return ctx->q8_0_soa_tensors.count(key) > 0; } -static size_t ggml_backend_opencl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - size_t size = ggml_nbytes(tensor); -#ifdef GGML_OPENCL_SOA_Q - // set_tensor carves quantized weights into per-component subbuffers (d/q, - // ql/qh/s/d, ...) whose origins are each rounded up to the device base - // alignment. When a component's size is not a multiple of the alignment - // (e.g. q6_K [1536,49155]: size_s = 49155*96 leaves a 96-byte gap at 128-byte - // alignment), the aligned carve extends past ggml_nbytes and the last - // subbuffer would overlap the next tensor in the pool. Reserve the worst-case - // carve slack: at most 5 components (q5_K), i.e. 4 aligned gaps. - if (ggml_is_quantized(tensor->type)) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buft->device->context; - size += 4 * dev_ctx->backend_ctx->alignment; +// check if a Q4_0 tensor has been SOA'ed in set_tensor +// we store SOA'ed tensors in a map in set_tensor, check against that map +static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->type != GGML_TYPE_Q4_0 || tensor->buffer == nullptr) { + return false; } -#endif // GGML_OPENCL_SOA_Q - return size; + auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; + if (ctx == nullptr) { + return false; + } + const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; + return ctx->q4_0_soa_tensors.count(key) > 0; } -static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = { - /* .get_name = */ ggml_backend_opencl_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer, - /* .alloc_buffer_n = */ NULL, - /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_opencl_buffer_type_get_alloc_size, - /* .get_alloc_size_n = */ NULL, - /* .is_host = */ NULL, -}; - -// -// backend device -// +static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32); -static const char * ggml_backend_opencl_device_get_name(ggml_backend_dev_t dev) { - return "GPUOpenCL"; + // ne0 = ne00 + // ne2 = ne02 + // ne3 = ne03 - GGML_UNUSED(dev); -} + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); -static const char * ggml_backend_opencl_device_get_description(ggml_backend_dev_t dev) { - ggml_backend_opencl_device_context *dev_ctx = (ggml_backend_opencl_device_context *) dev->context; - return dev_ctx->device_name.c_str(); -} + GGML_TENSOR_LOCALS(int, ne1, src1, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); -static void ggml_backend_opencl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); - static const size_t opencl_extra_margin = 1024ull*1024ull*1024ull; + const int nblk0 = ne0/ggml_blck_size(dst->type); - // OpenCL does not provide reliable currently-free device memory. - // Use total/global memory as a best-effort upper bound. - // Improved safety: Reduce by a 1GiB extra margin for common --fit - *total = dev_ctx->global_mem_size; - *free = *total > opencl_extra_margin ? *total - opencl_extra_margin : 0; -} + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; -static enum ggml_backend_dev_type ggml_backend_opencl_device_get_type(ggml_backend_dev_t dev) { - return GGML_BACKEND_DEVICE_TYPE_GPU; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - GGML_UNUSED(dev); -} + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; -static void ggml_backend_opencl_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { - props->name = ggml_backend_opencl_device_get_name(dev); - props->description = ggml_backend_opencl_device_get_description(dev); - props->type = ggml_backend_opencl_device_get_type(dev); - ggml_backend_opencl_device_get_memory(dev, &props->memory_free, &props->memory_total); - props->caps = ggml_backend_dev_caps { - /* .async = */ false, - /* .host_buffer = */ false, - /* .buffer_from_host_ptr = */ false, - /* .events = */ false, - /* .mmap_support = */ false, - }; -} + const bool q8_0_soa = dst->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(dst); + const bool q4_0_soa = dst->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(dst); + const bool is_soa = q8_0_soa || q4_0_soa; -static ggml_backend_t ggml_backend_opencl_device_init(ggml_backend_dev_t dev, const char * params) { - ggml_backend_opencl_context * backend_ctx = ggml_cl_init(dev); - // Getting a new reference to the backend, increase ref_count - backend_ctx->ref_count++; + cl_kernel kernel; - ggml_backend_t backend = new ggml_backend { - /* .guid = */ ggml_backend_opencl_guid(), - /* .interface = */ ggml_backend_opencl_i, - /* .device = */ dev, - /* .context = */ backend_ctx, - }; + if (q8_0_soa) { + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q8_0_soa_i64 + : backend_ctx->kernel_set_rows_q8_0_soa_i32; + } else if (q4_0_soa) { + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q4_0_soa_i64 + : backend_ctx->kernel_set_rows_q4_0_soa_i32; + } else { + switch (dst->type) { + case GGML_TYPE_F32: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_f32_i64 + : backend_ctx->kernel_set_rows_f32_i32; + break; + case GGML_TYPE_F16: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_f16_i64 + : backend_ctx->kernel_set_rows_f16_i32; + break; + case GGML_TYPE_Q8_0: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q8_0_i64 + : backend_ctx->kernel_set_rows_q8_0_i32; + break; + case GGML_TYPE_Q4_0: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q4_0_i64 + : backend_ctx->kernel_set_rows_q4_0_i32; + break; + default: + GGML_ABORT("not implemented"); + } + } - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; - ggml_opencl_print_backend_info(dev_ctx); - return backend; + fastdiv_vals ne11_ = init_fastdiv_values(ne11); + fastdiv_vals ne12_ = init_fastdiv_values(ne12); - GGML_UNUSED(params); -} + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); -static ggml_backend_buffer_type_t ggml_backend_opencl_device_get_buffer_type(ggml_backend_dev_t dev) { - auto * dev_ctx = static_cast<ggml_backend_opencl_device_context *>(dev->context); + if (is_soa) { + // The q/d subbuffers in q8_0/q4_0 extras are interchangeable here. + // For views (e.g. ggml_set_rows' `out`), follow view_src for the SoA extra. + const ggml_tensor * soa_src = dst->view_src != nullptr ? dst->view_src : dst; + cl_mem q_mem = nullptr; + cl_mem d_mem = nullptr; + if (q8_0_soa) { + ggml_tensor_extra_cl_q8_0 * e = (ggml_tensor_extra_cl_q8_0 *)soa_src->extra; + q_mem = e->q; + d_mem = e->d; + } else { + ggml_tensor_extra_cl_q4_0 * e = (ggml_tensor_extra_cl_q4_0 *)soa_src->extra; + q_mem = e->q; + d_mem = e->d; + } + cl_ulong offset_q = 0; + cl_ulong offset_d = 0; + const int ne1_dst = dst->ne[1]; + const int ne2_dst = dst->ne[2]; + const int ne3_dst = dst->ne[3]; - dev_ctx->buffer_type = ggml_backend_buffer_type{ - /* .iface = */ ggml_backend_opencl_buffer_type_interface, - /* .device = */ dev, - /* .context = */ nullptr, - }; + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &q_mem)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_q)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_mem)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_d)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(fastdiv_vals), &ne11_)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(fastdiv_vals), &ne12_)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &nblk0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1_dst)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne2_dst)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne3_dst)); + } else { + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong offsetd = extrad->offset + dst->view_offs; - return &dev_ctx->buffer_type; -} + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(fastdiv_vals), &ne11_)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(fastdiv_vals), &ne12_)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &nblk0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb3)); + } -static ggml_backend_buffer_t ggml_backend_opencl_device_buffer_from_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { - GGML_UNUSED(dev); - GGML_UNUSED(ptr); - GGML_UNUSED(size); - GGML_UNUSED(max_tensor_size); - return nullptr; -} + int nth0 = 64; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 32; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + } -static bool ggml_backend_opencl_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { - ggml_cl_init(dev); - return ggml_opencl_supports_op(dev, op); -} + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth0 < nblk0 && nth0 < max_workgroup_size) { + nth0 *= 2; + } -static bool ggml_backend_opencl_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - // Check 'dev' and 'buffer_type' are not objects belonging to this backend. - if (dev->iface.get_name != ggml_backend_opencl_device_get_name || - buft->iface.get_name != ggml_backend_opencl_buffer_type_get_name) { - return false; + int rows_per_workgroup = 1; + if (nth0 > nblk0) { + rows_per_workgroup = nth0 / nblk0; + nth0 = nblk0; } - // Check cl_context is the same. clEnqueue* commands may not use - // buffers from another cl_context. - ggml_backend_opencl_context * backend_ctx0 = ggml_cl_init(dev); - ggml_backend_opencl_context * backend_ctx1 = ggml_cl_init(buft->device); - return backend_ctx0->context == backend_ctx1->context; -} + size_t global_work_size[] = { + (size_t)(ne01 + rows_per_workgroup - 1)/rows_per_workgroup*nth0, + (size_t)ne02*rows_per_workgroup, + (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1}; -namespace /* anonymous */ { -struct ggml_backend_device_i ggml_backend_opencl_device_i = { - /* .get_name = */ ggml_backend_opencl_device_get_name, - /* .get_description = */ ggml_backend_opencl_device_get_description, - /* .get_memory = */ ggml_backend_opencl_device_get_memory, - /* .get_type = */ ggml_backend_opencl_device_get_type, - /* .get_props = */ ggml_backend_opencl_device_get_props, - /* .init_backend = */ ggml_backend_opencl_device_init, - /* .get_buffer_type = */ ggml_backend_opencl_device_get_buffer_type, - /* .get_host_buffer_type = */ NULL, - /* .buffer_from_host_ptr = */ ggml_backend_opencl_device_buffer_from_ptr, - /* .supports_op = */ ggml_backend_opencl_device_supports_op, - /* .supports_buft = */ ggml_backend_opencl_device_supports_buft, - /* .offload_op = */ NULL, - /* .event_new = */ NULL, - /* .event_free = */ NULL, - /* .event_synchronize = */ NULL, -}; + // ne01 == 0 makes global_work_size[0] zero here; enqueue_ndrange_kernel drops the empty range. + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -// Backend registry +static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); -static const char * ggml_backend_opencl_reg_get_name(ggml_backend_reg_t reg) { - return "OpenCL"; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - GGML_UNUSED(reg); -} + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; -static size_t ggml_backend_opencl_reg_device_count(ggml_backend_reg_t reg) { - return g_ggml_backend_opencl_devices.size(); + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; - GGML_UNUSED(reg); -} + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; -static ggml_backend_dev_t ggml_backend_opencl_reg_device_get(ggml_backend_reg_t reg, size_t index) { - GGML_ASSERT(index < ggml_backend_opencl_reg_device_count(reg)); + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; - return &g_ggml_backend_opencl_devices[index]; + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - GGML_UNUSED(reg); - GGML_UNUSED(index); -} + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; -static struct ggml_backend_reg_i ggml_backend_opencl_reg_i = { - /* .get_name = */ ggml_backend_opencl_reg_get_name, - /* .device_count = */ ggml_backend_opencl_reg_device_count, - /* .device_get = */ ggml_backend_opencl_reg_device_get, - /* .get_proc_address = */ NULL, -}; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; -ggml_backend_reg_t ggml_backend_opencl_reg(void) { - static std::mutex mutex; - static ggml_backend_reg reg; - static bool initialized = false; - std::lock_guard<std::mutex> lock(mutex); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - if (initialized) { - return ® + cl_kernel kernel; + + const bool bcast_row = ggml_is_contiguous(src0) && ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; + + if (bcast_row) { + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ne11 == 1); } - initialized = true; - g_ggml_backend_opencl_devices = ggml_opencl_probe_devices(®); + if (dst->type == GGML_TYPE_F32) { + GGML_ASSERT(src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32); + if (bcast_row) { + kernel = backend_ctx->kernel_add_row; + const int ne = ne00 / 4; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + } else { + kernel = backend_ctx->kernel_add; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); + } + } else if (dst->type == GGML_TYPE_F16) { + GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); + const int type_src0 = (src0->type == GGML_TYPE_F32); + const int type_src1 = (src1->type == GGML_TYPE_F32); + if (bcast_row) { + kernel = backend_ctx->kernel_add_row_f16; + const int ne = ne00 / 4; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &type_src0)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &type_src1)); + } else { + kernel = backend_ctx->kernel_add_f16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &type_src0)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &type_src1)); + } + } else { + GGML_ASSERT(false && "unsupported data types for add"); + } - reg = ggml_backend_reg{ - /* .api_version = */ GGML_BACKEND_API_VERSION, - /* .iface = */ ggml_backend_opencl_reg_i, - /* .context = */ NULL, - }; + if (bcast_row) { + int n = ggml_nelements(dst)/4; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - return ® + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size_ptr, dst); + } else { + unsigned int nth = MIN(64, ne0); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -GGML_BACKEND_DL_IMPL(ggml_backend_opencl_reg) +static void ggml_cl_add_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); -//------------------------------------------------------------------------------ -// Debugging utils -//------------------------------------------------------------------------------ -#if 0 -#define QK4_0 32 -typedef struct { - ggml_fp16_t d; // delta - uint8_t qs[QK4_0 / 2]; // nibbles / quants -} block_q4_0; -static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, - "wrong q4_0 block size/padding"); + const ggml_tensor * src2 = dst->src[2]; + GGML_ASSERT(src2); + GGML_ASSERT(src2->extra); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src2->type == GGML_TYPE_I32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ggml_is_contiguous_rows(src0)); -#define QK_MXFP4 32 + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; -#include <math.h> -#ifdef __cplusplus -#include "half.hpp" -#endif + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; -static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tensor) { - void * buf = malloc(ggml_nbytes(tensor)); + const cl_ulong nb11 = src1->nb[1]; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - cl_command_queue queue = backend_ctx->queue; -#ifdef GGML_OPENCL_SOA_Q - void * buf_q; - void * buf_d; -#endif + const cl_ulong nb21 = src2->nb[1]; - // Make sure everything is done. - CL_CHECK(clFinish(queue)); + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; -#ifdef GGML_OPENCL_SOA_Q - if (tensor->type == GGML_TYPE_Q4_0) { - ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *) tensor->extra; - GGML_ASSERT(extra); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t size_q = ggml_nelements(tensor)/QK4_0 * QK4_0/2; - size_t size_d = ggml_nelements(tensor)/QK4_0 * sizeof(ggml_fp16_t); - GGML_ASSERT(size_q + size_d == ggml_nbytes(tensor)); - buf_q = malloc(size_q); - buf_d = malloc(size_d); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_d, buf_d, 0, NULL, NULL)); - CL_CHECK(clFinish(queue)); - } else if (tensor->type == GGML_TYPE_MXFP4) { - ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *) tensor->extra; - GGML_ASSERT(extra); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - size_t size_q = ggml_nelements(tensor)/QK_MXFP4 * QK_MXFP4/2; - size_t size_e = ggml_nelements(tensor)/QK_MXFP4 * sizeof(char); - GGML_ASSERT(size_q + size_e == ggml_nbytes(tensor)); - buf_q = malloc(size_q); - buf_d = malloc(size_e); + cl_kernel kernel = backend_ctx->kernel_add_id; - CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->e, CL_TRUE, 0, size_e, buf_d, 0, NULL, NULL)); - CL_CHECK(clFinish(queue)); - } else { - // Read out the tensor from GPU memory. - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, - extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); - CL_CHECK(clFinish(queue)); - } -#else - // Read out the tensor from GPU memory. - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra); + int nth = MIN(ne00, (int) backend_ctx->get_kernel_workgroup_size(kernel)); + size_t global_work_size[] = { (size_t)ne01*nth, (size_t)ne02, 1 }; + size_t local_work_size[] = { (size_t)nth, 1, 1 }; - CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, - extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); - CL_CHECK(clFinish(queue)); -#endif // GGML_OPENCL_SOA_Q + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - // Open file and dump. - char fname[512]; - snprintf(fname, sizeof(fname), "./tensor-dumps/%s.txt", tensor->name); - FILE * f = fopen(fname, "w"); - if (!f) { - printf("Failed to open %s\n", fname); - return; - } +static void ggml_cl_mul(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - if (tensor->type == GGML_TYPE_F32) { - float * data = (float *) buf; - for (int i = 0; i < ggml_nelements(tensor); ++i) { - if (isnan(data[i])) { - printf("NaN found: %s\n", tensor->name); - break; - } - fprintf(f, "%f\n", data[i]); - } - } else if (tensor->type == GGML_TYPE_I32) { - int * data = (int *) buf; - for (int i = 0; i < ggml_nelements(tensor); ++i) { - if (isnan(data[i])) { - printf("NaN found: %s\n", tensor->name); - break; - } - fprintf(f, "%d\n", data[i]); - } - } else if (tensor->type == GGML_TYPE_F16) { -#ifdef __cplusplus - half_float::half * data = (half_float::half *) buf; - for (int i = 0; i < ggml_nelements(tensor); ++i) { - if (std::isnan(data[i])) { - printf("NaN found: %s\n", tensor->name); - break; - } - fprintf(f, "%f\n", float(data[i])); - } -#endif - } else if (tensor->type == GGML_TYPE_Q4_0) { -#ifdef GGML_OPENCL_SOA_Q - ggml_fp16_t * data_d = (ggml_fp16_t *)buf_d; - unsigned char * data_q = (unsigned char *)buf_q; + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { - fprintf(f, "%04x, ", data_d[i]); - for (int k = 0; k < QK4_0/2; ++k) { - fprintf(f, "%02x, ", data_q[k]); - } - fprintf(f, "\n"); - data_q += QK4_0/2; - } - free(buf_d); - free(buf_q); -#else - block_q4_0 * data = (block_q4_0 *) buf; - for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { - fprintf(f, "%04x, ", data[i].d); - for (int k = 0; k < QK4_0/2; ++k) { - fprintf(f, "%02x, ", data[i].qs[k]); - } - fprintf(f, "\n"); - } -#endif // GGML_OPENCL_SOA_Q - } - free(buf); - fflush(f); - fclose(f); -} -#else -#define dump_tensor(tensor) -#endif + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; -//------------------------------------------------------------------------------ -// Ops -//------------------------------------------------------------------------------ + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; -static bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { - const int64_t ne10 = src1->ne[0]; + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; UNUSED(ne13); - const int64_t ne0 = dst->ne[0]; - const int64_t ne1 = dst->ne[1]; + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; UNUSED(nb13); - // TODO: find the optimal values for these - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && - src1->type == GGML_TYPE_F32 && - dst->type == GGML_TYPE_F32 && - (ne0 >= 32 && ne1 >= 32 && ne10 >= 32); -} + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; + + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; -// Copy a noncontiguous tensor to contiguous tensor. ne[] remains the same but -// nb[] is recalculated such that tensor is contiguous. -static void ggml_cl_copy_to_contiguous(ggml_backend_t backend, const ggml_tensor * src, cl_mem dst, - cl_ulong &nb0, cl_ulong &nb1, cl_ulong &nb2, cl_ulong &nb3) { ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const int tensor_type_size = ggml_type_size(src->type); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - const int ne00 = src->ne[0]; - const int ne01 = src->ne[1]; - const int ne02 = src->ne[2]; - const int ne03 = src->ne[3]; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - const cl_ulong nb00 = src->nb[0]; - const cl_ulong nb01 = src->nb[1]; - const cl_ulong nb02 = src->nb[2]; - const cl_ulong nb03 = src->nb[3]; + bool bcast_row = false; + cl_kernel kernel; - const int ne0 = src->ne[0]; - const int ne1 = src->ne[1]; - const int ne2 = src->ne[2]; - const int ne3 = src->ne[3]; + if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { + GGML_ASSERT(ggml_is_contiguous(src0)); - nb0 = tensor_type_size; - nb1 = tensor_type_size*ne00; - nb2 = tensor_type_size*ne00*ne01; - nb3 = tensor_type_size*ne00*ne01*ne02; + // src1 is a row + GGML_ASSERT(ne11 == 1); - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *)src->extra; + bcast_row = true; + int ne = ne00 / 4; - cl_ulong offset0 = extra->offset + src->view_offs; - cl_ulong offsetd = 0; + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_mul_row; + } else { + kernel = backend_ctx->kernel_mul_row_f16; + } - cl_kernel kernel; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_mul; + } else { + kernel = backend_ctx->kernel_mul_f16; + } - switch (src->type) { - case GGML_TYPE_F32: - kernel = backend_ctx->kernel_cpy_f32_f32; - break; - case GGML_TYPE_F16: - case GGML_TYPE_BF16: // stored as f16 on device - kernel = backend_ctx->kernel_cpy_f16_f16; - break; - default: - GGML_ASSERT(false && "not implemented"); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dst)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb3)); - - const int nth = MIN(64, ne00); + if (bcast_row) { + int n = ggml_nelements(dst)/4; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src); -} + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + unsigned int nth = MIN(64, ne0); + size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {nth, 1, 1}; -static void ggml_cl_nop(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - UNUSED(backend); - UNUSED(src0); - UNUSED(src1); - UNUSED(dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_div(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); @@ -13333,12 +18836,36 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - GGML_TENSOR_LOCALS(int, ne1, src1, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); - GGML_TENSOR_LOCALS(int, ne, dst, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; + + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; + + const int ne0 = dst->ne[0]; + + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; @@ -13350,479 +18877,444 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; + bool bcast_row = false; cl_kernel kernel; - switch (src0->type) { - case GGML_TYPE_F32: - kernel = backend_ctx->kernel_get_rows_f32; - break; - case GGML_TYPE_F16: - kernel = backend_ctx->kernel_get_rows_f16; - break; - case GGML_TYPE_Q4_0: - kernel = backend_ctx->kernel_get_rows_q4_0; - break; - default: - GGML_ASSERT(false && "not implemented"); - } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); + if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { + GGML_ASSERT(ggml_is_contiguous(src0)); - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - int nth = 1; - while (nth < ne00 && 2*nth <= max_workgroup_size) { - nth *= 2; - } + // src1 is a row + GGML_ASSERT(ne11 == 1); - int nchunks = 1; - if (src0->type == GGML_TYPE_F32) { - const int chunk_target = nth * 4; - nchunks = (ne00 + chunk_target - 1) / chunk_target; - nchunks = MAX(1, MIN(nchunks, 64)); - } + bcast_row = true; + int ne = ne00 / 4; - size_t global_work_size[] = {(size_t)ne10*nth*nchunks, (size_t)ne11, (size_t)ne12}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_div_row; + } else { + kernel = backend_ctx->kernel_div_row_f16; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_div; + } else { + kernel = backend_ctx->kernel_div_f16; + } -// check if a Q8_0 tensor has been SOA'ed in set_tensor -// we store SOA'ed tensors in a map in set_tensor, check against that map -static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor) { - if (tensor == nullptr || tensor->type != GGML_TYPE_Q8_0 || tensor->buffer == nullptr) { - return false; - } - auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; - if (ctx == nullptr) { - return false; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); } - const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; - return ctx->q8_0_soa_tensors.count(key) > 0; -} -// check if a Q4_0 tensor has been SOA'ed in set_tensor -// we store SOA'ed tensors in a map in set_tensor, check against that map -static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor) { - if (tensor == nullptr || tensor->type != GGML_TYPE_Q4_0 || tensor->buffer == nullptr) { - return false; - } - auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; - if (ctx == nullptr) { - return false; + if (bcast_row) { + int n = ggml_nelements(dst)/4; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + unsigned int nth = MIN(64, ne0); + size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } - const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; - return ctx->q4_0_soa_tensors.count(key) > 0; } -static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_sub(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32); - // ne0 = ne00 - // ne2 = ne02 - // ne3 = ne03 + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - GGML_TENSOR_LOCALS(int, ne1, src1, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - GGML_TENSOR_LOCALS(int, ne, dst, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; - const int nblk0 = ne0/ggml_blck_size(dst->type); + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; + + const int ne0 = dst->ne[0]; + + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - const bool q8_0_soa = dst->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(dst); - const bool q4_0_soa = dst->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(dst); - const bool is_soa = q8_0_soa || q4_0_soa; - + bool bcast_row = false; cl_kernel kernel; - if (q8_0_soa) { - kernel = (src1->type == GGML_TYPE_I64) - ? backend_ctx->kernel_set_rows_q8_0_soa_i64 - : backend_ctx->kernel_set_rows_q8_0_soa_i32; - } else if (q4_0_soa) { - kernel = (src1->type == GGML_TYPE_I64) - ? backend_ctx->kernel_set_rows_q4_0_soa_i64 - : backend_ctx->kernel_set_rows_q4_0_soa_i32; - } else { - switch (dst->type) { - case GGML_TYPE_F32: - kernel = (src1->type == GGML_TYPE_I64) - ? backend_ctx->kernel_set_rows_f32_i64 - : backend_ctx->kernel_set_rows_f32_i32; - break; - case GGML_TYPE_F16: - kernel = (src1->type == GGML_TYPE_I64) - ? backend_ctx->kernel_set_rows_f16_i64 - : backend_ctx->kernel_set_rows_f16_i32; - break; - case GGML_TYPE_Q8_0: - kernel = (src1->type == GGML_TYPE_I64) - ? backend_ctx->kernel_set_rows_q8_0_i64 - : backend_ctx->kernel_set_rows_q8_0_i32; - break; - case GGML_TYPE_Q4_0: - kernel = (src1->type == GGML_TYPE_I64) - ? backend_ctx->kernel_set_rows_q4_0_i64 - : backend_ctx->kernel_set_rows_q4_0_i32; - break; - default: - GGML_ABORT("not implemented"); - } - } + if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { + GGML_ASSERT(ggml_is_contiguous(src0)); - fastdiv_vals ne11_ = init_fastdiv_values(ne11); - fastdiv_vals ne12_ = init_fastdiv_values(ne12); + // src1 is a row + GGML_ASSERT(ne11 == 1); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + bcast_row = true; + int ne = ne00 / 4; - if (is_soa) { - // The q/d subbuffers in q8_0/q4_0 extras are interchangeable here. - // For views (e.g. ggml_set_rows' `out`), follow view_src for the SoA extra. - const ggml_tensor * soa_src = dst->view_src != nullptr ? dst->view_src : dst; - cl_mem q_mem = nullptr; - cl_mem d_mem = nullptr; - if (q8_0_soa) { - ggml_tensor_extra_cl_q8_0 * e = (ggml_tensor_extra_cl_q8_0 *)soa_src->extra; - q_mem = e->q; - d_mem = e->d; + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sub_row; } else { - ggml_tensor_extra_cl_q4_0 * e = (ggml_tensor_extra_cl_q4_0 *)soa_src->extra; - q_mem = e->q; - d_mem = e->d; + kernel = backend_ctx->kernel_sub_row_f16; } - cl_ulong offset_q = 0; - cl_ulong offset_d = 0; - const int ne1_dst = dst->ne[1]; - const int ne2_dst = dst->ne[2]; - const int ne3_dst = dst->ne[3]; - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &q_mem)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_q)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_mem)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_d)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(fastdiv_vals), &ne11_)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(fastdiv_vals), &ne12_)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &nblk0)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1_dst)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne2_dst)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne3_dst)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); } else { - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_ulong offsetd = extrad->offset + dst->view_offs; + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sub; + } else { + kernel = backend_ctx->kernel_sub_f16; + } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(fastdiv_vals), &ne11_)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(fastdiv_vals), &ne12_)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &nblk0)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); } - int nth0 = 64; - if (backend_ctx->gpu_family == INTEL) { - nth0 = 32; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - } + if (bcast_row) { + int n = ggml_nelements(dst)/4; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - while (nth0 < nblk0 && nth0 < max_workgroup_size) { - nth0 *= 2; - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + unsigned int nth = MIN(64, ne0); + size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {nth, 1, 1}; - int rows_per_workgroup = 1; - if (nth0 > nblk0) { - rows_per_workgroup = nth0 / nblk0; - nth0 = nblk0; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } - - size_t global_work_size[] = { - (size_t)(ne01 + rows_per_workgroup - 1)/rows_per_workgroup*nth0, - (size_t)ne02*rows_per_workgroup, - (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1}; - - // ne01 == 0 makes global_work_size[0] zero here; enqueue_ndrange_kernel drops the empty range. - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_sqr(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + UNUSED(src1); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + cl_kernel kernel; + + // Currently assumes src0 is contiguous + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sqr_cont_f32_4; + } else { + kernel = backend_ctx->kernel_sqr_cont_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sqr_cont_f32; + } else { + kernel = backend_ctx->kernel_sqr_cont_f16; + } + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); +} - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; +static void ggml_cl_sqrt(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; - const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; - - if (bcast_row) { - GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ne11 == 1); - } - - if (dst->type == GGML_TYPE_F32) { - GGML_ASSERT(src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32); - if (bcast_row) { - kernel = backend_ctx->kernel_add_row; - const int ne = ne00 / 4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + // Currently assumes src0 is contiguous + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sqrt_cont_f32_4; } else { - kernel = backend_ctx->kernel_add; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); + kernel = backend_ctx->kernel_sqrt_cont_f16_4; } - } else if (dst->type == GGML_TYPE_F16) { - GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); - const int type_src0 = (src0->type == GGML_TYPE_F32); - const int type_src1 = (src1->type == GGML_TYPE_F32); - if (bcast_row) { - kernel = backend_ctx->kernel_add_row_f16; - const int ne = ne00 / 4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &type_src0)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &type_src1)); + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sqrt_cont_f32; } else { - kernel = backend_ctx->kernel_add_f16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &type_src0)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &type_src1)); + kernel = backend_ctx->kernel_sqrt_cont_f16; } - } else { - GGML_ASSERT(false && "unsupported data types for add"); } - if (bcast_row) { - int n = ggml_nelements(dst)/4; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size_ptr, dst); - } else { - unsigned int nth = MIN(64, ne0); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {nth, 1, 1}; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_add_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + GGML_UNUSED(src1); - const ggml_tensor * src2 = dst->src[2]; - GGML_ASSERT(src2); - GGML_ASSERT(src2->extra); + GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(src2->type == GGML_TYPE_I32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - GGML_ASSERT(ggml_is_contiguous_rows(src0)); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - const cl_ulong nb21 = src2->nb[1]; + cl_kernel kernel; - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; + const bool is_c4 = ne00 % 4 == 0; + if (is_c4) { + kernel = backend_ctx->kernel_mean_f32_4; + } else { + kernel = backend_ctx->kernel_mean_f32; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb3)); + + size_t global_work_size[] = {64 * (size_t)ne01, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)64, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(dst->src[0]); + GGML_ASSERT(dst->src[0]->extra); + GGML_ASSERT(dst->src[1]); + GGML_ASSERT(dst->src[1]->extra); + GGML_ASSERT(dst->src[2]); + GGML_ASSERT(dst->src[2]->extra); + GGML_ASSERT(dst->src[3]); + GGML_ASSERT(dst->src[3]->extra); + GGML_ASSERT(dst->src[4]); + GGML_ASSERT(dst->src[4]->extra); + GGML_ASSERT(dst->src[5]); + GGML_ASSERT(dst->src[5]->extra); + GGML_ASSERT(dst->src[6]); + GGML_ASSERT(dst->src[6]->extra); + + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) dst->src[0]->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) dst->src[1]->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) dst->src[2]->extra; + ggml_tensor_extra_cl * extra3 = (ggml_tensor_extra_cl *) dst->src[3]->extra; + ggml_tensor_extra_cl * extra4 = (ggml_tensor_extra_cl *) dst->src[4]->extra; + ggml_tensor_extra_cl * extra5 = (ggml_tensor_extra_cl *) dst->src[5]->extra; + ggml_tensor_extra_cl * extra6 = (ggml_tensor_extra_cl *) dst->src[6]->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const cl_ulong offset0 = extra0->offset + dst->src[0]->view_offs; + const cl_ulong offset1 = extra1->offset + dst->src[1]->view_offs; + const cl_ulong offset2 = extra2->offset + dst->src[2]->view_offs; + const cl_ulong offset3 = extra3->offset + dst->src[3]->view_offs; + const cl_ulong offset4 = extra4->offset + dst->src[4]->view_offs; + const cl_ulong offset5 = extra5->offset + dst->src[5]->view_offs; + const cl_ulong offset6 = extra6->offset + dst->src[6]->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + const ggml_tensor * s = dst->src[0]; + const ggml_tensor * x = dst->src[1]; + const ggml_tensor * dt = dst->src[2]; + const ggml_tensor * A = dst->src[3]; + const ggml_tensor * B = dst->src[4]; + const ggml_tensor * C = dst->src[5]; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offset2 = extra2->offset + src2->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + const cl_ulong s_nb1 = s->nb[1]; + const cl_ulong s_nb2 = s->nb[2]; + const cl_ulong s_nb3 = s->nb[3]; + const cl_ulong x_nb1 = x->nb[1]; + const cl_ulong x_nb2 = x->nb[2]; + const cl_ulong x_nb3 = x->nb[3]; + const cl_ulong dt_nb1 = dt->nb[1]; + const cl_ulong dt_nb2 = dt->nb[2]; + const cl_ulong A_nb1 = A->nb[1]; + const cl_ulong B_nb1 = B->nb[1]; + const cl_ulong B_nb2 = B->nb[2]; + const cl_ulong B_nb3 = B->nb[3]; + const cl_ulong C_nb1 = C->nb[1]; + const cl_ulong C_nb2 = C->nb[2]; + const cl_ulong C_nb3 = C->nb[3]; - cl_kernel kernel = backend_ctx->kernel_add_id; + const cl_uint A_ne0 = A->ne[0]; + const cl_uint d_state = s->ne[0]; + const cl_int head_dim = x->ne[0]; + const cl_int n_head = x->ne[1]; + const cl_int n_group = B->ne[1]; + const cl_int n_tokens = x->ne[2]; + const cl_uint n_seqs = x->ne[3]; + const cl_uint K = ggml_get_op_params_i32(dst, 0); + const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(x) * sizeof(float); + + cl_kernel kernel = backend_ctx->kernel_ssm_scan_f32; + size_t nth = d_state; + if (A_ne0 == 1 && K == 1) { + cl_kernel kernel_mamba2 = nullptr; + if (d_state == 128) { + kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d128; + } else if (d_state == 256) { + kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d256; + } + if (kernel_mamba2 != nullptr) { + kernel = kernel_mamba2; + nth = 64; + } + } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); @@ -13830,23 +19322,55 @@ static void ggml_cl_add_id(ggml_backend_t backend, const ggml_tensor * src0, con CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb21)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra3->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extra4->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offset4)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &extra5->data_device)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &offset5)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &extra6->data_device)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &offset6)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s_nb2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s_nb3)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &x_nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &x_nb3)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &dt_nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &dt_nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &A_nb1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &B_nb2)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &B_nb3)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(cl_ulong), &C_nb2)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &C_nb3)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &s_off_bytes)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_int), &head_dim)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_int), &n_group)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_int), &n_tokens)); - int nth = MIN(ne00, (int) backend_ctx->get_kernel_workgroup_size(kernel)); - size_t global_work_size[] = { (size_t)ne01*nth, (size_t)ne02, 1 }; - size_t local_work_size[] = { (size_t)nth, 1, 1 }; + if (kernel == backend_ctx->kernel_ssm_scan_f32) { + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &s_nb1)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &x_nb1)); + CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &B_nb1)); + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &C_nb1)); + CL_CHECK(clSetKernelArg(kernel, 36, sizeof(cl_uint), &A_ne0)); + CL_CHECK(clSetKernelArg(kernel, 37, sizeof(cl_uint), &d_state)); + CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_uint), &n_seqs)); + CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_uint), &K)); + CL_CHECK(clSetKernelArg(kernel, 40, d_state * sizeof(float), nullptr)); + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + size_t global_work_size[] = { + (size_t) head_dim * (size_t) n_head * nth, + (size_t) n_seqs, + }; + size_t local_work_size[] = { nth, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } -static void ggml_cl_mul(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); @@ -13854,40 +19378,6 @@ static void ggml_cl_mul(ggml_backend_t backend, const ggml_tensor * src0, const GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == src1->type); - GGML_ASSERT(src0->type == dst->type); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; - - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; - - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; UNUSED(ne13); - - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; UNUSED(nb13); - - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; - - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; @@ -13898,337 +19388,172 @@ static void ggml_cl_mul(ggml_backend_t backend, const ggml_tensor * src0, const cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - bool bcast_row = false; - cl_kernel kernel; - - if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { - GGML_ASSERT(ggml_is_contiguous(src0)); - - // src1 is a row - GGML_ASSERT(ne11 == 1); + int ne01 = src0->ne[1]; + cl_ulong nb00 = src0->nb[0]; + cl_ulong nb01 = src0->nb[1]; + cl_ulong nb02 = src0->nb[2]; - bcast_row = true; - int ne = ne00 / 4; + int ne10 = src1->ne[0]; + cl_ulong nb11 = src1->nb[1]; - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_mul_row; - } else { - kernel = backend_ctx->kernel_mul_row_f16; - } + int ne1 = dst->ne[1]; + int ne2 = dst->ne[2]; + cl_ulong nb0 = dst->nb[0]; + cl_ulong nb1 = dst->nb[1]; + cl_ulong nb2 = dst->nb[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_mul; - } else { - kernel = backend_ctx->kernel_mul_f16; - } + cl_kernel kernel = backend_ctx->kernel_ssm_conv_f32_f32; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); + if (ne10 % 4 == 0) { + kernel = backend_ctx->kernel_ssm_conv_f32_f32_4; } - if (bcast_row) { - int n = ggml_nelements(dst)/4; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); - } else { - unsigned int nth = MIN(64, ne0); - size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {nth, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + size_t global_work_size[] = {(size_t)ne01, (size_t)ne1, (size_t)ne2}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (ne01 % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_div(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_gelu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == src1->type); - GGML_ASSERT(src0->type == dst->type); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; - - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; - - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; - - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; - - const int ne0 = dst->ne[0]; - - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - bool bcast_row = false; cl_kernel kernel; - if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { - GGML_ASSERT(ggml_is_contiguous(src0)); - - // src1 is a row - GGML_ASSERT(ne11 == 1); - - bcast_row = true; - int ne = ne00 / 4; - - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_div_row; - } else { - kernel = backend_ctx->kernel_div_row_f16; - } + int n = ggml_nelements(dst); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + if (n % 4 == 0) { + kernel = backend_ctx->kernel_gelu_4; + n /= 4; } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_div; - } else { - kernel = backend_ctx->kernel_div_f16; - } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); + kernel = backend_ctx->kernel_gelu; } - if (bcast_row) { - int n = ggml_nelements(dst)/4; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - unsigned int nth = MIN(64, ne0); - size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {nth, 1, 1}; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_sub(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_gelu_erf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == src1->type); - GGML_ASSERT(src0->type == dst->type); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + UNUSED(src1); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + cl_kernel kernel; - const int ne0 = dst->ne[0]; + int n = ggml_nelements(dst); - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + if (n % 4 == 0) { + kernel = backend_ctx->kernel_gelu_erf_4; + n /= 4; + } else { + kernel = backend_ctx->kernel_gelu_erf; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_gelu_quick(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - bool bcast_row = false; cl_kernel kernel; - if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { - GGML_ASSERT(ggml_is_contiguous(src0)); - - // src1 is a row - GGML_ASSERT(ne11 == 1); - - bcast_row = true; - int ne = ne00 / 4; - - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sub_row; - } else { - kernel = backend_ctx->kernel_sub_row_f16; - } + int n = ggml_nelements(dst); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + if (n % 4 == 0) { + kernel = backend_ctx->kernel_gelu_quick_4; + n /= 4; } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sub; - } else { - kernel = backend_ctx->kernel_sub_f16; - } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); + kernel = backend_ctx->kernel_gelu_quick; } - if (bcast_row) { - int n = ggml_nelements(dst)/4; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - unsigned int nth = MIN(64, ne0); - size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {nth, 1, 1}; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_sqr(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_silu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; @@ -14241,21 +19566,13 @@ static void ggml_cl_sqr(ggml_backend_t backend, const ggml_tensor * src0, const cl_kernel kernel; - // Currently assumes src0 is contiguous int n = ggml_nelements(dst); + if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sqr_cont_f32_4; - } else { - kernel = backend_ctx->kernel_sqr_cont_f16_4; - } + kernel = backend_ctx->kernel_silu_4; n /= 4; } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sqr_cont_f32; - } else { - kernel = backend_ctx->kernel_sqr_cont_f16; - } + kernel = backend_ctx->kernel_silu; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); @@ -14268,17 +19585,18 @@ static void ggml_cl_sqr(ggml_backend_t backend, const ggml_tensor * src0, const size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_sqrt(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_relu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; @@ -14289,49 +19607,33 @@ static void ggml_cl_sqrt(ggml_backend_t backend, const ggml_tensor * src0, const cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel; - - // Currently assumes src0 is contiguous - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sqrt_cont_f32_4; - } else { - kernel = backend_ctx->kernel_sqrt_cont_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sqrt_cont_f32; - } else { - kernel = backend_ctx->kernel_sqrt_cont_f16; - } - } + cl_kernel kernel = backend_ctx->kernel_relu; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + const int64_t n = ggml_nelements(dst); + size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_sigmoid(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_UNUSED(src1); - GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); + UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; @@ -14341,292 +19643,166 @@ static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + cl_kernel kernel = src0->type == GGML_TYPE_F32 ? backend_ctx->kernel_sigmoid_f32_nc : backend_ctx->kernel_sigmoid_f16_nc; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - cl_kernel kernel; + size_t global_work_size[] = {(size_t)ne01 * 64, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } - const bool is_c4 = ne00 % 4 == 0; - if (is_c4) { - kernel = backend_ctx->kernel_mean_f32_4; + cl_kernel kernel; + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sigmoid_f32; + } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { + kernel = backend_ctx->kernel_sigmoid_f16; + } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { + // bf16 converted to f16 + kernel = backend_ctx->kernel_sigmoid_f16; } else { - kernel = backend_ctx->kernel_mean_f32; + GGML_ASSERT(false && "Unsupported data types for sigmoid (input and output must be both f32 or f16)"); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb3)); - - size_t global_work_size[] = {64 * (size_t)ne01, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)64, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} - -static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_ASSERT(dst->src[0]); - GGML_ASSERT(dst->src[0]->extra); - GGML_ASSERT(dst->src[1]); - GGML_ASSERT(dst->src[1]->extra); - GGML_ASSERT(dst->src[2]); - GGML_ASSERT(dst->src[2]->extra); - GGML_ASSERT(dst->src[3]); - GGML_ASSERT(dst->src[3]->extra); - GGML_ASSERT(dst->src[4]); - GGML_ASSERT(dst->src[4]->extra); - GGML_ASSERT(dst->src[5]); - GGML_ASSERT(dst->src[5]->extra); - GGML_ASSERT(dst->src[6]); - GGML_ASSERT(dst->src[6]->extra); - - ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; - - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) dst->src[0]->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) dst->src[1]->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) dst->src[2]->extra; - ggml_tensor_extra_cl * extra3 = (ggml_tensor_extra_cl *) dst->src[3]->extra; - ggml_tensor_extra_cl * extra4 = (ggml_tensor_extra_cl *) dst->src[4]->extra; - ggml_tensor_extra_cl * extra5 = (ggml_tensor_extra_cl *) dst->src[5]->extra; - ggml_tensor_extra_cl * extra6 = (ggml_tensor_extra_cl *) dst->src[6]->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; - - const cl_ulong offset0 = extra0->offset + dst->src[0]->view_offs; - const cl_ulong offset1 = extra1->offset + dst->src[1]->view_offs; - const cl_ulong offset2 = extra2->offset + dst->src[2]->view_offs; - const cl_ulong offset3 = extra3->offset + dst->src[3]->view_offs; - const cl_ulong offset4 = extra4->offset + dst->src[4]->view_offs; - const cl_ulong offset5 = extra5->offset + dst->src[5]->view_offs; - const cl_ulong offset6 = extra6->offset + dst->src[6]->view_offs; - const cl_ulong offsetd = extrad->offset + dst->view_offs; - - const ggml_tensor * s = dst->src[0]; - const ggml_tensor * x = dst->src[1]; - const ggml_tensor * dt = dst->src[2]; - const ggml_tensor * A = dst->src[3]; - const ggml_tensor * B = dst->src[4]; - const ggml_tensor * C = dst->src[5]; - - const cl_ulong s_nb1 = s->nb[1]; - const cl_ulong s_nb2 = s->nb[2]; - const cl_ulong s_nb3 = s->nb[3]; - const cl_ulong x_nb1 = x->nb[1]; - const cl_ulong x_nb2 = x->nb[2]; - const cl_ulong x_nb3 = x->nb[3]; - const cl_ulong dt_nb1 = dt->nb[1]; - const cl_ulong dt_nb2 = dt->nb[2]; - const cl_ulong A_nb1 = A->nb[1]; - const cl_ulong B_nb1 = B->nb[1]; - const cl_ulong B_nb2 = B->nb[2]; - const cl_ulong B_nb3 = B->nb[3]; - const cl_ulong C_nb1 = C->nb[1]; - const cl_ulong C_nb2 = C->nb[2]; - const cl_ulong C_nb3 = C->nb[3]; - - const cl_uint A_ne0 = A->ne[0]; - const cl_uint d_state = s->ne[0]; - const cl_int head_dim = x->ne[0]; - const cl_int n_head = x->ne[1]; - const cl_int n_group = B->ne[1]; - const cl_int n_tokens = x->ne[2]; - const cl_uint n_seqs = x->ne[3]; - const cl_uint K = ggml_get_op_params_i32(dst, 0); - const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(x) * sizeof(float); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - cl_kernel kernel = backend_ctx->kernel_ssm_scan_f32; - size_t nth = d_state; - if (A_ne0 == 1 && K == 1) { - cl_kernel kernel_mamba2 = nullptr; - if (d_state == 128) { - kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d128; - } else if (d_state == 256) { - kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d256; - } - if (kernel_mamba2 != nullptr) { - kernel = kernel_mamba2; - nth = 64; - } - } + const int64_t n = ggml_nelements(dst); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra3->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset3)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extra4->data_device)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offset4)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &extra5->data_device)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &offset5)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &extra6->data_device)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &offset6)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s_nb2)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s_nb3)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &x_nb2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &x_nb3)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &dt_nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &dt_nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &A_nb1)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &B_nb2)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &B_nb3)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(cl_ulong), &C_nb2)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &C_nb3)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &s_off_bytes)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_int), &head_dim)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_int), &n_group)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_int), &n_tokens)); + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - if (kernel == backend_ctx->kernel_ssm_scan_f32) { - CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &s_nb1)); - CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &x_nb1)); - CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &B_nb1)); - CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &C_nb1)); - CL_CHECK(clSetKernelArg(kernel, 36, sizeof(cl_uint), &A_ne0)); - CL_CHECK(clSetKernelArg(kernel, 37, sizeof(cl_uint), &d_state)); - CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_uint), &n_seqs)); - CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_uint), &K)); - CL_CHECK(clSetKernelArg(kernel, 40, d_state * sizeof(float), nullptr)); + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } - size_t global_work_size[] = { - (size_t) head_dim * (size_t) n_head * nth, - (size_t) n_seqs, - }; - size_t local_work_size[] = { nth, 1 }; - - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_tri(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + UNUSED(src1); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - int ne01 = src0->ne[1]; - cl_ulong nb00 = src0->nb[0]; - cl_ulong nb01 = src0->nb[1]; - cl_ulong nb02 = src0->nb[2]; + const int tri_type = ggml_get_op_params_i32(dst, 0); + const int64_t n = ggml_nelements(dst); + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; - int ne10 = src1->ne[0]; - cl_ulong nb11 = src1->nb[1]; + cl_kernel kernel = backend_ctx->kernel_tri; - int ne1 = dst->ne[1]; - int ne2 = dst->ne[2]; - cl_ulong nb0 = dst->nb[0]; - cl_ulong nb1 = dst->nb[1]; - cl_ulong nb2 = dst->nb[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &tri_type)); - cl_kernel kernel = backend_ctx->kernel_ssm_conv_f32_f32; + size_t local_work_size[1] = { 256 }; + size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; - if (ne10 % 4 == 0) { - kernel = backend_ctx->kernel_ssm_conv_f32_f32_4; - } + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); +} - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2)); +static void ggml_cl_fill(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - size_t global_work_size[] = {(size_t)ne01, (size_t)ne1, (size_t)ne2}; - size_t local_work_size[] = {64, 1, 1}; + UNUSED(src0); + UNUSED(src1); - size_t * local_work_size_ptr = local_work_size; - if (ne01 % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + float v = 0.0f; + memcpy(&v, ((int32_t *) dst->op_params), sizeof(float)); + + const int64_t n = ggml_nelements(dst); + + cl_kernel kernel = backend_ctx->kernel_fill; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(float), &v)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &n)); + + size_t local_work_size[1] = { 256 }; + size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); } -static void ggml_cl_gelu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); +static void ggml_cl_arange(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + UNUSED(src0); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - - cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel; + float start; + float step; + memcpy(&start, (float *) dst->op_params + 0, sizeof(float)); + memcpy(&step, (float *) dst->op_params + 2, sizeof(float)); - int n = ggml_nelements(dst); + const int n = dst->ne[0]; - if (n % 4 == 0) { - kernel = backend_ctx->kernel_gelu_4; - n /= 4; - } else { - kernel = backend_ctx->kernel_gelu; - } + cl_kernel kernel = backend_ctx->kernel_arange; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &start)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &step)); - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + size_t local_work_size[1] = { 64 }; + size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); } -static void ggml_cl_gelu_erf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); @@ -14642,29 +19818,74 @@ static void ggml_cl_gelu_erf(ggml_backend_t backend, const ggml_tensor * src0, c cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel; + float min; + float max; + memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float)); + memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float)); - int n = ggml_nelements(dst); + if (!ggml_is_contiguous(src0)) { + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + cl_kernel kernel = backend_ctx->kernel_clamp_nc; - if (n % 4 == 0) { - kernel = backend_ctx->kernel_gelu_erf_4; - n /= 4; - } else { - kernel = backend_ctx->kernel_gelu_erf; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb3)); + + const int nth = 64; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; } + cl_kernel kernel = backend_ctx->kernel_clamp; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max)); + + const int64_t n = ggml_nelements(dst); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_gelu_quick(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); @@ -14680,29 +19901,41 @@ static void ggml_cl_gelu_quick(ggml_backend_t backend, const ggml_tensor * src0, cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel; + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); - int n = ggml_nelements(dst); + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - if (n % 4 == 0) { - kernel = backend_ctx->kernel_gelu_quick_4; - n /= 4; - } else { - kernel = backend_ctx->kernel_gelu_quick; + int nth = 1; + while (nth < ne00 && nth < 64) { + nth *= 2; } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + cl_kernel kernel = backend_ctx->kernel_norm; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float)*nth, NULL)); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_silu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); @@ -14712,190 +19945,317 @@ static void ggml_cl_silu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + //ggml_backend_opencl_device_context * dev_ctx = + // (ggml_backend_opencl_device_context *)backend->device->context; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel; + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); - int n = ggml_nelements(dst); + const int ne00 = src0 ? src0->ne[0] : 0; + const int ne01 = src0 ? src0->ne[1] : 0; + const int ne02 = src0 ? src0->ne[2] : 0; + const int ne03 = src0 ? src0->ne[3] : 0; - if (n % 4 == 0) { - kernel = backend_ctx->kernel_silu_4; - n /= 4; - } else { - kernel = backend_ctx->kernel_silu; - } + const cl_ulong nb01 = src0 ? src0->nb[1] : 0; + const cl_ulong nb02 = src0 ? src0->nb[2] : 0; + const cl_ulong nb03 = src0 ? src0->nb[3] : 0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + GGML_ASSERT(ne00 % 4 == 0); - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + const int nth = 64; - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + cl_kernel kernel = backend_ctx->kernel_rms_norm; + + // Note, this kernel declares local memory in kernel args and the size + // depends on subgroup size. + // Note, this requires OpenCL 2.1 and above + // For now we use fixed subgroup size to simplify support for OpenCL 2.0. + size_t sgs; + //CL_CHECK(clGetKernelSubGroupInfo(kernel, dev_ctx->device, + // CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, + // sizeof(local_work_size), local_work_size, + // sizeof(size_t), &sgs, NULL)); + if (backend_ctx->gpu_family == ADRENO) { + sgs = 64; + } else if (backend_ctx->gpu_family == INTEL) { + sgs = 32; + } else { + GGML_ASSERT(false && "Unsupported GPU"); } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); + // This is local memory - the size depends on subgroup size. + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_relu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor) { + GGML_ASSERT(mul_tensor); + GGML_ASSERT(rms_norm_tensor); + + // src0 is the src of rms_norm, src1 is the other src of mul (one being rms_norm) + const ggml_tensor * src0 = rms_norm_tensor->src[0]; + const ggml_tensor * src1; + if (mul_tensor->src[0] == rms_norm_tensor) { + src1 = mul_tensor->src[1]; + } else if (mul_tensor->src[1] == rms_norm_tensor) { + src1 = mul_tensor->src[0]; + } else { + GGML_ASSERT(false && "Invalid args for rms_norm and mul"); + } + const ggml_tensor * dst = mul_tensor; + GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - UNUSED(src1); - - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel = backend_ctx->kernel_relu; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - - const int64_t n = ggml_nelements(dst); - - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } + float eps; + memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); -} + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; -static void ggml_cl_sigmoid(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - UNUSED(src1); + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + GGML_ASSERT(ne00 % 4 == 0); - cl_kernel kernel; - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sigmoid_f32; - } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - kernel = backend_ctx->kernel_sigmoid_f16; - } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { - // bf16 converted to f16 - kernel = backend_ctx->kernel_sigmoid_f16; + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) { + sgs = 64; + } else if (backend_ctx->gpu_family == INTEL) { + sgs = 32; } else { - GGML_ASSERT(false && "Unsupported data types for sigmoid (input and output must be both f32 or f16)"); + GGML_ASSERT(false && "Unsupported GPU"); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + cl_kernel kernel = backend_ctx->kernel_rms_norm_mul; - const int64_t n = ggml_nelements(dst); + int nth = sgs; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth < ne00 && nth < max_workgroup_size) { + nth *= 2; + } + nth = MIN(nth, max_workgroup_size); + nth = MIN(nth, ne00); - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs, NULL)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_tri(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - - UNUSED(src1); +static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { + GGML_ASSERT(norm_tensor && mul_tensor && add_tensor); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const ggml_tensor * src0 = norm_tensor->src[0]; + const ggml_tensor * src1 = mul_tensor->src[0] == norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; + const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; + const ggml_tensor * dst = add_tensor; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const int tri_type = ggml_get_op_params_i32(dst, 0); - const int64_t n = ggml_nelements(dst); - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - cl_kernel kernel = backend_ctx->kernel_tri; + float eps; + memcpy(&eps, norm_tensor->op_params, sizeof(float)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &tri_type)); + const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; + const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; + const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; + const cl_ulong nbd1 = dst->nb[1], nbd2 = dst->nb[2], nbd3 = dst->nb[3]; - size_t local_work_size[1] = { 256 }; - size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) sgs = 64; + else if (backend_ctx->gpu_family == INTEL) sgs = 32; + else GGML_ASSERT(false && "Unsupported GPU"); - backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); -} + cl_kernel kernel = backend_ctx->kernel_norm_mul_add; -static void ggml_cl_fill(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + int nth = sgs; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth < ne00/4 && nth < max_workgroup_size) nth *= 2; + nth = MIN(nth, max_workgroup_size); + nth = MIN(nth, ne00/4); - UNUSED(src0); - UNUSED(src1); + size_t gws[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t lws[] = {(size_t)nth, 1, 1}; + size_t num_subgroups = (nth + sgs - 1) / sgs; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nbd1)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nbd2)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nbd3)); + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_float2) * num_subgroups, NULL)); + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); +} + +static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { + GGML_ASSERT(gn_tensor && mul_tensor && add_tensor); + + const ggml_tensor * src0 = gn_tensor->src[0]; + const ggml_tensor * src1 = mul_tensor->src[0] == gn_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; + const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; + const ggml_tensor * dst = add_tensor; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - float v = 0.0f; - memcpy(&v, ((int32_t *) dst->op_params), sizeof(float)); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const int64_t n = ggml_nelements(dst); + int groups; + float eps; + memcpy(&groups, gn_tensor->op_params, sizeof(int)); + memcpy(&eps, (char *)gn_tensor->op_params + sizeof(int), sizeof(float)); - cl_kernel kernel = backend_ctx->kernel_fill; + cl_kernel kernel = backend_ctx->kernel_group_norm_mul_add; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + int ne = ggml_nelements(src0); + int group_size = ne / groups; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(float), &v)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &n)); + size_t lws[] = { (size_t)MIN(max_workgroup_size, group_size) }; + size_t gws[] = { (size_t)groups * lws[0] }; - size_t local_work_size[1] = { 256 }; - size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &group_size)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(float), &eps)); - backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 1, gws, lws, dst); } -static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_group_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); @@ -14911,34 +20271,41 @@ static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, cons cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - float min; - float max; - memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float)); - memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float)); + int32_t n_groups = ((const int32_t *) dst->op_params)[0]; + int32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + n_groups - 1) / n_groups); + float eps = ((const float *) dst->op_params)[1]; - cl_kernel kernel = backend_ctx->kernel_clamp; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne = ne00*ne01*ne02; + + cl_kernel kernel = backend_ctx->kernel_group_norm; + + size_t sgs = 64; + if (backend_ctx->gpu_family == ADRENO) { + sgs = 64; + } else if (backend_ctx->gpu_family == INTEL) { + sgs = 32; + } else { + GGML_ASSERT(false && "Unsupported GPU"); + } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max)); - - const int64_t n = ggml_nelements(dst); - - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &group_size)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(float), &eps)); - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } + size_t global_work_size[] = {(size_t)n_groups*sgs, 1, 1}; + size_t local_work_size[] = {(size_t)sgs, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_l2_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); @@ -14960,12 +20327,21 @@ static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - int nth = 1; - while (nth < ne00 && nth < 64) { - nth *= 2; + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) { + sgs = 64; + } else if (backend_ctx->gpu_family == INTEL) { + sgs = 32; + } else { + GGML_ASSERT(false && "Unsupported GPU"); } - cl_kernel kernel = backend_ctx->kernel_norm; + cl_kernel kernel = backend_ctx->kernel_l2_norm_f32; + + int nth = sgs; + while (nth < ne00 && nth < (int)backend_ctx->get_kernel_workgroup_size(kernel)) { + nth *= 2; + } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); @@ -14975,12 +20351,11 @@ static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float)*nth, NULL)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; @@ -14988,7 +20363,7 @@ static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); @@ -14998,317 +20373,353 @@ static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, c ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - //ggml_backend_opencl_device_context * dev_ctx = - // (ggml_backend_opencl_device_context *)backend->device->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - float eps; - memcpy(&eps, dst->op_params, sizeof(float)); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - const int ne00 = src0 ? src0->ne[0] : 0; - const int ne01 = src0 ? src0->ne[1] : 0; - const int ne02 = src0 ? src0->ne[2] : 0; - const int ne03 = src0 ? src0->ne[3] : 0; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - const cl_ulong nb01 = src0 ? src0->nb[1] : 0; - const cl_ulong nb02 = src0 ? src0->nb[2] : 0; - const cl_ulong nb03 = src0 ? src0->nb[3] : 0; + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - GGML_ASSERT(ne00 % 4 == 0); + cl_kernel kernel; - const int nth = MIN(64, ne00); + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_tanh_f32_4; + } else { + kernel = backend_ctx->kernel_tanh_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_tanh_f32; + } else { + kernel = backend_ctx->kernel_tanh_f16; + } + } - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - cl_kernel kernel = backend_ctx->kernel_rms_norm; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - // Note, this kernel declares local memory in kernel args and the size - // depends on subgroup size. - // Note, this requires OpenCL 2.1 and above - // For now we use fixed subgroup size to simplify support for OpenCL 2.0. - size_t sgs; - //CL_CHECK(clGetKernelSubGroupInfo(kernel, dev_ctx->device, - // CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, - // sizeof(local_work_size), local_work_size, - // sizeof(size_t), &sgs, NULL)); - if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - } else if (backend_ctx->gpu_family == INTEL) { - sgs = 32; + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { - GGML_ASSERT(false && "Unsupported GPU"); - } + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_tanh_f32_nc; + } else { + kernel = backend_ctx->kernel_tanh_f16_nc; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); - // This is local memory - the size depends on subgroup size. - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} + int nth = 64; -static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor) { - GGML_ASSERT(mul_tensor); - GGML_ASSERT(rms_norm_tensor); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - // src0 is the src of rms_norm, src1 is the other src of mul (one being rms_norm) - const ggml_tensor * src0 = rms_norm_tensor->src[0]; - const ggml_tensor * src1; - if (mul_tensor->src[0] == rms_norm_tensor) { - src1 = mul_tensor->src[1]; - } else if (mul_tensor->src[1] == rms_norm_tensor) { - src1 = mul_tensor->src[0]; - } else { - GGML_ASSERT(false && "Invalid args for rms_norm and mul"); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } - const ggml_tensor * dst = mul_tensor; +} +static void ggml_cl_neg(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - float eps; - memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; + cl_kernel kernel; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_neg_f32_4; + } else { + kernel = backend_ctx->kernel_neg_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_neg_f32; + } else { + kernel = backend_ctx->kernel_neg_f16; + } + } - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); - GGML_ASSERT(ne00 % 4 == 0); + size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - size_t sgs; - if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - } else if (backend_ctx->gpu_family == INTEL) { - sgs = 32; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { - GGML_ASSERT(false && "Unsupported GPU"); - } - - cl_kernel kernel = backend_ctx->kernel_rms_norm_mul; + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_neg_f32_nc; + } else { + kernel = backend_ctx->kernel_neg_f16_nc; + } - int nth = sgs; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - while (nth < ne00 && nth < max_workgroup_size) { - nth *= 2; - } - nth = MIN(nth, max_workgroup_size); - nth = MIN(nth, ne00); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + int nth = 64; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs, NULL)); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { - GGML_ASSERT(norm_tensor && mul_tensor && add_tensor); +static void ggml_cl_exp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - const ggml_tensor * src0 = norm_tensor->src[0]; - const ggml_tensor * src1 = mul_tensor->src[0] == norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; - const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; - const ggml_tensor * dst = add_tensor; + UNUSED(src1); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); - float eps; - memcpy(&eps, norm_tensor->op_params, sizeof(float)); + cl_kernel kernel; - const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; - const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; - const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; - const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; - const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; - const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; - const cl_ulong nbd1 = dst->nb[1], nbd2 = dst->nb[2], nbd3 = dst->nb[3]; + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_exp_f32_4; + } else { + kernel = backend_ctx->kernel_exp_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_exp_f32; + } else { + kernel = backend_ctx->kernel_exp_f16; + } + } - size_t sgs; - if (backend_ctx->gpu_family == ADRENO) sgs = 64; - else if (backend_ctx->gpu_family == INTEL) sgs = 32; - else GGML_ASSERT(false && "Unsupported GPU"); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); - cl_kernel kernel = backend_ctx->kernel_norm_mul_add; + size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - int nth = sgs; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - while (nth < ne00/4 && nth < max_workgroup_size) nth *= 2; - nth = MIN(nth, max_workgroup_size); - nth = MIN(nth, ne00/4); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_exp_f32_nc; + } else { + kernel = backend_ctx->kernel_exp_f16_nc; + } - size_t gws[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t lws[] = {(size_t)nth, 1, 1}; - size_t num_subgroups = (nth + sgs - 1) / sgs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nbd1)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nbd2)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nbd3)); - CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_float2) * num_subgroups, NULL)); + int nth = 64; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { - GGML_ASSERT(gn_tensor && mul_tensor && add_tensor); +static void ggml_cl_expm1(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - const ggml_tensor * src0 = gn_tensor->src[0]; - const ggml_tensor * src1 = mul_tensor->src[0] == gn_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; - const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; - const ggml_tensor * dst = add_tensor; + UNUSED(src1); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - int groups; - float eps; - memcpy(&groups, gn_tensor->op_params, sizeof(int)); - memcpy(&eps, (char *)gn_tensor->op_params + sizeof(int), sizeof(float)); + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - cl_kernel kernel = backend_ctx->kernel_group_norm_mul_add; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - int ne = ggml_nelements(src0); - int group_size = ne / groups; + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - size_t lws[] = { (size_t)MIN(max_workgroup_size, group_size) }; - size_t gws[] = { (size_t)groups * lws[0] }; + cl_kernel kernel; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &group_size)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(float), &eps)); + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_expm1_f32_4; + } else { + kernel = backend_ctx->kernel_expm1_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_expm1_f32; + } else { + kernel = backend_ctx->kernel_expm1_f16; + } + } - backend_ctx->enqueue_ndrange_kernel(kernel, 1, gws, lws, dst); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_expm1_f32_nc; + } else { + kernel = backend_ctx->kernel_expm1_f16_nc; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + + int nth = 64; + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -static void ggml_cl_group_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); @@ -15324,48 +20735,97 @@ static void ggml_cl_group_norm(ggml_backend_t backend, const ggml_tensor * src0, cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - int32_t n_groups = ((const int32_t *) dst->op_params)[0]; - int32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + n_groups - 1) / n_groups); - float eps = ((const float *) dst->op_params)[1]; - const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; - const int ne = ne00*ne01*ne02; + const int ne03 = src0->ne[3]; - cl_kernel kernel = backend_ctx->kernel_group_norm; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - size_t sgs = 64; - if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - } else if (backend_ctx->gpu_family == INTEL) { - sgs = 32; + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + cl_kernel kernel; + + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_abs_f32_4; + } else { + kernel = backend_ctx->kernel_abs_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_abs_f32; + } else { + kernel = backend_ctx->kernel_abs_f16; + } + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { - GGML_ASSERT(false && "Unsupported GPU"); - } + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_abs_f32_nc; + } else { + kernel = backend_ctx->kernel_abs_f16_nc; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &group_size)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - size_t global_work_size[] = {(size_t)n_groups*sgs, 1, 1}; - size_t local_work_size[] = {(size_t)sgs, 1, 1}; + int nth = 64; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -static void ggml_cl_l2_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +// Shared driver for the extended unary ops (unary_ext.cl), same selection as +// ggml_cl_abs: contiguous picks the vec4 kernel when the element count is a +// multiple of 4 (else scalar); non-contiguous uses the stride-addressed kernel. +static void ggml_cl_unary_ext(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst, + cl_kernel k_f32, cl_kernel k_f32_4, cl_kernel k_f32_nc, + cl_kernel k_f16, cl_kernel k_f16_4, cl_kernel k_f16_nc) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - UNUSED(src1); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; @@ -15374,49 +20834,79 @@ static void ggml_cl_l2_norm(ggml_backend_t backend, const ggml_tensor * src0, co cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - float eps; - memcpy(&eps, dst->op_params, sizeof(float)); + const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const cl_ulong nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const cl_ulong nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + const bool is_f16 = (src0->type == GGML_TYPE_F16); + cl_kernel kernel; - size_t sgs; - if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - } else if (backend_ctx->gpu_family == INTEL) { - sgs = 32; + if (ggml_is_contiguous(src0)) { + int n = ggml_nelements(dst); + if (n % 4 == 0) { + kernel = is_f16 ? k_f16_4 : k_f32_4; + n /= 4; + } else { + kernel = is_f16 ? k_f16 : k_f32; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { - GGML_ASSERT(false && "Unsupported GPU"); - } + kernel = is_f16 ? k_f16_nc : k_f32_nc; - cl_kernel kernel = backend_ctx->kernel_l2_norm_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - int nth = sgs; - while (nth < ne00 && nth < (int)backend_ctx->get_kernel_workgroup_size(kernel)) { - nth *= 2; + int nth = 64; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +} - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); +#define GGML_CL_UNARY_EXT_WRAP(FN, OP) \ +static void FN(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { \ + UNUSED(src1); \ + ggml_backend_opencl_context *c = (ggml_backend_opencl_context *)backend->context; \ + ggml_cl_unary_ext(backend, src0, dst, c->kernel_##OP##_f32, c->kernel_##OP##_f32_4, c->kernel_##OP##_f32_nc, \ + c->kernel_##OP##_f16, c->kernel_##OP##_f16_4, c->kernel_##OP##_f16_nc); \ +} - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; +GGML_CL_UNARY_EXT_WRAP(ggml_cl_sgn, sgn) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_step, step) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_elu, elu) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardswish, hardswish) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardsigmoid, hardsigmoid) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_floor, floor) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_ceil, ceil) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_round, round) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_trunc, trunc) - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} +#undef GGML_CL_UNARY_EXT_WRAP -static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); @@ -15442,10 +20932,10 @@ static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel; @@ -15454,16 +20944,16 @@ static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_tanh_f32_4; + kernel = backend_ctx->kernel_softplus_f32_4; } else { - kernel = backend_ctx->kernel_tanh_f16_4; + kernel = backend_ctx->kernel_softplus_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_tanh_f32; + kernel = backend_ctx->kernel_softplus_f32; } else { - kernel = backend_ctx->kernel_tanh_f16; + kernel = backend_ctx->kernel_softplus_f16; } } @@ -15484,9 +20974,9 @@ static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const } else { // Handle non-contiguous input if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_tanh_f32_nc; + kernel = backend_ctx->kernel_softplus_f32_nc; } else { - kernel = backend_ctx->kernel_tanh_f16_nc; + kernel = backend_ctx->kernel_softplus_f16_nc; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); @@ -15512,185 +21002,314 @@ static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const } } -static void ggml_cl_neg(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_repeat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1_shape_def, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + GGML_ASSERT(dst->type == src0->type); - UNUSED(src1); + UNUSED(src1_shape_def); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - GGML_TENSOR_LOCALS(int, ne, dst, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; + + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel; + switch (src0->type) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_repeat_f32; + break; + case GGML_TYPE_F16: + case GGML_TYPE_I16: + case GGML_TYPE_BF16: + kernel = backend_ctx->kernel_repeat_f16; + break; + case GGML_TYPE_I32: + kernel = backend_ctx->kernel_repeat_i32; + break; + default: + GGML_ABORT("unsupported type"); + } - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_neg_f32_4; - } else { - kernel = backend_ctx->kernel_neg_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_neg_f32; - } else { - kernel = backend_ctx->kernel_neg_f16; - } - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); + int nth = 64; - size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_neg_f32_nc; - } else { - kernel = backend_ctx->kernel_neg_f16_nc; - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); +static void ggml_cl_pad(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); - int nth = 64; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + if (backend_ctx->kernel_pad == nullptr) { + GGML_LOG_WARN("%s: pad kernel not available, skipping OpenCL execution.\n", __func__); + return; + } + + ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong off_src0 = extra_src0->offset + src0->view_offs; + cl_ulong off_dst = extra_dst->offset + dst->view_offs; + + const int s_ne0 = src0->ne[0]; + const int s_ne1 = src0->ne[1]; + const int s_ne2 = src0->ne[2]; + const int s_ne3 = src0->ne[3]; + + const int s_nb0 = src0->nb[0]; + const int s_nb1 = src0->nb[1]; + const int s_nb2 = src0->nb[2]; + const int s_nb3 = src0->nb[3]; + + const int d_ne0 = dst->ne[0]; + const int d_ne1 = dst->ne[1]; + const int d_ne2 = dst->ne[2]; + const int d_ne3 = dst->ne[3]; + + const int d_nb0 = dst->nb[0]; + const int d_nb1 = dst->nb[1]; + const int d_nb2 = dst->nb[2]; + const int d_nb3 = dst->nb[3]; + + const int lp0 = ((const int*)(dst->op_params))[0]; + const int rp0 = ((const int*)(dst->op_params))[1]; + const int lp1 = ((const int*)(dst->op_params))[2]; + const int rp1 = ((const int*)(dst->op_params))[3]; + const int lp2 = ((const int*)(dst->op_params))[4]; + const int rp2 = ((const int*)(dst->op_params))[5]; + const int lp3 = ((const int*)(dst->op_params))[6]; + const int rp3 = ((const int*)(dst->op_params))[7]; + + cl_kernel kernel = backend_ctx->kernel_pad; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &s_ne0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &s_ne1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &s_ne2)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &s_ne3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &s_nb0)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &s_nb1)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &s_nb2)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &s_nb3)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &d_ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &d_ne1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &d_ne2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &d_ne3)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &d_nb0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &d_nb1)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &d_nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &d_nb3)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &lp0)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &rp0)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &lp1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &rp1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &lp2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &rp2)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &lp3)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(int), &rp3)); + + size_t lws0 = 64; + size_t gws0 = (( (size_t)d_ne0 + lws0 - 1 ) / lws0) * lws0; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + size_t global_work_size[] = { gws0, (size_t)d_ne1, (size_t)d_ne2*d_ne3 }; + size_t local_work_size[] = { lws0, 1, 1 }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + size_t * local_work_size_ptr = local_work_size; + if (d_ne0 % lws0 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_exp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_upscale(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - - UNUSED(src1); + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + const int mode_flags = (ggml_scale_mode) ggml_get_op_params_i32(dst, 0); + const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF); + cl_kernel kernel = nullptr; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + if (mode == GGML_SCALE_MODE_NEAREST) { + kernel = backend_ctx->kernel_upscale; + if (kernel == nullptr) { + GGML_LOG_WARN("%s: nearest upscale kernel not available, skipping OpenCL execution.\n", __func__); + return; + } + } else if (mode == GGML_SCALE_MODE_BILINEAR) { + kernel = backend_ctx->kernel_upscale_bilinear; + if (kernel == nullptr) { + GGML_LOG_WARN("%s: bilinear upscale kernel not available, skipping OpenCL execution.\n", __func__); + return; + } + } else { + GGML_LOG_WARN("%s: unsupported upscale mode %d, skipping OpenCL execution.\n", __func__, mode); + return; + } - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - GGML_TENSOR_LOCALS(int, ne, dst, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; - cl_kernel kernel; + cl_ulong off_src0 = extra_src0->offset + src0->view_offs; + cl_ulong off_dst = extra_dst->offset + dst->view_offs; - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_exp_f32_4; - } else { - kernel = backend_ctx->kernel_exp_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_exp_f32; - } else { - kernel = backend_ctx->kernel_exp_f16; - } - } + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_exp_f32_nc; - } else { - kernel = backend_ctx->kernel_exp_f16_nc; + float sf0 = (float)ne0 / ne00; + float sf1 = (float)ne1 / ne01; + float sf2 = (float)ne2 / ne02; + float sf3 = (float)ne3 / ne03; + + float pixel_offset = 0.5f; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb03)); + + if (mode == GGML_SCALE_MODE_NEAREST) { + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &sf0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &sf1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf3)); + } else if (mode == GGML_SCALE_MODE_BILINEAR) { + if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) { + sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0; + sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1; + pixel_offset = 0.0f; } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf0)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf1)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(float), &sf2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(float), &sf3)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float), &pixel_offset)); + } - int nth = 64; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + size_t dst_total_elements = (size_t)ne0 * ne1 * ne2 * ne3; + if (dst_total_elements == 0) { + return; + } + size_t global_work_size[] = { dst_total_elements, 1, 1 }; + size_t local_work_size_pref = 256; + size_t local_work_size[] = { MIN(local_work_size_pref, dst_total_elements), 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + size_t * local_work_size_ptr = local_work_size; + if (dst_total_elements % local_work_size[0] != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_expm1(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - - UNUSED(src1); + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; @@ -15702,3519 +21321,4627 @@ static void ggml_cl_expm1(ggml_backend_t backend, const ggml_tensor * src0, cons const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; + const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; - cl_kernel kernel; - - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_expm1_f32_4; - } else { - kernel = backend_ctx->kernel_expm1_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_expm1_f32; - } else { - kernel = backend_ctx->kernel_expm1_f16; - } - } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + const cl_int dim = ((const int32_t *) dst->op_params)[0]; + GGML_ASSERT(dim >= 0 && dim <= 3); - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } + int nth = MIN(64, ne0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + const size_t ts = ggml_type_size(dst->type); + // the pack kernel copies 4-byte elements, so it is only valid for those. + const bool concat_pack = (dim == 0 && ne0 < 32 && ts == 4); + cl_kernel kernel; + if (concat_pack) { + kernel = backend_ctx->kernel_concat_b4_pack; } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_expm1_f32_nc; - } else { - kernel = backend_ctx->kernel_expm1_f16_nc; + switch (ts) { + case 1: kernel = backend_ctx->kernel_concat_b1; break; + case 2: kernel = backend_ctx->kernel_concat_b2; break; + case 4: kernel = backend_ctx->kernel_concat_b4; break; + case 8: kernel = backend_ctx->kernel_concat_b8; break; + default: GGML_ABORT("unsupported concat element size: %zu", ts); } + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_int), &dim)); - int nth = 64; + if (concat_pack) { + // packed kernel needs the dst dims to unflatten its 1-D row index. + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &ne3)); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + const int maxwg = (int)backend_ctx->get_kernel_workgroup_size(kernel); + const int base = MIN(64, maxwg); + const int tpr = MIN(ne0, base); // threads per row + const int rpw = MAX(1, base / tpr); // rows per workgroup + const int lsz = tpr * rpw; + const int nrows = ne1*ne2*ne3; + const int nwg = (nrows + rpw - 1) / rpw; + size_t global_work_size[] = {(size_t)nwg*lsz, 1, 1}; + size_t local_work_size[] = {(size_t)lsz, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); + } else { + size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } -static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_timestep_embedding(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - - UNUSED(src1); + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + if (backend_ctx->kernel_timestep_embedding == nullptr) { + GGML_LOG_WARN("%s: timestep_embedding kernel not available, skipping OpenCL execution.\n", __func__); + return; + } - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + cl_ulong off_src0 = extra_src0->offset + src0->view_offs; + cl_ulong off_dst = extra_dst->offset + dst->view_offs; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + const int logical_dim = dst->op_params[0]; + const int max_period = dst->op_params[1]; + const int dst_nb1_bytes = dst->nb[1]; - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + cl_kernel kernel = backend_ctx->kernel_timestep_embedding; - cl_kernel kernel; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &dst_nb1_bytes)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &logical_dim)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &max_period)); - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_abs_f32_4; - } else { - kernel = backend_ctx->kernel_abs_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_abs_f32; - } else { - kernel = backend_ctx->kernel_abs_f16; - } + size_t gws0 = (size_t)(((logical_dim + 1) / 2) + 1); + + size_t gws1 = (size_t)src0->ne[0]; + + size_t global_work_size[] = {gws0, gws1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, NULL, dst); +} + +struct ggml_cl_flash_attn_temp_buffer { + cl_mem data = nullptr; + + ~ggml_cl_flash_attn_temp_buffer() { + if (data != nullptr) { + CL_CHECK(clReleaseMemObject(data)); + data = nullptr; } + } +}; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); +// Resolve the source buffer + strides for an FA KV tensor: keep the +// caller-supplied AoS buffer if non-NULL, else fall back to tensor->extra. +static void ggml_cl_flash_attn_resolve_src( + const ggml_tensor * tensor, + cl_mem & buf, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3) { + if (buf != NULL) { + return; + } + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra && extra->data_device); + buf = extra->data_device; + offset = extra->offset + tensor->view_offs; + nb1 = tensor->nb[1]; + nb2 = tensor->nb[2]; + nb3 = tensor->nb[3]; +} - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; +// Read a (possibly strided-view) tensor from device into a tight host buffer. +// dim 0 is always tight; a strided view is gathered row-by-row. +static void ggml_cl_flash_attn_read_tensor_host( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + cl_mem src_buffer, cl_ulong src_offset, + cl_ulong src_nb1, cl_ulong src_nb2, cl_ulong src_nb3, + size_t row_bytes, void * dst, size_t total_bytes +) { + const bool contiguous_layout = + src_nb1 == row_bytes && + src_nb2 == row_bytes * (cl_ulong) tensor->ne[1] && + src_nb3 == src_nb2 * (cl_ulong) tensor->ne[2]; - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; + if (contiguous_layout) { + CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, + src_offset, total_bytes, dst, 0, NULL, NULL)); + return; + } + + size_t dst_off = 0; + for (int64_t i3 = 0; i3 < tensor->ne[3]; ++i3) { + for (int64_t i2 = 0; i2 < tensor->ne[2]; ++i2) { + for (int64_t i1 = 0; i1 < tensor->ne[1]; ++i1) { + const cl_ulong row_src_off = src_offset + + (cl_ulong) i3 * src_nb3 + + (cl_ulong) i2 * src_nb2 + + (cl_ulong) i1 * src_nb1; + CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, + row_src_off, row_bytes, + (uint8_t *) dst + dst_off, 0, NULL, NULL)); + dst_off += row_bytes; + } } + } + GGML_ASSERT(dst_off == total_bytes); +} - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); +// forward decl: used by the FA decode dispatch (K-image variant) below. +static cl_mem ggml_cl_img_pool_get_or_create( + ggml_backend_opencl_context * backend_ctx, + std::map<ggml_backend_opencl_context::ImagePoolKey, + ggml_backend_opencl_context::ImagePoolEntry> & pool, + cl_mem data_device, + cl_ulong offset0, + size_t required_bytes, + cl_channel_type channel_data_type); + +// Rebuild AoS q8_0/q4_0 bytes from a SoA tensor into a temp buffer. +// Returns false if the tensor is not SoA-quantised (already AoS). +static bool ggml_cl_flash_attn_reconstruct_aos( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & out_buf, + cl_ulong & out_offset, + cl_ulong & out_nb1, + cl_ulong & out_nb2, + cl_ulong & out_nb3 +) { + if (tensor == nullptr) { + return false; + } + const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(tensor); + const bool is_q4_0 = tensor->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(tensor); + if (!is_q8_0 && !is_q4_0) { + return false; + } + + // For views, SoA extra is on view_src (view->extra is pre-SoA). + // Noshuffle layout only applies to 2D weights, as determined by `use_adreno_kernels`, + // where ne2 == 1 and ne3 == 1 -- these are never FA inputs. + // Therefore, we use `restore_block_qk_0` kernels, not `restore_block_qk_0_noshuffle`. + const ggml_tensor * soa_src = tensor->view_src ? tensor->view_src : tensor; + cl_mem extra_q = NULL; + cl_mem extra_d = NULL; + if (is_q8_0) { + auto * e = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + GGML_ASSERT(e && e->q && e->d); + extra_q = e->q; + extra_d = e->d; } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_abs_f32_nc; - } else { - kernel = backend_ctx->kernel_abs_f16_nc; - } + auto * e = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + GGML_ASSERT(e && e->q && e->d); + extra_q = e->q; + extra_d = e->d; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + // Reconstruct the whole parent; view offsets then work naturally. + const size_t parent_nbytes = ggml_nbytes(soa_src); + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); + CL_CHECK(err); - int nth = 64; + cl_kernel kernel = is_q8_0 ? backend_ctx->kernel_restore_block_q8_0 + : backend_ctx->kernel_restore_block_q4_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &temp.data)); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + const size_t n_blocks = (size_t) ggml_nelements(soa_src) / ggml_blck_size(soa_src->type); + size_t global_work_size[] = { n_blocks, 1, 1 }; + size_t local_work_size[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, NULL)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } + out_buf = temp.data; + out_offset = tensor->view_offs; + out_nb1 = tensor->nb[1]; + out_nb2 = tensor->nb[2]; + out_nb3 = tensor->nb[3]; + return true; } -// Shared driver for the extended unary ops (unary_ext.cl), same selection as -// ggml_cl_abs: contiguous picks the vec4 kernel when the element count is a -// multiple of 4 (else scalar); non-contiguous uses the stride-addressed kernel. -static void ggml_cl_unary_ext(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst, - cl_kernel k_f32, cl_kernel k_f32_4, cl_kernel k_f32_nc, - cl_kernel k_f16, cl_kernel k_f16_4, cl_kernel k_f16_nc) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; +// GPU dequant of a contiguous q4_0/q8_0 KV tensor to f16/f32. Caller supplies +// src_buf when reconstructing from SoA. Returns false for non-contig layouts +// (the kernel indexes blocks tightly within ne[0]) so the caller can fall back +// to the host path. +static bool ggml_cl_flash_attn_dequant_kv_gpu( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_type target_type, + cl_mem in_src_buf, + cl_ulong in_src_offset, + cl_ulong in_src_nb1, + cl_ulong in_src_nb2, + cl_ulong in_src_nb3, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & out_buf, + cl_ulong & out_offset, + cl_ulong & out_nb1, + cl_ulong & out_nb2, + cl_ulong & out_nb3 +) { + GGML_ASSERT(tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q4_0); + GGML_ASSERT(target_type == GGML_TYPE_F16 || target_type == GGML_TYPE_F32); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0; - const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; - const cl_ulong nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; - const cl_ulong nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + cl_mem src_buf = in_src_buf; + cl_ulong src_offset = in_src_offset; + cl_ulong src_nb1 = in_src_nb1; + cl_ulong src_nb2 = in_src_nb2; + cl_ulong src_nb3 = in_src_nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buf, src_offset, src_nb1, src_nb2, src_nb3); - const bool is_f16 = (src0->type == GGML_TYPE_F16); - cl_kernel kernel; + if (tensor->nb[0] != (cl_ulong) ggml_type_size(tensor->type)) { + return false; + } - if (ggml_is_contiguous(src0)) { - int n = ggml_nelements(dst); - if (n % 4 == 0) { - kernel = is_f16 ? k_f16_4 : k_f32_4; - n /= 4; - } else { - kernel = is_f16 ? k_f16 : k_f32; - } + const size_t n_blocks = (size_t) ggml_nelements(tensor) / 32; // block size is 32 + const size_t elem_size = ggml_type_size(target_type); + const size_t out_bytes = n_blocks * 32 * elem_size; + const cl_int nblk0_arg = (cl_int) (tensor->ne[0] / 32); + const cl_int ne1_arg = (cl_int) tensor->ne[1]; + const cl_int ne2_arg = (cl_int) tensor->ne[2]; + const cl_int ne3_arg = (cl_int) tensor->ne[3]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); + CL_CHECK(err); - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + cl_kernel kernel; + if (target_type == GGML_TYPE_F16) { + kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f16_view_aos + : backend_ctx->kernel_dequant_q4_0_f16_view_aos; } else { - kernel = is_f16 ? k_f16_nc : k_f32_nc; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - - int nth = 64; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f32_view_aos + : backend_ctx->kernel_dequant_q4_0_f32_view_aos; } -} -#define GGML_CL_UNARY_EXT_WRAP(FN, OP) \ -static void FN(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { \ - UNUSED(src1); \ - ggml_backend_opencl_context *c = (ggml_backend_opencl_context *)backend->context; \ - ggml_cl_unary_ext(backend, src0, dst, c->kernel_##OP##_f32, c->kernel_##OP##_f32_4, c->kernel_##OP##_f32_nc, \ - c->kernel_##OP##_f16, c->kernel_##OP##_f16_4, c->kernel_##OP##_f16_nc); \ -} + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &src_offset)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &src_nb1)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &src_nb2)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &src_nb3)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &nblk0_arg)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne1_arg)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne2_arg)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne3_arg)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &temp.data)); -GGML_CL_UNARY_EXT_WRAP(ggml_cl_sgn, sgn) -GGML_CL_UNARY_EXT_WRAP(ggml_cl_step, step) -GGML_CL_UNARY_EXT_WRAP(ggml_cl_elu, elu) -GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardswish, hardswish) -GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardsigmoid, hardsigmoid) -GGML_CL_UNARY_EXT_WRAP(ggml_cl_floor, floor) -GGML_CL_UNARY_EXT_WRAP(ggml_cl_ceil, ceil) -GGML_CL_UNARY_EXT_WRAP(ggml_cl_round, round) -GGML_CL_UNARY_EXT_WRAP(ggml_cl_trunc, trunc) + size_t global_ws[3] = { (size_t) nblk0_arg, (size_t) ne1_arg, (size_t) ne2_arg * (size_t) ne3_arg }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, + global_ws, NULL, 0, NULL, NULL)); -#undef GGML_CL_UNARY_EXT_WRAP + out_buf = temp.data; + out_offset = 0; + out_nb1 = (cl_ulong) tensor->ne[0] * elem_size; + out_nb2 = out_nb1 * (cl_ulong) tensor->ne[1]; + out_nb3 = out_nb2 * (cl_ulong) tensor->ne[2]; + return true; +} -static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); +static bool ggml_cl_flash_attn_prepare_quantized_tensor( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_type target_type, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & data_device, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3 +) { + if (!ggml_is_quantized(tensor->type)) { + return false; + } - UNUSED(src1); + // Caller-supplied AoS buffer wins over tensor->extra when present. + cl_mem src_buffer = data_device; + cl_ulong src_offset = offset; + cl_ulong src_nb1 = nb1; + cl_ulong src_nb2 = nb2; + cl_ulong src_nb3 = nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const int64_t n = ggml_nelements(tensor); + const size_t row_bytes = (size_t) (tensor->ne[0] / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); + // tight-packed byte count (ggml_nbytes includes stride gaps). + const size_t total_bytes = (size_t) (n / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); + std::vector<uint8_t> host_quant(total_bytes); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + sync_with_other_backends(backend_ctx); + ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, + src_nb1, src_nb2, src_nb3, + row_bytes, host_quant.data(), total_bytes); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + std::vector<float> host_f32(n); + ggml_get_type_traits(tensor->type)->to_float(host_quant.data(), host_f32.data(), n); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + const size_t bytes_per_elem = ggml_type_size(target_type); + const size_t buffer_size = (size_t) n * bytes_per_elem; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + std::vector<uint8_t> host_linear(buffer_size); + if (target_type == GGML_TYPE_F32) { + memcpy(host_linear.data(), host_f32.data(), buffer_size); + } else { + GGML_ASSERT(target_type == GGML_TYPE_F16); + ggml_fp32_to_fp16_row(host_f32.data(), (ggml_fp16_t *) host_linear.data(), n); + } - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, buffer_size, NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, buffer_size, host_linear.data(), 0, NULL, NULL)); - cl_kernel kernel; + data_device = temp.data; + offset = 0; + nb1 = (cl_ulong) (tensor->ne[0] * bytes_per_elem); + nb2 = (cl_ulong) (tensor->ne[1] * nb1); + nb3 = (cl_ulong) (tensor->ne[2] * nb2); - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_softplus_f32_4; - } else { - kernel = backend_ctx->kernel_softplus_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_softplus_f32; - } else { - kernel = backend_ctx->kernel_softplus_f16; - } - } + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: OpenCL flash attention dequantizes GPU-resident quantized KV cache into temporary linear buffers; performance may be poor\n"); + warned = true; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + return true; +} - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; +// Host-side F16 -> F32 for the asymmetric-KV F32 fallback path. +static bool ggml_cl_flash_attn_convert_f16_to_f32( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & data_device, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3 +) { + if (tensor->type != GGML_TYPE_F16) { + return false; + } - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } + cl_mem src_buffer = data_device; + cl_ulong src_offset = offset; + cl_ulong src_nb1 = nb1; + cl_ulong src_nb2 = nb2; + cl_ulong src_nb3 = nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); - } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_softplus_f32_nc; - } else { - kernel = backend_ctx->kernel_softplus_f16_nc; - } + const int64_t n = ggml_nelements(tensor); + const size_t row_bytes = (size_t) tensor->ne[0] * sizeof(ggml_fp16_t); + const size_t total_bytes = (size_t) n * sizeof(ggml_fp16_t); + std::vector<uint8_t> host_f16(total_bytes); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + sync_with_other_backends(backend_ctx); + ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, + src_nb1, src_nb2, src_nb3, + row_bytes, host_f16.data(), total_bytes); - int nth = 64; + std::vector<float> host_f32(n); + ggml_fp16_to_fp32_row((const ggml_fp16_t *) host_f16.data(), host_f32.data(), n); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + const size_t f32_bytes = (size_t) n * sizeof(float); + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, f32_bytes, NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, + f32_bytes, host_f32.data(), 0, NULL, NULL)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + data_device = temp.data; + offset = 0; + nb1 = (cl_ulong) (tensor->ne[0] * sizeof(float)); + nb2 = (cl_ulong) (tensor->ne[1] * nb1); + nb3 = (cl_ulong) (tensor->ne[2] * nb2); + + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: OpenCL flash attention asymmetric KV converts an F16 cache to F32 host-side; performance may be poor\n"); + warned = true; } + + return true; } -static void ggml_cl_repeat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1_shape_def, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_ASSERT(dst->type == src0->type); +// Flash-Decoding (K-split) dispatch thresholds. FD fires for non-causal +// attention with n_kv >= FD_MIN_N_KV and d_head <= FD_MAX_DK; the KV range is +// split into ~n_kv/FD_KV_PER_SPLIT partials, clamped to [FD_MIN_SPLITS, +// FD_MAX_SPLITS]. Multi-query FD is restricted to small heads +// (d_head <= FD_MAX_DK_MULTI) and capped at FD_MAX_N_Q_MULTI queries. +static constexpr int FD_MIN_N_KV = 2048; +static constexpr int FD_KV_PER_SPLIT = 2048; +// f16 KV decode wants more splits than the 2048 default; quantized KV keeps 2048. +static constexpr int FD_KV_PER_SPLIT_F16 = 512; +static constexpr int FD_MIN_SPLITS = 2; +static constexpr int FD_MAX_SPLITS = 16; +static constexpr int FD_MAX_DK = 128; +static constexpr int FD_MAX_DK_MULTI = 64; +static constexpr int FD_MAX_N_Q_MULTI = 8; +// MQ FD split-groups have few subgroups (MQ_NSG_SPLIT), so use a smaller +// kv_per_split to keep the softmax recurrence short; non-MQ keeps FD_KV_PER_SPLIT. +static constexpr int FD_MQ_KV_PER_SPLIT = 256; +static constexpr int FD_MQ_MAX_SPLITS = 128; - UNUSED(src1_shape_def); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +struct ggml_cl_adreno_xmem_attn_schedule { + int qk_lws0 = 256; + int qk_lws2 = 1; + int softmax_reduce_lws0 = 256; + int softmax_apply_lws0 = 64; + int softmax_apply_lws2 = 4; + int pv_lws0 = 64; + int pv_lws2 = 4; +}; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; +static inline size_t ggml_cl_round_up(size_t x, size_t a) { + return ((x + a - 1) / a) * a; +} - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; +static inline int ggml_cl_round_up_div(int x, int y) { + return (x + y - 1) / y; +} - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; +static inline void ggml_cl_set_arg_int4(cl_kernel kernel, cl_uint index, int x, int y, int z, int w) { + struct { int x, y, z, w; } value { x, y, z, w }; + CL_CHECK(clSetKernelArg(kernel, index, sizeof(value), &value)); +} - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; +static cl_mem ggml_cl_make_image2d_half4(cl_context context, cl_mem_flags flags, size_t width, size_t height) { + cl_int err = CL_SUCCESS; + cl_image_format format = { CL_RGBA, CL_HALF_FLOAT }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE2D; + desc.image_width = width; + desc.image_height = height; + cl_mem image = clCreateImage(context, flags, &format, &desc, nullptr, &err); + CL_CHECK(err); + return image; +} - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; +static cl_mem ggml_cl_make_image1d_buffer_half4(cl_context context, cl_mem_flags flags, size_t width, cl_mem backing_buffer) { + cl_int err = CL_SUCCESS; + cl_image_format format = { CL_RGBA, CL_HALF_FLOAT }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = width; + desc.buffer = backing_buffer; + cl_mem image = clCreateImage(context, flags, &format, &desc, nullptr, &err); + CL_CHECK(err); + return image; +} - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; +static void ggml_cl_release_mem(cl_mem & mem) { + if (mem != nullptr) { + CL_CHECK(clReleaseMemObject(mem)); + mem = nullptr; + } +} - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; +static void ggml_cl_adreno_xmem_attn_release_scratch(ggml_backend_opencl_context * backend_ctx) { + auto & s = backend_ctx->adreno_xmem_attn.scratch; + ggml_cl_release_mem(s.q_img); + ggml_cl_release_mem(s.k_img); + ggml_cl_release_mem(s.v_img); + ggml_cl_release_mem(s.out_img); + ggml_cl_release_mem(s.k_transpose_img1d); + ggml_cl_release_mem(s.k_transpose_buf); + ggml_cl_release_mem(s.k_packed_buf); + ggml_cl_release_mem(s.v_packed_buf); + ggml_cl_release_mem(s.score_img1d); + ggml_cl_release_mem(s.prob_img1d); + ggml_cl_release_mem(s.score_buf); + ggml_cl_release_mem(s.prob_buf); + ggml_cl_release_mem(s.softmax_stats_img2d); + ggml_cl_release_mem(s.xmem_qk); + ggml_cl_release_mem(s.xmem_pv); + s = {}; +} - cl_kernel kernel = backend_ctx->kernel_repeat_f32; +static ggml_cl_adreno_xmem_attn_schedule ggml_cl_adreno_xmem_attn_select_schedule( + const ggml_backend_opencl_context * backend_ctx, + int n_q, + int n_kv, + int heads_total, + int q_width, + int gqa_ratio) { + const bool big_h = heads_total >= 8; + ggml_cl_adreno_xmem_attn_schedule sched; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); + if (gqa_ratio == 1) { + if (n_q >= 512) { sched.qk_lws0 = 512; } + else if (n_q >= 256) { sched.qk_lws0 = 128; } + else { sched.qk_lws0 = 64; } + sched.qk_lws2 = (big_h && n_q >= 512) ? 2 : 1; + } else { + if (q_width >= 2048) { sched.qk_lws0 = 512; } + else if (q_width >= 256) { sched.qk_lws0 = 128; } + else { sched.qk_lws0 = 64; } + sched.qk_lws2 = MIN(8, (int) backend_ctx->max_workgroup_size / sched.qk_lws0); + } - int nth = 64; + if (n_kv >= 2048) { sched.softmax_reduce_lws0 = 1024; } + else if (n_kv >= 512) { sched.softmax_reduce_lws0 = big_h ? 256 : 512; } + else { sched.softmax_reduce_lws0 = 256; } - size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + if (n_kv < 256) { sched.softmax_apply_lws0 = 64; } + else { sched.softmax_apply_lws0 = big_h ? 128 : 64; } + sched.softmax_apply_lws2 = n_kv >= 512 ? 8 : 4; + + if (n_q < 256) { sched.pv_lws0 = 64; } + else { sched.pv_lws0 = big_h ? 128 : 64; } + sched.pv_lws2 = big_h ? 8 : (n_q <= 256 ? 8 : 4); + + const int max_wg = (int) backend_ctx->max_workgroup_size; + auto fix = [&](int & l0, int & l2) { + while (l0 * l2 > max_wg) { + if (l2 > 1) { l2 /= 2; } + else if (l0 > 32) { l0 /= 2; } + else { break; } + } + }; + fix(sched.qk_lws0, sched.qk_lws2); + fix(sched.softmax_apply_lws0, sched.softmax_apply_lws2); + fix(sched.pv_lws0, sched.pv_lws2); + while (sched.softmax_reduce_lws0 > max_wg) { + sched.softmax_reduce_lws0 /= 2; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return sched; } -static void ggml_cl_pad(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); +static bool ggml_cl_adreno_xmem_attn_prepare( + ggml_backend_opencl_context * backend_ctx, + int n_q, + int n_kv, + int d_head_q, + int d_head_v, + int n_head, + int n_head_kv, + int n_batch) { + auto & s = backend_ctx->adreno_xmem_attn.scratch; + const int gqa_ratio = n_head / n_head_kv; + const int q_width = n_q * gqa_ratio; + const int kv_heads_total = n_head_kv * n_batch; + const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); + if (s.q_img != nullptr && + s.n_q == n_q && + s.n_kv == n_kv && + s.n_kv_padded == n_kv_padded && + s.d_head_q == d_head_q && + s.d_head_v == d_head_v && + s.q_width == q_width && + s.kv_heads_total == kv_heads_total) { + return true; + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_cl_adreno_xmem_attn_release_scratch(backend_ctx); - if (backend_ctx->kernel_pad == nullptr) { - GGML_LOG_WARN("%s: pad kernel not available, skipping OpenCL execution.\n", __func__); - return; - } + const int qpack = d_head_q / 4; + const int vpack = d_head_v / 4; + const int npack = n_kv_padded / 4; + const size_t q_img_h = (size_t) kv_heads_total * qpack; + const size_t v_img_h = (size_t) kv_heads_total * vpack; - ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; + s.q_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, q_img_h); + s.k_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) n_kv_padded, q_img_h); + s.v_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) n_kv_padded, v_img_h); + s.out_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, v_img_h); - cl_ulong off_src0 = extra_src0->offset + src0->view_offs; - cl_ulong off_dst = extra_dst->offset + dst->view_offs; + const size_t k_transpose_half4_elems = (size_t) npack * kv_heads_total * d_head_q; + s.k_transpose_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_transpose_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); + GGML_ASSERT(s.k_transpose_buf != nullptr); + s.k_transpose_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, k_transpose_half4_elems, s.k_transpose_buf); - const int s_ne0 = src0->ne[0]; - const int s_ne1 = src0->ne[1]; - const int s_ne2 = src0->ne[2]; - const int s_ne3 = src0->ne[3]; + const size_t k_groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_q, 16); + const size_t v_groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_v, 16); + const size_t k_packed_half4_elems = (size_t) n_kv_padded * k_groups16 * 4; + const size_t v_packed_half4_elems = (size_t) n_kv_padded * v_groups16 * 4; + s.k_packed_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_packed_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); + s.v_packed_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_packed_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); + GGML_ASSERT(s.k_packed_buf != nullptr && s.v_packed_buf != nullptr); - const int s_nb0 = src0->nb[0]; - const int s_nb1 = src0->nb[1]; - const int s_nb2 = src0->nb[2]; - const int s_nb3 = src0->nb[3]; + const size_t score_half4_elems = (size_t) npack * kv_heads_total * q_width; + const size_t score_bytes = score_half4_elems * sizeof(uint16_t) * 4; + s.score_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, score_bytes, nullptr, nullptr); + s.prob_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, score_bytes, nullptr, nullptr); + GGML_ASSERT(s.score_buf != nullptr && s.prob_buf != nullptr); + s.score_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, score_half4_elems, s.score_buf); + s.prob_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, score_half4_elems, s.prob_buf); + s.softmax_stats_img2d = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, + (size_t) q_width, (size_t) kv_heads_total); + s.xmem_qk = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY, 6144, nullptr, nullptr); + s.xmem_pv = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY, 6144, nullptr, nullptr); + GGML_ASSERT(s.softmax_stats_img2d != nullptr && s.xmem_qk != nullptr && s.xmem_pv != nullptr); - const int d_ne0 = dst->ne[0]; - const int d_ne1 = dst->ne[1]; - const int d_ne2 = dst->ne[2]; - const int d_ne3 = dst->ne[3]; + s.n_q = n_q; + s.n_kv = n_kv; + s.n_kv_padded = n_kv_padded; + s.d_head_q = d_head_q; + s.d_head_v = d_head_v; + s.q_width = q_width; + s.kv_heads_total = kv_heads_total; + return true; +} - const int d_nb0 = dst->nb[0]; - const int d_nb1 = dst->nb[1]; - const int d_nb2 = dst->nb[2]; - const int d_nb3 = dst->nb[3]; +static bool ggml_cl_adreno_xmem_attn_can_use( + const ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * q, + const ggml_tensor * k, + const ggml_tensor * dst) { + static const char * xmem_sdpa_env = getenv("GGML_OPENCL_XMEM_SDPA"); + if (xmem_sdpa_env == nullptr || xmem_sdpa_env[0] == '0') { + return false; + } - const int lp0 = ((const int*)(dst->op_params))[0]; - const int rp0 = ((const int*)(dst->op_params))[1]; - const int lp1 = ((const int*)(dst->op_params))[2]; - const int rp1 = ((const int*)(dst->op_params))[3]; - const int lp2 = ((const int*)(dst->op_params))[4]; - const int rp2 = ((const int*)(dst->op_params))[5]; - const int lp3 = ((const int*)(dst->op_params))[6]; - const int rp3 = ((const int*)(dst->op_params))[7]; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; - cl_kernel kernel = backend_ctx->kernel_pad; + if (!backend_ctx->adreno_xmem_attn.compiled || backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { + return false; + } + if (q->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || + (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_F32) || + (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32)) { + return false; + } + if (sinks != nullptr) { + return false; + } + if (q->nb[0] != ggml_type_size(q->type) || k->nb[0] != ggml_type_size(k->type) || + v->nb[0] != ggml_type_size(v->type) || dst->nb[0] != ggml_type_size(dst->type)) { + return false; + } + if (mask != nullptr && (mask->type != GGML_TYPE_F16 || mask->nb[0] != sizeof(ggml_fp16_t))) { + return false; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &s_ne0)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &s_ne1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &s_ne2)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &s_ne3)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &s_nb0)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &s_nb1)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &s_nb2)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &s_nb3)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &d_ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &d_ne1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &d_ne2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &d_ne3)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &d_nb0)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &d_nb1)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &d_nb2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &d_nb3)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &lp0)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &rp0)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &lp1)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &rp1)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &lp2)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &rp2)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &lp3)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(int), &rp3)); + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; - size_t lws0 = 64; - size_t gws0 = (( (size_t)d_ne0 + lws0 - 1 ) / lws0) * lws0; + if (n_q <= 1 || n_kv <= 0 || n_kv > 8192) { + return false; + } + if (d_head_q != k->ne[0] || d_head_v != v->ne[0] || k->ne[1] != v->ne[1] || k->ne[3] != v->ne[3]) { + return false; + } + if (q->ne[3] != k->ne[3]) { + return false; + } + if (n_head_kv <= 0 || n_head % n_head_kv != 0 || k->ne[2] != v->ne[2]) { + return false; + } + if (dst->ne[0] != d_head_v || dst->ne[1] != n_head || dst->ne[2] != n_q || dst->ne[3] != n_batch) { + return false; + } + if ((d_head_q % 8) != 0 || (d_head_v % 32) != 0) { + return false; + } + if (mask != nullptr && + (mask->ne[0] < n_kv || mask->ne[1] < n_q || mask->ne[2] <= 0 || mask->ne[3] <= 0)) { + return false; + } - size_t global_work_size[] = { gws0, (size_t)d_ne1, (size_t)d_ne2*d_ne3 }; - size_t local_work_size[] = { lws0, 1, 1 }; + float params[3]; + memcpy(params, dst->op_params, sizeof(params)); + if (params[1] != 0.0f || params[2] != 0.0f) { + return false; + } - size_t * local_work_size_ptr = local_work_size; - if (d_ne0 % lws0 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; + const int gqa_ratio = n_head / n_head_kv; + const int q_width = n_q * gqa_ratio; + const int kv_heads_total = n_head_kv * n_batch; + const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); + const int qpack = d_head_q / 4; + const int vpack = d_head_v / 4; + const int npack = n_kv_padded / 4; + + if ((size_t) q_width > backend_ctx->image2d_max_width || + (size_t) n_kv_padded > backend_ctx->image2d_max_width) { + return false; + } + if ((size_t) kv_heads_total * (size_t) qpack > backend_ctx->image2d_max_height || + (size_t) kv_heads_total * (size_t) vpack > backend_ctx->image2d_max_height) { + return false; + } + if ((size_t) npack * (size_t) kv_heads_total * (size_t) d_head_q > backend_ctx->image_max_buffer_size || + (size_t) npack * (size_t) kv_heads_total * (size_t) q_width > backend_ctx->image_max_buffer_size) { + return false; } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + return true; } -static void ggml_cl_upscale(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); +static void ggml_cl_adreno_xmem_attn_run( + ggml_backend_t backend, + const ggml_tensor * q, + const ggml_tensor * k, + ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + auto & xstate = backend_ctx->adreno_xmem_attn; + auto & s = xstate.scratch; + if (!xstate.logged) { + GGML_LOG_INFO("ggml_opencl: using Adreno xmem attention path\n"); + xstate.logged = true; + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; - const int mode_flags = (ggml_scale_mode) ggml_get_op_params_i32(dst, 0); - const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF); - cl_kernel kernel = nullptr; + ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *) q->extra; + ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *) k->extra; + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *) v->extra; + ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *) dst->extra; + ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *) mask->extra : nullptr; - if (mode == GGML_SCALE_MODE_NEAREST) { - kernel = backend_ctx->kernel_upscale; - if (kernel == nullptr) { - GGML_LOG_WARN("%s: nearest upscale kernel not available, skipping OpenCL execution.\n", __func__); - return; - } - } else if (mode == GGML_SCALE_MODE_BILINEAR) { - kernel = backend_ctx->kernel_upscale_bilinear; - if (kernel == nullptr) { - GGML_LOG_WARN("%s: bilinear upscale kernel not available, skipping OpenCL execution.\n", __func__); - return; - } - } else { - GGML_LOG_WARN("%s: unsupported upscale mode %d, skipping OpenCL execution.\n", __func__, mode); - return; - } + const cl_ulong offset_q = extra_q->offset + q->view_offs; + const cl_ulong offset_k = extra_k->offset + k->view_offs; + const cl_ulong offset_v = extra_v->offset + v->view_offs; + const cl_ulong offset_o = extra_o->offset + dst->view_offs; + const cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; - ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; + const int heads_total = n_head * n_batch; + const int gqa_ratio = n_head / n_head_kv; + const int q_width = n_q * gqa_ratio; + const int kv_heads_total = n_head_kv * n_batch; + const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); + const int qpack = d_head_q / 4; + const int opack = d_head_v / 4; + const int npack = n_kv_padded / 4; + const float scale = ((const float *) dst->op_params)[0]; - cl_ulong off_src0 = extra_src0->offset + src0->view_offs; - cl_ulong off_dst = extra_dst->offset + dst->view_offs; + GGML_ASSERT(ggml_cl_adreno_xmem_attn_prepare( + backend_ctx, n_q, n_kv, d_head_q, d_head_v, n_head, n_head_kv, n_batch)); + const ggml_cl_adreno_xmem_attn_schedule sched = + ggml_cl_adreno_xmem_attn_select_schedule( + backend_ctx, n_q, n_kv_padded, heads_total, q_width, gqa_ratio); - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + { + size_t gws[3] = {ggml_cl_round_up((size_t) n_q, 8), (size_t) heads_total, (size_t) qpack}; + size_t lws[3] = {8, 1, (size_t) ((qpack <= 32) ? qpack : 1)}; + cl_kernel kernel = xstate.kernel_q_f32_to_img_scaled; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.q_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &d_head_q)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &q->nb[1])); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &q->nb[2])); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &q->nb[3])); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + { + size_t gws[3] = {(size_t) n_kv_padded, (size_t) kv_heads_total, (size_t) qpack}; + size_t lws[3] = {8, 1, (size_t) ((qpack <= 32) ? qpack : 1)}; + cl_kernel kernel = k->type == GGML_TYPE_F16 ? + xstate.kernel_kv_f16_to_img_gqa : xstate.kernel_kv_f32_to_img_gqa; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_k->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_k)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.k_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_q)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_kv_padded)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &k->nb[1])); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &k->nb[2])); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &k->nb[3])); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; + { + size_t gws[3] = {(size_t) n_kv_padded, (size_t) kv_heads_total, (size_t) opack}; + size_t lws[3] = {8, 1, (size_t) ((opack <= 32) ? opack : 1)}; + cl_kernel kernel = v->type == GGML_TYPE_F16 ? + xstate.kernel_kv_f16_to_img_gqa : xstate.kernel_kv_f32_to_img_gqa; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.v_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_v)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_kv_padded)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &v->nb[1])); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &v->nb[2])); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &v->nb[3])); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } - float sf0 = (float)ne0 / ne00; - float sf1 = (float)ne1 / ne01; - float sf2 = (float)ne2 / ne02; - float sf3 = (float)ne3 / ne03; + { + size_t gws[3] = {(size_t) d_head_q, (size_t) kv_heads_total, (size_t) npack}; + size_t lws[3] = {(size_t) MIN(64, d_head_q), (size_t) (kv_heads_total >= 2 ? 2 : 1), (size_t) MIN(8, npack)}; + if (lws[0] * lws[1] * lws[2] > backend_ctx->max_workgroup_size) { + lws[1] = 1; + } + cl_kernel kernel = xstate.kernel_k_gather; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.k_transpose_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_img)); + ggml_cl_set_arg_int4(kernel, 2, n_kv_padded, kv_heads_total, npack, d_head_q); + ggml_cl_set_arg_int4(kernel, 3, qpack, 0, 0, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + { + const size_t groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_q, 16); + const size_t packed_linear = (size_t) n_kv_padded * groups16; + const size_t lws0 = MIN((size_t) 1024, backend_ctx->max_workgroup_size); + size_t gws[3] = {ggml_cl_round_up(packed_linear, lws0), 1, 1}; + size_t lws[3] = {lws0, 1, 1}; + cl_kernel kernel = xstate.kernel_pack_k; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.k_packed_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_transpose_img1d)); + ggml_cl_set_arg_int4(kernel, 2, 8, (int) packed_linear, qpack, d_head_q); + ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, kv_heads_total, kv_heads_total, npack); + ggml_cl_set_arg_int4(kernel, 4, d_head_q, 0, 0, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } - float pixel_offset = 0.5f; + { + size_t lws[3] = {(size_t) sched.qk_lws0, 1, (size_t) sched.qk_lws2}; + const int slices_per_group = sched.qk_lws2 * 8; + const size_t groups_z = (size_t) ggml_cl_round_up_div(npack, slices_per_group); + const size_t groups_x = (size_t) ggml_cl_round_up_div(q_width, sched.qk_lws0); + size_t gws[3] = { + lws[0] * groups_z, + groups_x, + (size_t) kv_heads_total * lws[2], + }; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb03)); + cl_kernel kernel = xstate.kernel_qk_gemm; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.score_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_packed_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.xmem_qk)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &s.q_img)); + ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, npack, q_width, 32); + ggml_cl_set_arg_int4(kernel, 5, qpack, 0, 0, kv_heads_total); + ggml_cl_set_arg_int4(kernel, 6, qpack, 1, 1, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + cl_mem softmax_input_img = s.score_img1d; + cl_mem softmax_output_buf = s.prob_buf; + cl_mem pv_prob_img = s.prob_img1d; - if (mode == GGML_SCALE_MODE_NEAREST) { - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &sf0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &sf1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf3)); - } else if (mode == GGML_SCALE_MODE_BILINEAR) { - if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) { - sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0; - sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1; - pixel_offset = 0.0f; - } + if (mask != nullptr) { + const cl_ulong mask_nb1 = mask->nb[1]; + const cl_ulong mask_nb2 = mask->nb[2]; + const cl_ulong mask_nb3 = mask->nb[3]; + const int mask_ne2 = mask->ne[2]; + const int mask_ne3 = mask->ne[3]; + size_t lws[3] = {(size_t) sched.softmax_apply_lws0, 1, (size_t) sched.softmax_apply_lws2}; + size_t gws[3] = { + ggml_cl_round_up((size_t) q_width, lws[0]), + (size_t) kv_heads_total, + ggml_cl_round_up((size_t) npack, lws[2]), + }; + cl_kernel kernel = xstate.kernel_mask_scores; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.prob_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.score_img1d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_mask->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &q_width)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_kv_padded)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &kv_heads_total)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &mask_ne3)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf0)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf1)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(float), &sf2)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(float), &sf3)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float), &pixel_offset)); + softmax_input_img = s.prob_img1d; + softmax_output_buf = s.score_buf; + pv_prob_img = s.score_img1d; + } + + { + size_t lws[3] = {(size_t) sched.softmax_reduce_lws0, 1, 1}; + size_t gws[3] = {ggml_cl_round_up((size_t) q_width, lws[0]), (size_t) kv_heads_total, 1}; + cl_kernel kernel = xstate.kernel_softmax_reduce_basic; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &softmax_input_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.softmax_stats_img2d)); + ggml_cl_set_arg_int4(kernel, 2, kv_heads_total, 1, q_width, n_kv); + ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, q_width, 0, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + { + size_t lws[3] = {(size_t) sched.softmax_apply_lws0, 1, (size_t) sched.softmax_apply_lws2}; + size_t gws[3] = { + ggml_cl_round_up((size_t) q_width, lws[0]), + (size_t) kv_heads_total, + ggml_cl_round_up((size_t) npack, lws[2]), + }; + cl_kernel kernel = xstate.kernel_softmax_apply_basic; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &softmax_output_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &softmax_input_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.softmax_stats_img2d)); + ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, npack, q_width, 1); + ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, q_width, n_kv, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + { + const size_t groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_v, 16); + const size_t packed_linear = (size_t) n_kv_padded * groups16; + const size_t lws0 = MIN((size_t) 1024, backend_ctx->max_workgroup_size); + size_t gws[3] = {ggml_cl_round_up(packed_linear, lws0), 1, 1}; + size_t lws[3] = {lws0, 1, 1}; + cl_kernel kernel = xstate.kernel_pack_v; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.v_packed_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.v_img)); + ggml_cl_set_arg_int4(kernel, 2, 8, (int) packed_linear, npack, n_kv_padded); + ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, kv_heads_total, opack, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } + { + size_t lws[3] = {(size_t) sched.pv_lws0, 1, (size_t) sched.pv_lws2}; + const int blocks = ggml_cl_round_up_div(opack, 8); + const size_t groups_z = (size_t) ggml_cl_round_up_div(blocks, sched.pv_lws2); + const size_t groups_x = (size_t) ggml_cl_round_up_div(q_width, sched.pv_lws0); + size_t gws[3] = { + lws[0] * groups_z, + groups_x, + (size_t) kv_heads_total * lws[2], + }; - size_t dst_total_elements = (size_t)ne0 * ne1 * ne2 * ne3; - if (dst_total_elements == 0) { - return; + cl_kernel kernel = xstate.kernel_pv_gemm; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.v_packed_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.xmem_pv)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &pv_prob_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &s.out_img)); + ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, opack, q_width, 32); + ggml_cl_set_arg_int4(kernel, 5, npack, 0, 0, kv_heads_total); + ggml_cl_set_arg_int4(kernel, 6, kv_heads_total * q_width, npack, q_width, 1); + ggml_cl_set_arg_int4(kernel, 7, 1, 0, 0, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } - size_t global_work_size[] = { dst_total_elements, 1, 1 }; - size_t local_work_size_pref = 256; - size_t local_work_size[] = { MIN(local_work_size_pref, dst_total_elements), 1, 1}; - size_t * local_work_size_ptr = local_work_size; - if (dst_total_elements % local_work_size[0] != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; + { + size_t gws[3] = {ggml_cl_round_up((size_t) n_q, 8), (size_t) heads_total, (size_t) opack}; + size_t lws[3] = {8, 1, (size_t) ((opack <= 32) ? opack : 1)}; + cl_kernel kernel = xstate.kernel_img_to_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_o->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_o)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.out_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_v)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &dst->nb[1])); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &dst->nb[2])); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &dst->nb[3])); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + GGML_ASSERT(q->extra); + GGML_ASSERT(k->extra); + GGML_ASSERT(v->extra); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == src1->type); - GGML_ASSERT(src0->type == dst->type); + if (mask) { + GGML_ASSERT(mask->extra); + } + if (sinks) { + GGML_ASSERT(sinks->extra); + } ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_context context = backend_ctx->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + const std::pair<int, int> dk_dv = {d_head_q, d_head_v}; + cl_kernel kernel = backend_ctx->fa.kernel_flash_attn_f32_f16_bin; + GGML_ASSERT(kernel != NULL); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; + ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *)k->extra; + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; + ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; + ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + cl_ulong offset_q = extra_q->offset + q->view_offs; + cl_ulong offset_o = extra_o->offset + dst->view_offs; - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; + cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; + cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; + cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; + const cl_ulong q_nb1 = q->nb[1]; + const cl_ulong q_nb2 = q->nb[2]; + const cl_ulong q_nb3 = q->nb[3]; - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + cl_mem k_data_device = extra_k->data_device; + cl_ulong offset_k = extra_k->offset + k->view_offs; + cl_ulong k_nb1 = k->nb[1]; + cl_ulong k_nb2 = k->nb[2]; + cl_ulong k_nb3 = k->nb[3]; - const cl_int dim = ((const int32_t *) dst->op_params)[0]; - GGML_ASSERT(dim >= 0 && dim <= 3); + cl_mem v_data_device = extra_v->data_device; + cl_ulong offset_v = extra_v->offset + v->view_offs; + cl_ulong v_nb1 = v->nb[1]; + cl_ulong v_nb2 = v->nb[2]; + cl_ulong v_nb3 = v->nb[3]; - int nth = MIN(64, ne0); + const cl_ulong o_nb1 = dst->nb[1]; + const cl_ulong o_nb2 = dst->nb[2]; + const cl_ulong o_nb3 = dst->nb[3]; - const size_t ts = ggml_type_size(dst->type); - // the pack kernel copies 4-byte elements, so it is only valid for those. - const bool concat_pack = (dim == 0 && ne0 < 32 && ts == 4); - cl_kernel kernel; - if (concat_pack) { - kernel = backend_ctx->kernel_concat_b4_pack; - } else { - switch (ts) { - case 1: kernel = backend_ctx->kernel_concat_b1; break; - case 2: kernel = backend_ctx->kernel_concat_b2; break; - case 4: kernel = backend_ctx->kernel_concat_b4; break; - case 8: kernel = backend_ctx->kernel_concat_b8; break; - default: GGML_ABORT("unsupported concat element size: %zu", ts); - } - } + const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; + const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; + const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; + const int mask_ne2 = mask ? mask->ne[2] : 0; + const int mask_ne3 = mask ? mask->ne[3] : 0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_int), &dim)); + float * params = (float *)dst->op_params; + float scale = params[0]; + float max_bias = params[1]; + float logit_softcap = params[2]; - if (concat_pack) { - // packed kernel needs the dst dims to unflatten its 1-D row index. - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &ne3)); + const int is_causal = (mask == NULL && n_q > 1 && n_q == n_kv); // redundant n_q > 1 check ? - const int maxwg = (int)backend_ctx->get_kernel_workgroup_size(kernel); - const int base = MIN(64, maxwg); - const int tpr = MIN(ne0, base); // threads per row - const int rpw = MAX(1, base / tpr); // rows per workgroup - const int lsz = tpr * rpw; - const int nrows = ne1*ne2*ne3; - const int nwg = (nrows + rpw - 1) / rpw; - size_t global_work_size[] = {(size_t)nwg*lsz, 1, 1}; - size_t local_work_size[] = {(size_t)lsz, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); - } else { - size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; + const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; + const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } -} + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + + ggml_cl_flash_attn_temp_buffer temp_k; + ggml_cl_flash_attn_temp_buffer temp_v; + ggml_cl_flash_attn_temp_buffer temp_k_aos; + ggml_cl_flash_attn_temp_buffer temp_v_aos; + + if (is_q8_0) { + ggml_cl_flash_attn_reconstruct_aos( + backend_ctx, k, temp_k_aos, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + + ggml_cl_flash_attn_reconstruct_aos( + backend_ctx, v, temp_v_aos, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); -static void ggml_cl_timestep_embedding(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + bool k_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, k, GGML_TYPE_F16, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, + temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + bool v_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, v, GGML_TYPE_F16, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, + temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); - if (backend_ctx->kernel_timestep_embedding == nullptr) { - GGML_LOG_WARN("%s: timestep_embedding kernel not available, skipping OpenCL execution.\n", __func__); - return; + GGML_ASSERT(k_done && v_done); } - ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; - - cl_ulong off_src0 = extra_src0->offset + src0->view_offs; - cl_ulong off_dst = extra_dst->offset + dst->view_offs; + // Allocate input/output memory buffers + cl_mem mem_matrixQ; + cl_mem mem_matrixK; + cl_mem mem_matrixV; + cl_mem mem_matrixO; + cl_buffer_region region; + cl_int err; - const int logical_dim = dst->op_params[0]; - const int max_period = dst->op_params[1]; - const int dst_nb1_bytes = dst->nb[1]; + region.origin = offset_q; + region.size = ggml_nbytes(q); + mem_matrixQ = clCreateSubBuffer(extra_q->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - cl_kernel kernel = backend_ctx->kernel_timestep_embedding; + region.origin = offset_k; + region.size = is_q8_0 ? (size_t) k_nb3 * (size_t) k->ne[3] : ggml_nbytes(k); + mem_matrixK = clCreateSubBuffer(k_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &dst_nb1_bytes)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &logical_dim)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &max_period)); + region.origin = offset_v; + region.size = is_q8_0 ? (size_t) v_nb3 * (size_t) v->ne[3] : ggml_nbytes(v); + mem_matrixV = clCreateSubBuffer(v_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - size_t gws0 = (size_t)(((logical_dim + 1) / 2) + 1); + region.origin = offset_o; + region.size = ggml_nbytes(dst); + mem_matrixO = clCreateSubBuffer(extra_o->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - size_t gws1 = (size_t)src0->ne[0]; + cl_image_format img_fmt_1d = { CL_RGBA, CL_FLOAT}; + cl_image_desc img_desc_1d; - size_t global_work_size[] = {gws0, gws1, 1}; + // use image 1d buffer used as fallback when on mask is applied + cl_mem mem_tex_mask_fallback_1dbuf; + img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_1d.image_width = 1; + img_desc_1d.buffer = mem_matrixK; + mem_tex_mask_fallback_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); + CL_CHECK(err); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, NULL, dst); -} + cl_mem mem_tex_matrixO_1dbuf; + img_fmt_1d = { CL_RGBA, CL_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_1d.image_width = ggml_nbytes(dst) / 4 / 4; + img_desc_1d.buffer = mem_matrixO; + mem_tex_matrixO_1dbuf = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); + CL_CHECK(err); -struct ggml_cl_flash_attn_temp_buffer { - cl_mem data = nullptr; + // The bin kernel requires 2d (or 3d) buffers packed for data loading/multiplication. + // These repack kernels launch across all buffers to ensure compatibility + cl_mem mem_tex_matrixMask_1dbuf = NULL; + cl_mem mem_matrixMask = NULL; + cl_mem mem_matrixMask_padded = NULL; + cl_ulong mask_nb1_padded = mask_nb1, mask_nb2_padded = mask_nb2, mask_nb3_padded = mask_nb3; + if (extra_mask) { + // allocate mem_matrixMask w/ new padded size + size_t n_kv_padded = GGML_PAD(n_kv, 4); + size_t mask_nb_padded = n_kv_padded * sizeof(cl_half) * mask->ne[1] * mask->ne[2] * mask->ne[3]; - ~ggml_cl_flash_attn_temp_buffer() { - if (data != nullptr) { - CL_CHECK(clReleaseMemObject(data)); - data = nullptr; - } - } -}; + // apply offset and create subBuffer for mask + region.origin = offset_mask; + region.size = ggml_nbytes(mask); + mem_matrixMask = clCreateSubBuffer(extra_mask->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); -// Resolve the source buffer + strides for an FA KV tensor: keep the -// caller-supplied AoS buffer if non-NULL, else fall back to tensor->extra. -static void ggml_cl_flash_attn_resolve_src( - const ggml_tensor * tensor, - cl_mem & buf, - cl_ulong & offset, - cl_ulong & nb1, - cl_ulong & nb2, - cl_ulong & nb3) { - if (buf != NULL) { - return; - } - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra && extra->data_device); - buf = extra->data_device; - offset = extra->offset + tensor->view_offs; - nb1 = tensor->nb[1]; - nb2 = tensor->nb[2]; - nb3 = tensor->nb[3]; -} + { + // create padded mask to contain all data + mem_matrixMask_padded = clCreateBuffer(context, CL_MEM_ALLOC_HOST_PTR, mask_nb_padded, NULL, &err); + CL_CHECK(err); -// Read a (possibly strided-view) tensor from device into a tight host buffer. -// dim 0 is always tight; a strided view is gathered row-by-row. -static void ggml_cl_flash_attn_read_tensor_host( - ggml_backend_opencl_context * backend_ctx, - const ggml_tensor * tensor, - cl_mem src_buffer, cl_ulong src_offset, - cl_ulong src_nb1, cl_ulong src_nb2, cl_ulong src_nb3, - size_t row_bytes, void * dst, size_t total_bytes -) { - const bool contiguous_layout = - src_nb1 == row_bytes && - src_nb2 == row_bytes * (cl_ulong) tensor->ne[1] && - src_nb3 == src_nb2 * (cl_ulong) tensor->ne[2]; + // pass extra_mask->data_device, mem_matrixMask to kernel for copying/padding + mask_nb1_padded = (cl_ulong)n_kv_padded * sizeof(cl_half); + mask_nb2_padded = mask_nb1_padded * (cl_ulong)mask->ne[1]; + mask_nb3_padded = mask_nb2_padded * (cl_ulong)mask->ne[2]; - if (contiguous_layout) { - CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, - src_offset, total_bytes, dst, 0, NULL, NULL)); - return; - } + cl_kernel repack_mask = backend_ctx->fa.kernel_repack_mask_for_wmm; + CL_CHECK(clSetKernelArg(repack_mask, 0, sizeof(cl_mem), &mem_matrixMask)); + CL_CHECK(clSetKernelArg(repack_mask, 1, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(repack_mask, 2, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(repack_mask, 3, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(repack_mask, 4, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(repack_mask, 5, sizeof(cl_mem), &mem_matrixMask_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 6, sizeof(cl_ulong), &mask_nb1_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 7, sizeof(cl_ulong), &mask_nb2_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 8, sizeof(cl_ulong), &mask_nb3_padded)); - size_t dst_off = 0; - for (int64_t i3 = 0; i3 < tensor->ne[3]; ++i3) { - for (int64_t i2 = 0; i2 < tensor->ne[2]; ++i2) { - for (int64_t i1 = 0; i1 < tensor->ne[1]; ++i1) { - const cl_ulong row_src_off = src_offset + - (cl_ulong) i3 * src_nb3 + - (cl_ulong) i2 * src_nb2 + - (cl_ulong) i1 * src_nb1; - CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, - row_src_off, row_bytes, - (uint8_t *) dst + dst_off, 0, NULL, NULL)); - dst_off += row_bytes; - } + size_t repack_mask_gws[3] = {(size_t)n_kv, (size_t)mask->ne[1], (size_t)mask_ne2 * (size_t)mask->ne[3]}; + backend_ctx->enqueue_ndrange_kernel(repack_mask, 3, repack_mask_gws, NULL, dst); } + + // use image 1d buffer for matrix Mask (padded row stride) + cl_image_format img_fmt_mask_1d = { CL_RGBA, CL_HALF_FLOAT}; + cl_image_desc img_desc_mask_1d; + memset(&img_desc_mask_1d, 0, sizeof(img_desc_mask_1d)); + img_desc_mask_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_mask_1d.image_width = mask_nb_padded / 2 / 4; + img_desc_mask_1d.buffer = mem_matrixMask_padded; + mem_tex_matrixMask_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_mask_1d, &img_desc_mask_1d, NULL, &err); + CL_CHECK(err); } - GGML_ASSERT(dst_off == total_bytes); -} -// forward decl: used by the FA decode dispatch (K-image variant) below. -static cl_mem ggml_cl_img_pool_get_or_create( - ggml_backend_opencl_context * backend_ctx, - std::map<ggml_backend_opencl_context::ImagePoolKey, - ggml_backend_opencl_context::ImagePoolEntry> & pool, - cl_mem data_device, - cl_ulong offset0, - size_t required_bytes, - cl_channel_type channel_data_type); + // WMM QK uses repacked 3D images. + // Q image: rows, heads, packed depth. + cl_image_format img_fmt_3d = { CL_RGBA, CL_HALF_FLOAT }; + cl_image_desc img_desc_3d; -// Rebuild AoS q8_0/q4_0 bytes from a SoA tensor into a temp buffer. -// Returns false if the tensor is not SoA-quantised (already AoS). -static bool ggml_cl_flash_attn_reconstruct_aos( - ggml_backend_opencl_context * backend_ctx, - const ggml_tensor * tensor, - ggml_cl_flash_attn_temp_buffer & temp, - cl_mem & out_buf, - cl_ulong & out_offset, - cl_ulong & out_nb1, - cl_ulong & out_nb2, - cl_ulong & out_nb3 -) { - if (tensor == nullptr) { - return false; - } - const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(tensor); - const bool is_q4_0 = tensor->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(tensor); - if (!is_q8_0 && !is_q4_0) { - return false; - } + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)n_q; + img_desc_3d.image_height = (size_t)n_batch * (size_t)n_head; + img_desc_3d.image_depth = (size_t)d_head_q / 4; + cl_mem img_q_wmm = NULL; + img_q_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); - // For views, SoA extra is on view_src (view->extra is pre-SoA). - // Noshuffle layout only applies to 2D weights, as determined by `use_adreno_kernels`, - // where ne2 == 1 and ne3 == 1 -- these are never FA inputs. - // Therefore, we use `restore_block_qk_0` kernels, not `restore_block_qk_0_noshuffle`. - const ggml_tensor * soa_src = tensor->view_src ? tensor->view_src : tensor; - cl_mem extra_q = NULL; - cl_mem extra_d = NULL; - if (is_q8_0) { - auto * e = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; - GGML_ASSERT(e && e->q && e->d); - extra_q = e->q; - extra_d = e->d; - } else { - auto * e = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; - GGML_ASSERT(e && e->q && e->d); - extra_q = e->q; - extra_d = e->d; + { + cl_kernel repack_q = backend_ctx->fa.kernel_repack_q_for_wmm; + CL_CHECK(clSetKernelArg(repack_q, 0, sizeof(cl_mem), &mem_matrixQ)); + CL_CHECK(clSetKernelArg(repack_q, 1, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(repack_q, 2, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(repack_q, 3, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(repack_q, 4, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(repack_q, 5, sizeof(cl_mem), &img_q_wmm)); + + size_t repack_q_gws[3] = {(size_t)d_head_q / 4, (size_t)n_q, (size_t)n_batch * (size_t)n_head}; + backend_ctx->enqueue_ndrange_kernel(repack_q, 3, repack_q_gws, NULL, dst); } - // Reconstruct the whole parent; view offsets then work naturally. - const size_t parent_nbytes = ggml_nbytes(soa_src); - cl_int err; - temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); - CL_CHECK(err); - - cl_kernel kernel = is_q8_0 ? backend_ctx->kernel_restore_block_q8_0 - : backend_ctx->kernel_restore_block_q4_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &temp.data)); + // K image: columns, row groups, KV heads. + const size_t n_kv_row4 = ((size_t)n_kv + 3) / 4; - const size_t n_blocks = (size_t) ggml_nelements(soa_src) / ggml_blck_size(soa_src->type); - size_t global_work_size[] = { n_blocks, 1, 1 }; - size_t local_work_size[] = { 1, 1, 1 }; - CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, NULL)); + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)d_head_q; + img_desc_3d.image_height = n_kv_row4; + img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; + cl_mem img_k_wmm = NULL; + img_k_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); - out_buf = temp.data; - out_offset = tensor->view_offs; - out_nb1 = tensor->nb[1]; - out_nb2 = tensor->nb[2]; - out_nb3 = tensor->nb[3]; - return true; -} + { + cl_kernel repack_k = backend_ctx->fa.kernel_repack_k_for_wmm; + CL_CHECK(clSetKernelArg(repack_k, 0, sizeof(cl_mem), &mem_matrixK)); + CL_CHECK(clSetKernelArg(repack_k, 1, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(repack_k, 2, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(repack_k, 3, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(repack_k, 4, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(repack_k, 5, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(repack_k, 6, sizeof(cl_mem), &img_k_wmm)); -// GPU dequant of a contiguous q4_0/q8_0 KV tensor to f16/f32. Caller supplies -// src_buf when reconstructing from SoA. Returns false for non-contig layouts -// (the kernel indexes blocks tightly within ne[0]) so the caller can fall back -// to the host path. -static bool ggml_cl_flash_attn_dequant_kv_gpu( - ggml_backend_opencl_context * backend_ctx, - const ggml_tensor * tensor, - ggml_type target_type, - cl_mem in_src_buf, - cl_ulong in_src_offset, - cl_ulong in_src_nb1, - cl_ulong in_src_nb2, - cl_ulong in_src_nb3, - ggml_cl_flash_attn_temp_buffer & temp, - cl_mem & out_buf, - cl_ulong & out_offset, - cl_ulong & out_nb1, - cl_ulong & out_nb2, - cl_ulong & out_nb3 -) { - GGML_ASSERT(tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q4_0); - GGML_ASSERT(target_type == GGML_TYPE_F16 || target_type == GGML_TYPE_F32); + size_t repack_k_gws[3] = {(size_t)d_head_q, n_kv_row4, (size_t)n_batch * (size_t)n_head_kv}; + backend_ctx->enqueue_ndrange_kernel(repack_k, 3, repack_k_gws, NULL, dst); + } - const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0; + // V image: kv-rows (contracted), packed head-dim groups, KV heads. + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)n_kv; + img_desc_3d.image_height = (size_t)d_head_v / 4; + img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; + cl_mem img_v_wmm = NULL; + img_v_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); - cl_mem src_buf = in_src_buf; - cl_ulong src_offset = in_src_offset; - cl_ulong src_nb1 = in_src_nb1; - cl_ulong src_nb2 = in_src_nb2; - cl_ulong src_nb3 = in_src_nb3; - ggml_cl_flash_attn_resolve_src(tensor, src_buf, src_offset, src_nb1, src_nb2, src_nb3); + { + cl_kernel repack_v = backend_ctx->fa.kernel_repack_v_for_wmm; + CL_CHECK(clSetKernelArg(repack_v, 0, sizeof(cl_mem), &mem_matrixV)); + CL_CHECK(clSetKernelArg(repack_v, 1, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(repack_v, 2, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(repack_v, 3, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(repack_v, 4, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(repack_v, 5, sizeof(cl_mem), &img_v_wmm)); - if (tensor->nb[0] != (cl_ulong) ggml_type_size(tensor->type)) { - return false; + size_t repack_v_gws[3] = {(size_t)d_head_v / 4, (size_t)n_kv, (size_t)n_batch * (size_t)n_head_kv}; + backend_ctx->enqueue_ndrange_kernel(repack_v, 3, repack_v_gws, NULL, dst); } - const size_t n_blocks = (size_t) ggml_nelements(tensor) / 32; // block size is 32 - const size_t elem_size = ggml_type_size(target_type); - const size_t out_bytes = n_blocks * 32 * elem_size; - const cl_int nblk0_arg = (cl_int) (tensor->ne[0] / 32); - const cl_int ne1_arg = (cl_int) tensor->ne[1]; - const cl_int ne2_arg = (cl_int) tensor->ne[2]; - const cl_int ne3_arg = (cl_int) tensor->ne[3]; + cl_int enable_mask = (extra_mask) ? 1 : 0; + mask_buffer = extra_mask ? mem_tex_matrixMask_1dbuf : mem_tex_mask_fallback_1dbuf; - cl_int err; - temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); - CL_CHECK(err); + cl_mem mem_sinksBuf = NULL; + cl_mem mem_tex_sinks_1dbuf = NULL; + cl_int enable_sinks = (sinks_buffer != NULL) ? 1 : 0; + if (enable_sinks) { + region.origin = offset_sinks; + region.size = ggml_nbytes(sinks); + mem_sinksBuf = clCreateSubBuffer(extra_sinks->data_device, CL_MEM_READ_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - cl_kernel kernel; - if (target_type == GGML_TYPE_F16) { - kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f16_view_aos - : backend_ctx->kernel_dequant_q4_0_f16_view_aos; + cl_image_format img_fmt_sinks_1d = { CL_R, CL_FLOAT }; + cl_image_desc img_desc_sinks_1d; + memset(&img_desc_sinks_1d, 0, sizeof(img_desc_sinks_1d)); + img_desc_sinks_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_sinks_1d.image_width = (size_t)n_head; + img_desc_sinks_1d.buffer = mem_sinksBuf; + mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_1d, &img_desc_sinks_1d, NULL, &err); + CL_CHECK(err); } else { - kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f32_view_aos - : backend_ctx->kernel_dequant_q4_0_f32_view_aos; + // The image obj cannot be null so we back with buffer of size 1 and use matrixK to back because it always exists + cl_image_format img_fmt_sinks_fallback = { CL_R, CL_FLOAT }; + cl_image_desc img_desc_sinks_fallback; + memset(&img_desc_sinks_fallback, 0, sizeof(img_desc_sinks_fallback)); + img_desc_sinks_fallback.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_sinks_fallback.image_width = 1; + img_desc_sinks_fallback.buffer = mem_matrixK; + mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_fallback, &img_desc_sinks_fallback, NULL, &err); + CL_CHECK(err); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &src_offset)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &src_nb1)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &src_nb2)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &src_nb3)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &nblk0_arg)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne1_arg)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne2_arg)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne3_arg)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &temp.data)); + cl_uint arg = 0; - size_t global_ws[3] = { (size_t) nblk0_arg, (size_t) ne1_arg, (size_t) ne2_arg * (size_t) ne3_arg }; - CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, - global_ws, NULL, 0, NULL, NULL)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_matrixO_1dbuf)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &is_causal)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_log2_val)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &logit_softcap)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_mask)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb1_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb2_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb3_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_sinks_1dbuf)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_sinks)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_q_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_k_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_v_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &d_head_q)); - out_buf = temp.data; - out_offset = 0; - out_nb1 = (cl_ulong) tensor->ne[0] * elem_size; - out_nb2 = out_nb1 * (cl_ulong) tensor->ne[1]; - out_nb3 = out_nb2 * (cl_ulong) tensor->ne[2]; - return true; -} + size_t global_work_size[3], local_work_size[3]; -static bool ggml_cl_flash_attn_prepare_quantized_tensor( - ggml_backend_opencl_context * backend_ctx, - const ggml_tensor * tensor, - ggml_type target_type, - ggml_cl_flash_attn_temp_buffer & temp, - cl_mem & data_device, - cl_ulong & offset, - cl_ulong & nb1, - cl_ulong & nb2, - cl_ulong & nb3 -) { - if (!ggml_is_quantized(tensor->type)) { - return false; - } + const int n_waves_v = d_head_q / 64; - // Caller-supplied AoS buffer wins over tensor->extra when present. - cl_mem src_buffer = data_device; - cl_ulong src_offset = offset; - cl_ulong src_nb1 = nb1; - cl_ulong src_nb2 = nb2; - cl_ulong src_nb3 = nb3; - ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); + local_work_size[0] = 64; + local_work_size[1] = n_waves_v; + local_work_size[2] = 1; + + global_work_size[0] = 64; + global_work_size[1] = ((n_q + 64 - 1) / 64) * n_waves_v; + global_work_size[2] = n_batch * n_head; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(mem_tex_matrixO_1dbuf)); + CL_CHECK(clReleaseMemObject(img_q_wmm)); + CL_CHECK(clReleaseMemObject(img_k_wmm)); + CL_CHECK(clReleaseMemObject(img_v_wmm)); + + if (mem_tex_matrixMask_1dbuf) { + CL_CHECK(clReleaseMemObject(mem_tex_matrixMask_1dbuf)); + } + if (mem_matrixMask) { + CL_CHECK(clReleaseMemObject(mem_matrixMask)); + } + if (mem_matrixMask_padded) { + CL_CHECK(clReleaseMemObject(mem_matrixMask_padded)); + } + if (mem_tex_sinks_1dbuf) { + CL_CHECK(clReleaseMemObject(mem_tex_sinks_1dbuf)); + } + if (mem_sinksBuf) { + CL_CHECK(clReleaseMemObject(mem_sinksBuf)); + } + CL_CHECK(clReleaseMemObject(mem_matrixQ)); + CL_CHECK(clReleaseMemObject(mem_matrixK)); + CL_CHECK(clReleaseMemObject(mem_matrixV)); + CL_CHECK(clReleaseMemObject(mem_matrixO)); +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - const int64_t n = ggml_nelements(tensor); - const size_t row_bytes = (size_t) (tensor->ne[0] / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); - // tight-packed byte count (ggml_nbytes includes stride gaps). - const size_t total_bytes = (size_t) (n / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); - std::vector<uint8_t> host_quant(total_bytes); +static void ggml_cl_fa_kq_p8_dispatch( + ggml_backend_t backend, + const ggml_tensor * q, const ggml_tensor * k, + const ggml_tensor * mask, const ggml_tensor * sinks, + const ggml_tensor * dst, + int64_t q0, int nqc, float scale, float max_bias, int p_pitch +) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra); + auto * extra_q = (ggml_tensor_extra_cl *) q->extra; + auto * extra_m = (ggml_tensor_extra_cl *) mask->extra; - sync_with_other_backends(backend_ctx); - ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, - src_nb1, src_nb2, src_nb3, - row_bytes, host_quant.data(), total_bytes); + const int M = k->ne[1]; + const int K = k->ne[0]; + const int N = nqc; + const int D_A = k->ne[2]; + const int D_B = q->ne[2]; - std::vector<float> host_f32(n); - ggml_get_type_traits(tensor->type)->to_float(host_quant.data(), host_f32.data(), n); + cl_int err; - const size_t bytes_per_elem = ggml_type_size(target_type); - const size_t buffer_size = (size_t) n * bytes_per_elem; + cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]}; + cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - std::vector<uint8_t> host_linear(buffer_size); - if (target_type == GGML_TYPE_F32) { - memcpy(host_linear.data(), host_f32.data(), buffer_size); - } else { - GGML_ASSERT(target_type == GGML_TYPE_F16); - ggml_fp32_to_fp16_row(host_f32.data(), (ggml_fp16_t *) host_linear.data(), n); - } + cl_image_format fmt = {CL_RGBA, CL_FLOAT}; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = region.size / 16; + desc.buffer = k_sub; + cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err); + CL_CHECK(err); - cl_int err; - temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, buffer_size, NULL, &err); + region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1]; + region.size = (size_t) K * N * D_B * sizeof(float); + cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, buffer_size, host_linear.data(), 0, NULL, NULL)); - data_device = temp.data; - offset = 0; - nb1 = (cl_ulong) (tensor->ne[0] * bytes_per_elem); - nb2 = (cl_ulong) (tensor->ne[1] * nb1); - nb3 = (cl_ulong) (tensor->ne[2] * nb2); + const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1]; + const cl_ulong mask_nb1 = mask->nb[1]; + const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B)); + const float m0 = powf(2.0f, -max_bias / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - static bool warned = false; - if (!warned) { - GGML_LOG_WARN("ggml_opencl: OpenCL flash attention dequantizes GPU-resident quantized KV cache into temporary linear buffers; performance may be poor\n"); - warned = true; - } + cl_uint a = 0; + + cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch)); + + const size_t n_tiles = (N + 31) / 32; + size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B}; + + const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128; + size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1}; - return true; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + CL_CHECK(clReleaseMemObject(k_img)); + CL_CHECK(clReleaseMemObject(k_sub)); + CL_CHECK(clReleaseMemObject(q_sub)); + + cl_mem sinks_buf = extra_q->data_device; + cl_ulong sinks_off = 0; + const int has_sinks = sinks != nullptr; + + if (sinks) { + auto * extra_s = (ggml_tensor_extra_cl *) sinks->extra; + sinks_buf = extra_s->data_device; + sinks_off = extra_s->offset + sinks->view_offs; + } + + const int nblk = M / 32; + const int row_blk = p_pitch / 32; + + kernel = backend_ctx->kernel_fa_p8_fixup; + + a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_sums.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &sinks_buf)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &sinks_off)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &has_sinks)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nblk)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &row_blk)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N)); + + size_t gws_fixup[3] = {(size_t) 64 * N, (size_t) D_B, 1}; + size_t lws_fixup[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws_fixup, lws_fixup, dst); } -// Host-side F16 -> F32 for the asymmetric-KV F32 fallback path. -static bool ggml_cl_flash_attn_convert_f16_to_f32( - ggml_backend_opencl_context * backend_ctx, - const ggml_tensor * tensor, - ggml_cl_flash_attn_temp_buffer & temp, - cl_mem & data_device, - cl_ulong & offset, - cl_ulong & nb1, - cl_ulong & nb2, - cl_ulong & nb3 +static void ggml_cl_fa_decompose_scratch_tensor( + ggml_tensor & t, ggml_tensor_extra_cl & extra, cl_mem buffer, ggml_type type, + int64_t ne0, int64_t ne1, int64_t ne2 ) { - if (tensor->type != GGML_TYPE_F16) { - return false; - } - - cl_mem src_buffer = data_device; - cl_ulong src_offset = offset; - cl_ulong src_nb1 = nb1; - cl_ulong src_nb2 = nb2; - cl_ulong src_nb3 = nb3; - ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); + memset(&t, 0, sizeof(t)); + t.type = type; + t.ne[0] = ne0; t.ne[1] = ne1; t.ne[2] = ne2; t.ne[3] = 1; + t.nb[0] = ggml_type_size(type); + t.nb[1] = t.nb[0]*ne0; + t.nb[2] = t.nb[1]*ne1; + t.nb[3] = t.nb[2]*ne2; + + extra.reset(); + extra.data_device = buffer; + t.extra = &extra; +} - const int64_t n = ggml_nelements(tensor); - const size_t row_bytes = (size_t) tensor->ne[0] * sizeof(ggml_fp16_t); - const size_t total_bytes = (size_t) n * sizeof(ggml_fp16_t); - std::vector<uint8_t> host_f16(total_bytes); +static void ggml_cl_fa_decompose_write_chunk( + ggml_backend_t backend, const ggml_tensor & kqv, int64_t q0, ggml_tensor * dst +) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const int64_t dv = kqv.ne[0]; + const int64_t nqc = kqv.ne[1]; + const int64_t n_head = kqv.ne[2]; - sync_with_other_backends(backend_ctx); - ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, - src_nb1, src_nb2, src_nb3, - row_bytes, host_f16.data(), total_bytes); + // apply the deferred 1/sum to the KQV rows + { + cl_kernel kernel = backend_ctx->kernel_fa_decompose_scale_rows_f32; + const cl_ulong offset_kqv = 0; + const cl_ulong offset_sums = 0; + const int dv_i = (int) dv; + const int nqc_i = (int) nqc; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &backend_ctx->prealloc_fa_kqv.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_kqv)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &backend_ctx->prealloc_fa_sums.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_sums)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &dv_i)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &nqc_i)); + + size_t gws[3] = { (size_t) 64 * nqc, (size_t) n_head, 1 }; + size_t lws[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } - std::vector<float> host_f32(n); - ggml_fp16_to_fp32_row((const ggml_fp16_t *) host_f16.data(), host_f32.data(), n); + // permute into dst: [dv, nqc, n_head] -> dst[dv, n_head, q0 .. q0+nqc) + { + ggml_tensor kqv_perm = kqv; + kqv_perm.ne[1] = n_head; + kqv_perm.ne[2] = nqc; + kqv_perm.nb[1] = kqv.nb[2]; + kqv_perm.nb[2] = kqv.nb[1]; - const size_t f32_bytes = (size_t) n * sizeof(float); - cl_int err; - temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, f32_bytes, NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, - f32_bytes, host_f32.data(), 0, NULL, NULL)); + ggml_tensor dst_chunk = *dst; + dst_chunk.ne[2] = nqc; + dst_chunk.view_offs = dst->view_offs + (size_t) q0 * dst->nb[2]; - data_device = temp.data; - offset = 0; - nb1 = (cl_ulong) (tensor->ne[0] * sizeof(float)); - nb2 = (cl_ulong) (tensor->ne[1] * nb1); - nb3 = (cl_ulong) (tensor->ne[2] * nb2); + ggml_cl_cpy(backend, &kqv_perm, &dst_chunk, nullptr); + } +} - static bool warned = false; - if (!warned) { - GGML_LOG_WARN("ggml_opencl: OpenCL flash attention asymmetric KV converts an F16 cache to F32 host-side; performance may be poor\n"); - warned = true; +static void ggml_cl_flash_attn_decompose( + ggml_backend_t backend, + const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst, + int64_t n_q_chunk, float scale, float max_bias +) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + const int64_t n_q = q->ne[1]; + const int64_t n_kv = k->ne[1]; + const int64_t dv = v->ne[0]; + const int64_t n_head = q->ne[2]; + const int64_t n_head_kv = k->ne[2]; + + const std::pair<int, int> dk_dv = {(int) q->ne[0], (int) dv}; + + // The image GEMMs work in 64 x 32 tiles and can touch past the exact extent, + // so size each scratch by the padded shape plus one extra head. + auto pad_up = [](int64_t x, int64_t a) { return ((x + a - 1) / a) * a; }; + const int64_t kq_m = pad_up(n_kv, 64); + const int64_t kqv_m = pad_up(dv, 64); + const int64_t tile_n = pad_up(n_q_chunk, 32); + + backend_ctx->prealloc_fa_vt.allocate(backend_ctx->context, (size_t)(n_kv * kqv_m * (n_head_kv + 1)) * sizeof(ggml_fp16_t)); + backend_ctx->prealloc_fa_kq.allocate(backend_ctx->context, (size_t)(kq_m * tile_n * (n_head + 1)) * sizeof(float)); + backend_ctx->prealloc_fa_kqv.allocate(backend_ctx->context, (size_t)(kqv_m * tile_n * (n_head + 1)) * sizeof(float)); + backend_ctx->prealloc_fa_sums.allocate(backend_ctx->context, (size_t)(n_q_chunk * n_head) * sizeof(float)); + + // V^T once per call: [dv, n_kv, n_head_kv] -> [n_kv, dv, n_head_kv] + { + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; + cl_kernel kernel = backend_ctx->fa.v_transpose_f16.at(dk_dv); + + const cl_ulong offset_v = extra_v->offset + v->view_offs; + const cl_ulong v_nb1 = v->nb[1]; + const cl_ulong v_nb2 = v->nb[2]; + const int n_kv_i = (int) n_kv; + const int dv_i = (int) dv; + const int n_head_kv_i = (int) n_head_kv; + + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vt.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_kv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_kv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb2)); + + size_t gws[3] = { (size_t)((dv + 31) / 32) * 32, (size_t)((n_kv + 31) / 32) * 8, (size_t) n_head_kv }; + size_t lws[3] = { 32, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } - return true; -} + ggml_tensor_extra_cl extra_vt; + ggml_tensor_extra_cl extra_kq; + ggml_tensor_extra_cl extra_kqv; + ggml_tensor vt; + ggml_tensor kq; + ggml_tensor kqv; + ggml_cl_fa_decompose_scratch_tensor(vt, extra_vt, backend_ctx->prealloc_fa_vt.buffer, GGML_TYPE_F16, n_kv, dv, n_head_kv); -// Flash-Decoding (K-split) dispatch thresholds. FD fires for non-causal -// attention with n_kv >= FD_MIN_N_KV and d_head <= FD_MAX_DK; the KV range is -// split into ~n_kv/FD_KV_PER_SPLIT partials, clamped to [FD_MIN_SPLITS, -// FD_MAX_SPLITS]. Multi-query FD is restricted to small heads -// (d_head <= FD_MAX_DK_MULTI) and capped at FD_MAX_N_Q_MULTI queries. -static constexpr int FD_MIN_N_KV = 2048; -static constexpr int FD_KV_PER_SPLIT = 2048; -// f16 KV decode wants more splits than the 2048 default; quantized KV keeps 2048. -static constexpr int FD_KV_PER_SPLIT_F16 = 512; -static constexpr int FD_MIN_SPLITS = 2; -static constexpr int FD_MAX_SPLITS = 16; -static constexpr int FD_MAX_DK = 128; -static constexpr int FD_MAX_DK_MULTI = 64; -static constexpr int FD_MAX_N_Q_MULTI = 8; -// MQ FD split-groups have few subgroups (MQ_NSG_SPLIT), so use a smaller -// kv_per_split to keep the softmax recurrence short; non-MQ keeps FD_KV_PER_SPLIT. -static constexpr int FD_MQ_KV_PER_SPLIT = 256; -static constexpr int FD_MQ_MAX_SPLITS = 128; + for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) { + const int64_t nqc = MIN(n_q_chunk, n_q - q0); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS -struct ggml_cl_adreno_xmem_attn_schedule { - int qk_lws0 = 256; - int qk_lws2 = 1; - int softmax_reduce_lws0 = 256; - int softmax_apply_lws0 = 64; - int softmax_apply_lws2 = 4; - int pv_lws0 = 64; - int pv_lws2 = 4; -}; + // KQ: [dk, n_kv, n_head_kv] x [dk, nqc, n_head] -> [n_kv, nqc, n_head] + ggml_cl_fa_decompose_scratch_tensor(kq, extra_kq, backend_ctx->prealloc_fa_kq.buffer, GGML_TYPE_F32, n_kv, nqc, n_head); -static inline size_t ggml_cl_round_up(size_t x, size_t a) { - return ((x + a - 1) / a) * a; -} + { + ggml_tensor q_chunk = *q; + q_chunk.ne[1] = nqc; + q_chunk.view_offs = q->view_offs + (size_t) q0 * q->nb[1]; + ggml_cl_mul_mat(backend, k, &q_chunk, &kq); + } -static inline int ggml_cl_round_up_div(int x, int y) { - return (x + y - 1) / y; -} + // softmax in place, the row sums go to prealloc_fa_sums + { + memcpy((float *) kq.op_params + 0, &scale, sizeof(float)); + memcpy((float *) kq.op_params + 1, &max_bias, sizeof(float)); -static inline void ggml_cl_set_arg_int4(cl_kernel kernel, cl_uint index, int x, int y, int z, int w) { - struct { int x, y, z, w; } value { x, y, z, w }; - CL_CHECK(clSetKernelArg(kernel, index, sizeof(value), &value)); -} + ggml_tensor mask_chunk = *mask; + mask_chunk.ne[1] = nqc; + mask_chunk.view_offs = mask->view_offs + (size_t) q0 * mask->nb[1]; -static cl_mem ggml_cl_make_image2d_half4(cl_context context, cl_mem_flags flags, size_t width, size_t height) { - cl_int err = CL_SUCCESS; - cl_image_format format = { CL_RGBA, CL_HALF_FLOAT }; - cl_image_desc desc = {}; - desc.image_type = CL_MEM_OBJECT_IMAGE2D; - desc.image_width = width; - desc.image_height = height; - cl_mem image = clCreateImage(context, flags, &format, &desc, nullptr, &err); - CL_CHECK(err); - return image; -} + ggml_cl_fa_decompose_soft_max(backend, &kq, &mask_chunk, sinks, &kq, backend_ctx->prealloc_fa_sums.buffer); + } -static cl_mem ggml_cl_make_image1d_buffer_half4(cl_context context, cl_mem_flags flags, size_t width, cl_mem backing_buffer) { - cl_int err = CL_SUCCESS; - cl_image_format format = { CL_RGBA, CL_HALF_FLOAT }; - cl_image_desc desc = {}; - desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - desc.image_width = width; - desc.buffer = backing_buffer; - cl_mem image = clCreateImage(context, flags, &format, &desc, nullptr, &err); - CL_CHECK(err); - return image; -} + // KQV: [n_kv, dv, n_head_kv] x [n_kv, nqc, n_head] -> [dv, nqc, n_head] + // V^T is contiguous, so mul_mat would not pick the KQV image kernel; call it here. + ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (backend_ctx->kernel_mul_mm_f16_f32_kqv != nullptr && + (dv % 64) == 0 && (n_kv % 16) == 0 && nqc >= 32 && + ((size_t)(vt.nb[2] * n_head_kv / 4) / 4 <= backend_ctx->image_max_buffer_size) && + ((size_t)(dv * nqc * n_head) <= backend_ctx->image_max_buffer_size)) { + ggml_cl_mul_mat_kq_kqv_adreno(backend, &vt, &kq, &kqv, /*is_kq =*/ false); + } else +#endif + { + ggml_cl_mul_mat(backend, &vt, &kq, &kqv); + } -static void ggml_cl_release_mem(cl_mem & mem) { - if (mem != nullptr) { - CL_CHECK(clReleaseMemObject(mem)); - mem = nullptr; + ggml_cl_fa_decompose_write_chunk(backend, kqv, q0, dst); } } -static void ggml_cl_adreno_xmem_attn_release_scratch(ggml_backend_opencl_context * backend_ctx) { - auto & s = backend_ctx->adreno_xmem_attn.scratch; - ggml_cl_release_mem(s.q_img); - ggml_cl_release_mem(s.k_img); - ggml_cl_release_mem(s.v_img); - ggml_cl_release_mem(s.out_img); - ggml_cl_release_mem(s.k_transpose_img1d); - ggml_cl_release_mem(s.k_transpose_buf); - ggml_cl_release_mem(s.k_packed_buf); - ggml_cl_release_mem(s.v_packed_buf); - ggml_cl_release_mem(s.score_img1d); - ggml_cl_release_mem(s.prob_img1d); - ggml_cl_release_mem(s.score_buf); - ggml_cl_release_mem(s.prob_buf); - ggml_cl_release_mem(s.softmax_stats_img2d); - ggml_cl_release_mem(s.xmem_qk); - ggml_cl_release_mem(s.xmem_pv); - s = {}; -} - -static ggml_cl_adreno_xmem_attn_schedule ggml_cl_adreno_xmem_attn_select_schedule( - const ggml_backend_opencl_context * backend_ctx, - int n_q, - int n_kv, - int heads_total, - int q_width, - int gqa_ratio) { - const bool big_h = heads_total >= 8; - ggml_cl_adreno_xmem_attn_schedule sched; +static void ggml_cl_fa_pv_direct(ggml_backend_t backend, ggml_tensor * dst, int kv_pitch) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * v = dst->src[2]; + auto * extra_v = (ggml_tensor_extra_cl *) v->extra; - if (gqa_ratio == 1) { - if (n_q >= 512) { sched.qk_lws0 = 512; } - else if (n_q >= 256) { sched.qk_lws0 = 128; } - else { sched.qk_lws0 = 64; } - sched.qk_lws2 = (big_h && n_q >= 512) ? 2 : 1; - } else { - if (q_width >= 2048) { sched.qk_lws0 = 512; } - else if (q_width >= 256) { sched.qk_lws0 = 128; } - else { sched.qk_lws0 = 64; } - sched.qk_lws2 = MIN(8, (int) backend_ctx->max_workgroup_size / sched.qk_lws0); + const int dv = v->ne[0]; + const int n_q = q->ne[1]; + const int n_head = q->ne[2]; + const int n_head_kv = v->ne[2]; + const int gqa = n_head / n_head_kv; + const int nblk = v->ne[1] / 32; + const int d_blocks = dv / 128; + // Split the KV range to give a small query batch about 256 workgroups. + const int want = MAX(1, 256 / (d_blocks * n_head_kv)); + const int blocks_per_split = (nblk + want - 1) / want; + const int n_split = (nblk + blocks_per_split - 1) / blocks_per_split; + const int n_elem = n_head * n_q * dv; + backend_ctx->prealloc_fa_kq.allocate(backend_ctx->context, (size_t) n_split * n_elem * sizeof(float)); + + const cl_ulong off_v = extra_v->offset + v->view_offs; + const cl_ulong v_nb1 = v->nb[1]; + const cl_ulong v_nb2 = v->nb[2]; + { + cl_kernel kernel = backend_ctx->kernel_fa_kqv_direct_f16; + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_v)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_kq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dv)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &gqa)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nblk)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &blocks_per_split)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &kv_pitch)); + size_t gws[3] = {(size_t) d_blocks * 64, (size_t) n_head_kv, (size_t) n_split}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } + { + cl_kernel kernel = backend_ctx->kernel_fa_kqv_direct_reduce; + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_kq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_kqv.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_elem)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_split)); + size_t gws[3] = {((size_t) n_elem + 63) / 64 * 64, 1, 1}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } +} - if (n_kv >= 2048) { sched.softmax_reduce_lws0 = 1024; } - else if (n_kv >= 512) { sched.softmax_reduce_lws0 = big_h ? 256 : 512; } - else { sched.softmax_reduce_lws0 = 256; } - - if (n_kv < 256) { sched.softmax_apply_lws0 = 64; } - else { sched.softmax_apply_lws0 = big_h ? 128 : 64; } - sched.softmax_apply_lws2 = n_kv >= 512 ? 8 : 4; - - if (n_q < 256) { sched.pv_lws0 = 64; } - else { sched.pv_lws0 = big_h ? 128 : 64; } - sched.pv_lws2 = big_h ? 8 : (n_q <= 256 ? 8 : 4); - - const int max_wg = (int) backend_ctx->max_workgroup_size; - auto fix = [&](int & l0, int & l2) { - while (l0 * l2 > max_wg) { - if (l2 > 1) { l2 /= 2; } - else if (l0 > 32) { l0 /= 2; } - else { break; } - } - }; - fix(sched.qk_lws0, sched.qk_lws2); - fix(sched.softmax_apply_lws0, sched.softmax_apply_lws2); - fix(sched.pv_lws0, sched.pv_lws2); - while (sched.softmax_reduce_lws0 > max_wg) { - sched.softmax_reduce_lws0 /= 2; +static void ggml_cl_flash_attn_decompose_kq_p8( + ggml_backend_t backend, + const ggml_tensor * q, const ggml_tensor * k, + ggml_tensor * dst, + int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct +) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + const int64_t n_q = q->ne[1]; + const int64_t n_kv = k->ne[1]; + const int64_t dv = v->ne[0]; + const int64_t n_head = q->ne[2]; + const int64_t n_head_kv = k->ne[2]; + + auto pad_up = [](int64_t x, int64_t a) { return ((x + a - 1) / a) * a; }; + const int64_t kqv_m = pad_up(dv, 64); + const int64_t tile_n = pad_up(n_q_chunk, 32); + + // Pad the int8 row pitch by one cache line, as in the P8 KQV kernel. + const int kv_pitch = (int) n_kv + 64; + const size_t nblk = (size_t) kv_pitch / 32; + + backend_ctx->prealloc_fa_pq.allocate(backend_ctx->context, (size_t) kv_pitch * n_q_chunk * n_head); + backend_ctx->prealloc_fa_pd.allocate(backend_ctx->context, nblk * n_q_chunk * n_head * sizeof(float)); + backend_ctx->prealloc_fa_bmax.allocate(backend_ctx->context, nblk * n_q_chunk * n_head * sizeof(float)); + backend_ctx->prealloc_fa_bsum.allocate(backend_ctx->context, nblk * n_q_chunk * n_head * sizeof(float)); + backend_ctx->prealloc_fa_kqv.allocate(backend_ctx->context, (size_t)(kqv_m * tile_n * (n_head + 1)) * sizeof(float)); + backend_ctx->prealloc_fa_sums.allocate(backend_ctx->context, (size_t)(n_q_chunk * n_head) * sizeof(float)); + + // Direct PV reads the original V cache and needs no transposed copy. + if (!kqv_direct) { + backend_ctx->prealloc_fa_vtq.allocate(backend_ctx->context, (size_t) kv_pitch * dv * n_head_kv); + backend_ctx->prealloc_fa_vtd.allocate(backend_ctx->context, nblk * dv * n_head_kv * sizeof(cl_half)); + + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; + cl_kernel kernel = backend_ctx->kernel_fa_v_transpose_q8; + + const cl_ulong offset_v = extra_v->offset + v->view_offs; + const cl_ulong v_nb1 = v->nb[1]; + const cl_ulong v_nb2 = v->nb[2]; + const cl_ulong v_nb3 = v->nb[3]; + const int n_kv_i = (int) n_kv; + const int dv_i = (int) dv; + const int n_head_kv_i = (int) n_head_kv; + + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vtq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vtd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_kv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_kv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &kv_pitch)); + + size_t gws[3] = { (size_t)((dv + 31) / 32) * 32, (size_t)((n_kv + 31) / 32) * 8, (size_t) n_head_kv }; + size_t lws[3] = { 32, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } - return sched; -} + ggml_tensor_extra_cl extra_kqv; + ggml_tensor kqv; -static bool ggml_cl_adreno_xmem_attn_prepare( - ggml_backend_opencl_context * backend_ctx, - int n_q, - int n_kv, - int d_head_q, - int d_head_v, - int n_head, - int n_head_kv, - int n_batch) { - auto & s = backend_ctx->adreno_xmem_attn.scratch; - const int gqa_ratio = n_head / n_head_kv; - const int q_width = n_q * gqa_ratio; - const int kv_heads_total = n_head_kv * n_batch; - const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); - if (s.q_img != nullptr && - s.n_q == n_q && - s.n_kv == n_kv && - s.n_kv_padded == n_kv_padded && - s.d_head_q == d_head_q && - s.d_head_v == d_head_v && - s.q_width == q_width && - s.kv_heads_total == kv_heads_total) { - return true; - } + for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) { + const int64_t nqc = MIN(n_q_chunk, n_q - q0); - ggml_cl_adreno_xmem_attn_release_scratch(backend_ctx); + ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch); - const int qpack = d_head_q / 4; - const int vpack = d_head_v / 4; - const int npack = n_kv_padded / 4; - const size_t q_img_h = (size_t) kv_heads_total * qpack; - const size_t v_img_h = (size_t) kv_heads_total * vpack; + ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head); - s.q_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, q_img_h); - s.k_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) n_kv_padded, q_img_h); - s.v_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) n_kv_padded, v_img_h); - s.out_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, v_img_h); + if (kqv_direct) { + GGML_ASSERT(q0 == 0 && nqc == n_q); + ggml_cl_fa_pv_direct(backend, dst, kv_pitch); + } else { + const cl_ulong zero = 0; + const int dv_i = dv; + const int nkv_i = n_kv; + const int nqc_i = nqc; + const int nh_i = n_head; + const int nhkv_i = n_head_kv; + + cl_kernel kernel = backend_ctx->kernel_mul_mm_q8_f32_kqv; + + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vtq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vtd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_kqv.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nkv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nqc_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nh_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nhkv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &kv_pitch)); + size_t gws[3] = {(size_t)(dv / 64) * 64, (size_t)(n_head / n_head_kv) * ((nqc + 31) / 32), (size_t) n_head_kv}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + + ggml_cl_fa_decompose_write_chunk(backend, kqv, q0, dst); + } +} - const size_t k_transpose_half4_elems = (size_t) npack * kv_heads_total * d_head_q; - s.k_transpose_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_transpose_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); - GGML_ASSERT(s.k_transpose_buf != nullptr); - s.k_transpose_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, k_transpose_half4_elems, s.k_transpose_buf); +// Decomposed flash attention prefill. +// +// On Adreno the fused prefill tile is slow at large head sizes, while the +// KQ / KQV image GEMMs are fast. -fa 0 uses those GEMMs, but it keeps KQ for +// the whole ubatch in the graph. Here the FLASH_ATTN_EXT node stays, and the +// queries are split into chunks. For each chunk: KQ -> softmax -> KQV into +// backend scratch, so the scratch size depends on the chunk and not on the +// ubatch, and the KV cache keeps the flash attention layout. +// +// Returns false when the shape is not supported, then the fused tile runs. +// GGML_OPENCL_FA_DECOMPOSE=0 disables it, =1 enables it for any Adreno GPU and +// head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256). +// GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV. +// GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches. +static bool ggml_cl_flash_attn_decompose_dispatch( + ggml_backend_t backend, + const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst +) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; - const size_t k_groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_q, 16); - const size_t v_groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_v, 16); - const size_t k_packed_half4_elems = (size_t) n_kv_padded * k_groups16 * 4; - const size_t v_packed_half4_elems = (size_t) n_kv_padded * v_groups16 * 4; - s.k_packed_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_packed_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); - s.v_packed_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_packed_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); - GGML_ASSERT(s.k_packed_buf != nullptr && s.v_packed_buf != nullptr); + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; - const size_t score_half4_elems = (size_t) npack * kv_heads_total * q_width; - const size_t score_bytes = score_half4_elems * sizeof(uint16_t) * 4; - s.score_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, score_bytes, nullptr, nullptr); - s.prob_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, score_bytes, nullptr, nullptr); - GGML_ASSERT(s.score_buf != nullptr && s.prob_buf != nullptr); - s.score_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, score_half4_elems, s.score_buf); - s.prob_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, score_half4_elems, s.prob_buf); - s.softmax_stats_img2d = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, - (size_t) q_width, (size_t) kv_heads_total); - s.xmem_qk = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY, 6144, nullptr, nullptr); - s.xmem_pv = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY, 6144, nullptr, nullptr); - GGML_ASSERT(s.softmax_stats_img2d != nullptr && s.xmem_qk != nullptr && s.xmem_pv != nullptr); + if (backend_ctx->gpu_family != ADRENO) { + return false; + } - s.n_q = n_q; - s.n_kv = n_kv; - s.n_kv_padded = n_kv_padded; - s.d_head_q = d_head_q; - s.d_head_v = d_head_v; - s.q_width = q_width; - s.kv_heads_total = kv_heads_total; - return true; -} + static const int env = []{ + const char * e = getenv("GGML_OPENCL_FA_DECOMPOSE"); + return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1; + }(); -static bool ggml_cl_adreno_xmem_attn_can_use( - const ggml_backend_opencl_context * backend_ctx, - const ggml_tensor * q, - const ggml_tensor * k, - const ggml_tensor * dst) { - static const char * xmem_sdpa_env = getenv("GGML_OPENCL_XMEM_SDPA"); - if (xmem_sdpa_env == nullptr || xmem_sdpa_env[0] == '0') { + if (env == 0) { return false; } - const ggml_tensor * v = dst->src[2]; - const ggml_tensor * mask = dst->src[3]; - const ggml_tensor * sinks = dst->src[4]; + const int64_t n_q = q->ne[1]; + const int64_t n_kv = k->ne[1]; + const int64_t dk = q->ne[0]; + const int64_t dv = v->ne[0]; + const int64_t n_head = q->ne[2]; + const int64_t n_head_kv = k->ne[2]; - if (!backend_ctx->adreno_xmem_attn.compiled || backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { + const bool measured_gen = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E; + + if (n_q < 2) { return false; } if (q->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || - (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_F32) || - (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32)) { + k->type != GGML_TYPE_F16 || v->type != GGML_TYPE_F16) { return false; } - if (sinks != nullptr) { + if (q->ne[3] != 1 || k->ne[3] != 1 || v->ne[3] != 1 || dst->ne[3] != 1) { return false; } - if (q->nb[0] != ggml_type_size(q->type) || k->nb[0] != ggml_type_size(k->type) || - v->nb[0] != ggml_type_size(v->type) || dst->nb[0] != ggml_type_size(dst->type)) { + if (n_head_kv <= 0 || n_head < n_head_kv || n_head % n_head_kv != 0) { return false; } - if (mask != nullptr && (mask->type != GGML_TYPE_F16 || mask->nb[0] != sizeof(ggml_fp16_t))) { + if (k->nb[0] != ggml_type_size(k->type) || v->nb[0] != ggml_type_size(v->type)) { return false; } - - const int n_q = q->ne[1]; - const int n_kv = k->ne[1]; - const int d_head_q = q->ne[0]; - const int d_head_v = v->ne[0]; - const int n_head = q->ne[2]; - const int n_head_kv = k->ne[2]; - const int n_batch = q->ne[3]; - - if (n_q <= 1 || n_kv <= 0 || n_kv > 8192) { + // K and Q must be packed [dk, head, row], as the KV cache and the permuted Q + // are. mul_mat sends the KQ to the image KQ kernel only for this layout; + // other K views can reach the KQV image kernel there and give wrong results. + if (k->nb[2] != (size_t) dk * sizeof(ggml_fp16_t) || k->nb[1] != (size_t) dk * n_head_kv * sizeof(ggml_fp16_t) || + q->nb[0] != sizeof(float) || q->nb[2] != (size_t) dk * sizeof(float) || q->nb[1] != (size_t) dk * n_head * sizeof(float)) { return false; } - if (d_head_q != k->ne[0] || d_head_v != v->ne[0] || k->ne[1] != v->ne[1] || k->ne[3] != v->ne[3]) { + + // soft_max applies scale, mask, ALiBi and sinks, but not the softcap. + float scale; + float max_bias; + float logit_softcap; + memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + if (logit_softcap != 0.0f) { return false; } - if (q->ne[3] != k->ne[3]) { + + // The deferred softmax kernel needs an f16 mask and rows that divide by 4. + if (mask == nullptr || mask->type != GGML_TYPE_F16 || mask->ne[0] < n_kv || (n_kv % 4) != 0) { return false; } - if (n_head_kv <= 0 || n_head % n_head_kv != 0 || k->ne[2] != v->ne[2]) { + + const std::pair<int, int> dk_dv = {(int) dk, (int) dv}; + if (backend_ctx->fa.v_transpose_f16.count(dk_dv) == 0) { return false; } - if (dst->ne[0] != d_head_v || dst->ne[1] != n_head || dst->ne[2] != n_q || dst->ne[3] != n_batch) { + + static const char * p8_env = getenv("GGML_OPENCL_FA_KQ_P8"); + const bool kq_p8 = (!p8_env || p8_env[0] != '0') && measured_gen && + backend_ctx->kernel_mul_mm_f16_f32_kq_p8 != nullptr && + backend_ctx->kernel_mul_mm_q8_f32_kqv != nullptr && + backend_ctx->kernel_fa_v_transpose_q8 != nullptr && + n_kv <= INT_MAX - 64 && k->nb[1] <= INT_MAX && + (n_kv % 64) == 0 && (dk % 16) == 0 && (dv % 64) == 0 && + mask->ne[2] == 1 && mask->ne[3] == 1 && mask->nb[0] == sizeof(ggml_fp16_t) && + (mask->nb[1] % sizeof(ggml_fp16_t)) == 0 && + k->nb[1] * n_kv / 16 <= backend_ctx->image_max_buffer_size; + + // Keep DK64 with attention sinks opt-in until its accuracy is validated. + const bool dk64_ok = kq_p8 && dst->src[4] == nullptr && n_head / n_head_kv >= 2 && + n_kv >= 1024 && n_q <= n_kv / 2; + const int min_dk = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? (dk64_ok ? 64 : 128) : 256; + if (env != 1 && !(measured_gen && dk >= min_dk)) { return false; } - if ((d_head_q % 8) != 0 || (d_head_v % 32) != 0) { + + static const char * direct_env = getenv("GGML_OPENCL_FA_KQV_DIRECT"); + const bool kqv_direct = (!direct_env || direct_env[0] != '0') && kq_p8 && + backend_ctx->kernel_fa_kqv_direct_f16 != nullptr && backend_ctx->kernel_fa_kqv_direct_reduce != nullptr && + v->ne[1] == n_kv && v->ne[2] == n_head_kv && + n_q <= 32 / (n_head / n_head_kv) && dv >= 128 && dv <= INT_MAX && (dv % 128) == 0 && + n_head <= INT_MAX / (n_q * dv) && (v->nb[1] % 4) == 0 && (v->nb[2] % 4) == 0; + const bool small_p8 = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && + kq_p8 && dk >= 128 && n_kv >= 1024; + // Small batches without direct PV need P8 and a long KV cache on X2E. + if (n_q < 32 && !kqv_direct && !small_p8) { return false; } - if (mask != nullptr && - (mask->ne[0] < n_kv || mask->ne[1] < n_q || mask->ne[2] <= 0 || mask->ne[3] <= 0)) { + + // Chunk size: a byte budget for KQ, at least 8 tiles of 32 queries so the + // GEMMs get usable shapes, and below the image1d_buffer limit of the KQ dst. + static const size_t budget_bytes = []{ + const char * e = getenv("GGML_OPENCL_FA_DECOMPOSE_MB"); + const int mb = (e && e[0]) ? atoi(e) : 256; + return (size_t)(mb > 0 ? mb : 256) * 1024 * 1024; + }(); + + const size_t kq_row_bytes = (size_t) n_kv * n_head * sizeof(float); + int64_t n_q_chunk = MIN((int64_t)(budget_bytes / kq_row_bytes), n_q); + n_q_chunk = MAX(n_q_chunk, MIN((int64_t) 8 * 32, n_q)); + + if (!kq_p8) { + n_q_chunk = MIN(n_q_chunk, (int64_t)(backend_ctx->image_max_buffer_size / (n_kv * n_head))); + } + if (n_q_chunk < 32 && n_q_chunk < n_q) { return false; } - float params[3]; - memcpy(params, dst->op_params, sizeof(params)); - if (params[1] != 0.0f || params[2] != 0.0f) { - return false; + // even out the chunks, then align to the 32-wide n-tile + const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk; + n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31); + + if (kq_p8) { + ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct); + } else { + ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias); } + return true; +} - const int gqa_ratio = n_head / n_head_kv; - const int q_width = n_q * gqa_ratio; - const int kv_heads_total = n_head_kv * n_batch; - const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); - const int qpack = d_head_q / 4; - const int vpack = d_head_v / 4; - const int npack = n_kv_padded / 4; +static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; - if ((size_t) q_width > backend_ctx->image2d_max_width || - (size_t) n_kv_padded > backend_ctx->image2d_max_width) { - return false; + GGML_ASSERT(q->extra); + GGML_ASSERT(k->extra); + GGML_ASSERT(v->extra); + GGML_ASSERT(dst->extra); + + if (mask) { + GGML_ASSERT(mask->extra); } - if ((size_t) kv_heads_total * (size_t) qpack > backend_ctx->image2d_max_height || - (size_t) kv_heads_total * (size_t) vpack > backend_ctx->image2d_max_height) { - return false; + if (sinks) { + GGML_ASSERT(sinks->extra); } - if ((size_t) npack * (size_t) kv_heads_total * (size_t) d_head_q > backend_ctx->image_max_buffer_size || - (size_t) npack * (size_t) kv_heads_total * (size_t) q_width > backend_ctx->image_max_buffer_size) { - return false; + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (ggml_cl_adreno_xmem_attn_can_use(backend_ctx, q, k, dst)) { + ggml_cl_adreno_xmem_attn_run(backend, q, k, dst); + return; + } +#endif + + // Compile DK512 decode separately from the prefill programs. + const bool fa_decode_only_512 = (d_head_q == 512); + + // per-variant lazy compile for this (dk, dv) + // DK=512 decode (n_q==1) needs no prepass + // DK=512 prefill (n_q>1) does, so compile it only when needed + if (!fa_decode_only_512 || n_q > 1) { + ggml_opencl_ensure_fa_pre_kernels(backend_ctx, d_head_q, d_head_v); + } + + cl_kernel kernel = NULL; + bool use_prefill_k_img = false; // K is image1d_buffer_t for DK=512 prefill + + const bool is_f16 = q->type == GGML_TYPE_F16; + const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + const bool is_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && v->type == GGML_TYPE_Q4_0; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_fa_bin_kernels_prefill(backend_ctx, q, k, v)) { + // We support the prefill path of flash attn with a specialized d_head = 64/128/256 + ggml_cl_flash_attn_prefill_bin(backend, q, k, dst); + return; + } +#endif + + if (is_f16) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F16); + } else if (is_mixed) { + if (fa_decode_only_512 && n_q == 1) { + GGML_ASSERT(ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)); + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16); + } + if (fa_decode_only_512) { + // DK=512: the BM-tile prefill kernels are specifically compiled from + // FA_PREFILL_ONLY + if (n_q > 1) { + ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false); + ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/true); + } + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16_SPLIT); + } + } else if (is_q8_0) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0); + if (d_head_q == 96 && d_head_v == 96) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/true); + } else if (d_head_q == 256 && d_head_v == 256) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/true); + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0_SPLIT); + } + } else if (is_q4_0) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0); + if (d_head_q == 96 && d_head_v == 96) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/false); + } else if (d_head_q == 256 && d_head_v == 256) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/false); + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0_SPLIT); + } + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32); + } + + const std::pair<int, int> dk_dv = {d_head_q, d_head_v}; + const int mq_gqa = n_head_kv > 0 ? n_head / n_head_kv : 0; + const std::tuple<int, int, int> mq_decode_key = {d_head_q, d_head_v, mq_gqa}; + // X2E also takes GQA 2 and 6 at head size 256 (gemma-4 SWA layers, Qwen3.5-27B), + // GQA 16 at 512 and 128 (gemma-4-12b global layers, muse-glimmer) and GQA 2 at 512 (gemma-4-E4B MTP drafter) + const bool mq_decode_x2e = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && + ((d_head_q == 256 && (mq_gqa == 2 || mq_gqa == 6)) || + (d_head_q == 512 && (mq_gqa == 2 || mq_gqa == 16)) || + (d_head_q == 128 && mq_gqa == 16 && backend_ctx->has_subgroup_shuffle)); + const bool mq_decode_kernel = backend_ctx->gpu_family == ADRENO && is_mixed && + d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 && + ((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) && + backend_ctx->has_subgroup_shuffle) || + ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)) || + mq_decode_x2e); + const bool mq_decode_shape = mq_decode_kernel && n_q == 1; + // compile on the first call for this shape (the warmup is a prefill), not on the first decode + if (mq_decode_kernel) { + ggml_opencl_ensure_fa_f32_f16_mq_decode(backend_ctx, d_head_q, d_head_v, mq_gqa); + } + + // after all lazy compiles, so the decode kernels are built during warmup as on the fused path + if (n_q > 1 && ggml_cl_flash_attn_decompose_dispatch(backend, q, k, dst)) { + return; } - return true; -} + const bool use_native_q8_0_q1 = is_q8_0 && n_q == 1 && + backend_ctx->fa.f32_q8_0_q1.count(dk_dv) > 0; + // Native q8_0 prefill — reads q8_0 directly, wg_size = cfg->bm. + const bool use_native_q8_0 = is_q8_0 && n_q > 1 && + backend_ctx->fa.f32_q8_0.count(dk_dv) > 0; + const bool use_native_q4_0_q1 = is_q4_0 && n_q == 1 && + backend_ctx->fa.f32_q4_0_q1.count(dk_dv) > 0; + const bool use_native_q4_0 = is_q4_0 && n_q > 1 && + backend_ctx->fa.f32_q4_0.count(dk_dv) > 0; + const int block_m = n_q > 1 + ? (is_mixed ? backend_ctx->fa.f32_f16_bm.at(dk_dv) : backend_ctx->fa.bm.at(dk_dv)) + : 0; + // block_n is only used by the n_q > 1 prefill path; its map is not + // populated for DK=512 decode, so do not read it for decode. + const int block_n = (n_q > 1) + ? (is_mixed ? backend_ctx->fa.f32_f16_bn.at(dk_dv) + : backend_ctx->fa.bn.at(dk_dv)) + : 0; + // Pick split variant only when n_kv crosses the per-(dk,dv) threshold. + // the N_SPLIT>1 prefill tile reduces DK partials via subgroup shuffle, + // on Intel it uses the non-split BM tile and does not depend on subgroup size + const bool use_split_kernel = (n_q > 1 && is_mixed && + backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_f16_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_f16_split_nkv_threshold.at(dk_dv)); + const bool use_split_q8_0 = (use_native_q8_0 && backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_q8_0_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_q8_0_split_nkv_threshold.at(dk_dv)); + const bool use_split_q4_0 = (use_native_q4_0 && backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_q4_0_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_q4_0_split_nkv_threshold.at(dk_dv)); + const int wg_size_fa = (n_q > 1 && is_mixed) + ? (use_split_kernel + ? backend_ctx->fa.f32_f16_split_wg_size.at(dk_dv) + : backend_ctx->fa.f32_f16_wg_size.at(dk_dv)) + : block_m; + + ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; + ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; + ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; -static void ggml_cl_adreno_xmem_attn_run( - ggml_backend_t backend, - const ggml_tensor * q, - const ggml_tensor * k, - ggml_tensor * dst) { - ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; - auto & xstate = backend_ctx->adreno_xmem_attn; - auto & s = xstate.scratch; - if (!xstate.logged) { - GGML_LOG_INFO("ggml_opencl: using Adreno xmem attention path\n"); - xstate.logged = true; - } + // SoA q8_0/q4_0 K/V: data_device aliases the `q` subbuffer; reconstruct + // AoS into a temp buffer below. AoS tensors use extra_k/v->data_device. + const bool k_soa = ggml_cl_is_q8_0_soa(k) || ggml_cl_is_q4_0_soa(k); + const bool v_soa = ggml_cl_is_q8_0_soa(v) || ggml_cl_is_q4_0_soa(v); + ggml_tensor_extra_cl * extra_k = k_soa ? nullptr : (ggml_tensor_extra_cl *)k->extra; + ggml_tensor_extra_cl * extra_v = v_soa ? nullptr : (ggml_tensor_extra_cl *)v->extra; - const ggml_tensor * v = dst->src[2]; - const ggml_tensor * mask = dst->src[3]; + cl_ulong offset_q = extra_q->offset + q->view_offs; + cl_ulong offset_k = k_soa ? 0 : extra_k->offset + k->view_offs; + cl_ulong offset_v = v_soa ? 0 : extra_v->offset + v->view_offs; + cl_ulong offset_o = extra_o->offset + dst->view_offs; + cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; + cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; + cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; + cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; - ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *) q->extra; - ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *) k->extra; - ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *) v->extra; - ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *) dst->extra; - ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *) mask->extra : nullptr; + const cl_ulong q_nb1 = q->nb[1]; + const cl_ulong q_nb2 = q->nb[2]; + const cl_ulong q_nb3 = q->nb[3]; - const cl_ulong offset_q = extra_q->offset + q->view_offs; - const cl_ulong offset_k = extra_k->offset + k->view_offs; - const cl_ulong offset_v = extra_v->offset + v->view_offs; - const cl_ulong offset_o = extra_o->offset + dst->view_offs; - const cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; + cl_ulong k_nb1 = k->nb[1]; + cl_ulong k_nb2 = k->nb[2]; + cl_ulong k_nb3 = k->nb[3]; - const int n_q = q->ne[1]; - const int n_kv = k->ne[1]; - const int d_head_q = q->ne[0]; - const int d_head_v = v->ne[0]; - const int n_head = q->ne[2]; - const int n_head_kv = k->ne[2]; - const int n_batch = q->ne[3]; - const int heads_total = n_head * n_batch; - const int gqa_ratio = n_head / n_head_kv; - const int q_width = n_q * gqa_ratio; - const int kv_heads_total = n_head_kv * n_batch; - const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); - const int qpack = d_head_q / 4; - const int opack = d_head_v / 4; - const int npack = n_kv_padded / 4; - const float scale = ((const float *) dst->op_params)[0]; + cl_ulong v_nb1 = v->nb[1]; + cl_ulong v_nb2 = v->nb[2]; + cl_ulong v_nb3 = v->nb[3]; - GGML_ASSERT(ggml_cl_adreno_xmem_attn_prepare( - backend_ctx, n_q, n_kv, d_head_q, d_head_v, n_head, n_head_kv, n_batch)); - const ggml_cl_adreno_xmem_attn_schedule sched = - ggml_cl_adreno_xmem_attn_select_schedule( - backend_ctx, n_q, n_kv_padded, heads_total, q_width, gqa_ratio); + const cl_ulong o_nb1 = dst->nb[1]; + const cl_ulong o_nb2 = dst->nb[2]; + const cl_ulong o_nb3 = dst->nb[3]; - { - size_t gws[3] = {ggml_cl_round_up((size_t) n_q, 8), (size_t) heads_total, (size_t) qpack}; - size_t lws[3] = {8, 1, (size_t) ((qpack <= 32) ? qpack : 1)}; - cl_kernel kernel = xstate.kernel_q_f32_to_img_scaled; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.q_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &scale)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &d_head_q)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &n_batch)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &q->nb[1])); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &q->nb[2])); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &q->nb[3])); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - } + const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; + const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; + const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; + const int mask_ne2 = mask ? mask->ne[2] : 0; + const int mask_ne3 = mask ? mask->ne[3] : 0; - { - size_t gws[3] = {(size_t) n_kv_padded, (size_t) kv_heads_total, (size_t) qpack}; - size_t lws[3] = {8, 1, (size_t) ((qpack <= 32) ? qpack : 1)}; - cl_kernel kernel = k->type == GGML_TYPE_F16 ? - xstate.kernel_kv_f16_to_img_gqa : xstate.kernel_kv_f32_to_img_gqa; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_k->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_k)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.k_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_q)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_kv_padded)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &k->nb[1])); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &k->nb[2])); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &k->nb[3])); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - } + float scale; + float max_bias; + float logit_softcap; - { - size_t gws[3] = {(size_t) n_kv_padded, (size_t) kv_heads_total, (size_t) opack}; - size_t lws[3] = {8, 1, (size_t) ((opack <= 32) ? opack : 1)}; - cl_kernel kernel = v->type == GGML_TYPE_F16 ? - xstate.kernel_kv_f16_to_img_gqa : xstate.kernel_kv_f32_to_img_gqa; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_v->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_v)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.v_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_v)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_kv_padded)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &v->nb[1])); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &v->nb[2])); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &v->nb[3])); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - } + const float * params = (const float *)dst->op_params; + scale = params[0]; + max_bias = params[1]; + logit_softcap = params[2]; - { - size_t gws[3] = {(size_t) d_head_q, (size_t) kv_heads_total, (size_t) npack}; - size_t lws[3] = {(size_t) MIN(64, d_head_q), (size_t) (kv_heads_total >= 2 ? 2 : 1), (size_t) MIN(8, npack)}; - if (lws[0] * lws[1] * lws[2] > backend_ctx->max_workgroup_size) { - lws[1] = 1; + bool use_q1_vec = false; + bool use_q1_vec_mq = false; + bool use_local_tile = false; + // KV-head-coalesced gate: gqa_ratio == compile-time MQ_GQA + // restricts to DK=DV=256 for now due to local memory size + const int gqa_ratio_dispatch = n_head_kv > 0 ? (n_head / n_head_kv) : 0; + if (n_q == 1) { + if (use_native_q8_0_q1) { + if (d_head_v >= 256 && + backend_ctx->fa.f32_q8_0_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_q8_0_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); + } + } else if (use_native_q4_0_q1) { + // q4_0 vec kernel uses per-lane dp4a (cl_khr_integer_dot_product) + const char * q4vec_env = getenv("GGML_OPENCL_FA_Q4_VEC"); + const bool q4vec_off = (q4vec_env != NULL) && (q4vec_env[0] == '0'); + if (!q4vec_off && d_head_v >= 256 && + backend_ctx->fa.f32_q4_0_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_q4_0_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); + } + } else if (is_mixed) { + static const char * lt_env = getenv("GGML_OPENCL_FA_LOCAL_TILE"); + static const bool lt_on = (lt_env != NULL) && (lt_env[0] != '0'); + if (lt_on && d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_f16_q1_local_tile.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_q1_local_tile.at(dk_dv); + use_local_tile = true; + } else { + static const char * f16_vec_dk128_env = getenv("GGML_OPENCL_FA_F16_VEC_DK128"); + static const bool f16_vec_dk128_off = (f16_vec_dk128_env != NULL) && (f16_vec_dk128_env[0] == '0'); + const int dv_gate = f16_vec_dk128_off ? 256 : 128; + if (d_head_v >= dv_gate && + backend_ctx->fa.f32_f16_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); + } + } + } else if (is_f16) { + kernel = backend_ctx->fa.f16_q1.at(dk_dv); + } else { + kernel = backend_ctx->fa.f32_q1.at(dk_dv); + } + } else { + if (use_native_q8_0) { + kernel = use_split_q8_0 + ? backend_ctx->fa.f32_q8_0_split.at(dk_dv) + : backend_ctx->fa.f32_q8_0.at(dk_dv); + } else if (use_native_q4_0) { + kernel = use_split_q4_0 + ? backend_ctx->fa.f32_q4_0_split.at(dk_dv) + : backend_ctx->fa.f32_q4_0.at(dk_dv); + } else if (is_mixed) { + if (use_split_kernel) { + // DK=512 prefill: opt-in texture-cache K reads (image1d_buffer_t). + static const char * pkimg_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); + const bool pkimg_on = (pkimg_env != NULL) && (pkimg_env[0] != '0'); + if (d_head_q == 512 && pkimg_on && + backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_split_k_img.at(dk_dv); + use_prefill_k_img = true; + } else { + kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); + } + } else { + kernel = backend_ctx->fa.f32_f16.at(dk_dv); + } + } else if (is_f16) { + kernel = backend_ctx->fa.f16.at(dk_dv); + } else { + kernel = backend_ctx->fa.f32.at(dk_dv); } - cl_kernel kernel = xstate.kernel_k_gather; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.k_transpose_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_img)); - ggml_cl_set_arg_int4(kernel, 2, n_kv_padded, kv_heads_total, npack, d_head_q); - ggml_cl_set_arg_int4(kernel, 3, qpack, 0, 0, 0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } - { - const size_t groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_q, 16); - const size_t packed_linear = (size_t) n_kv_padded * groups16; - const size_t lws0 = MIN((size_t) 1024, backend_ctx->max_workgroup_size); - size_t gws[3] = {ggml_cl_round_up(packed_linear, lws0), 1, 1}; - size_t lws[3] = {lws0, 1, 1}; - cl_kernel kernel = xstate.kernel_pack_k; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.k_packed_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_transpose_img1d)); - ggml_cl_set_arg_int4(kernel, 2, 8, (int) packed_linear, qpack, d_head_q); - ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, kv_heads_total, kv_heads_total, npack); - ggml_cl_set_arg_int4(kernel, 4, d_head_q, 0, 0, 0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + + // Intel goes to the basic q1 kernel + if (backend_ctx->gpu_family == INTEL && n_q == 1) { + use_q1_vec = use_q1_vec_mq = use_local_tile = false; + if (is_mixed && backend_ctx->fa.f32_f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); } + else if (is_f16 && backend_ctx->fa.f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f16_q1.at(dk_dv); } + else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); } + else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); } + else if (backend_ctx->fa.f32_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q1.at(dk_dv); } } + GGML_ASSERT(kernel != NULL); - { - size_t lws[3] = {(size_t) sched.qk_lws0, 1, (size_t) sched.qk_lws2}; - const int slices_per_group = sched.qk_lws2 * 8; - const size_t groups_z = (size_t) ggml_cl_round_up_div(npack, slices_per_group); - const size_t groups_x = (size_t) ggml_cl_round_up_div(q_width, sched.qk_lws0); - size_t gws[3] = { - lws[0] * groups_z, - groups_x, - (size_t) kv_heads_total * lws[2], - }; + ggml_cl_flash_attn_temp_buffer temp_k; + ggml_cl_flash_attn_temp_buffer temp_v; + ggml_cl_flash_attn_temp_buffer temp_k_pad; + ggml_cl_flash_attn_temp_buffer temp_v_pad; + ggml_cl_flash_attn_temp_buffer temp_mask_pad; + ggml_cl_flash_attn_temp_buffer temp_blk; + const ggml_type kv_target_type = is_f16 ? GGML_TYPE_F16 : GGML_TYPE_F32; - cl_kernel kernel = xstate.kernel_qk_gemm; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.score_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_packed_buf)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.xmem_qk)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &s.q_img)); - ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, npack, q_width, 32); - ggml_cl_set_arg_int4(kernel, 5, qpack, 0, 0, kv_heads_total); - ggml_cl_set_arg_int4(kernel, 6, qpack, 1, 1, 0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - } - cl_mem softmax_input_img = s.score_img1d; - cl_mem softmax_output_buf = s.prob_buf; - cl_mem pv_prob_img = s.prob_img1d; + cl_mem k_data_device = k_soa ? NULL : extra_k->data_device; + cl_mem v_data_device = v_soa ? NULL : extra_v->data_device; - if (mask != nullptr) { - const cl_ulong mask_nb1 = mask->nb[1]; - const cl_ulong mask_nb2 = mask->nb[2]; - const cl_ulong mask_nb3 = mask->nb[3]; - const int mask_ne2 = mask->ne[2]; - const int mask_ne3 = mask->ne[3]; - size_t lws[3] = {(size_t) sched.softmax_apply_lws0, 1, (size_t) sched.softmax_apply_lws2}; - size_t gws[3] = { - ggml_cl_round_up((size_t) q_width, lws[0]), - (size_t) kv_heads_total, - ggml_cl_round_up((size_t) npack, lws[2]), - }; - cl_kernel kernel = xstate.kernel_mask_scores; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.prob_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.score_img1d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_mask->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_mask)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &q_width)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_kv_padded)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &kv_heads_total)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &mask_nb1)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &mask_nb2)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &mask_nb3)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &mask_ne2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &mask_ne3)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + // SoA q8_0/q4_0 -> reconstruct AoS for downstream kernels that expect + // tight records (no-op when k/v is already AoS). + ggml_cl_flash_attn_temp_buffer temp_k_aos; + ggml_cl_flash_attn_temp_buffer temp_v_aos; + ggml_cl_flash_attn_reconstruct_aos(backend_ctx, k, temp_k_aos, + k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + ggml_cl_flash_attn_reconstruct_aos(backend_ctx, v, temp_v_aos, + v_data_device, offset_v, v_nb1, v_nb2, v_nb3); - softmax_input_img = s.prob_img1d; - softmax_output_buf = s.score_buf; - pv_prob_img = s.score_img1d; + // currently FA kernels support KV cache with f16, f32, q4_0 and q8_0. + // there two cases that these kernels cannot cover, + // 1. KV cache types are q4_0 or q8_0, but the FA kernels fail to compile + // 2. KV cache types not currently supported by an FA kernel, e.g., q4_1 + // these two cases are supported here by dequantizing to f32/f16 and this + // causes performance degradation. + // For q4_0 or q8_0 cases that fail kernel compilation, dequant happens in GPU; + // for types that do not have FA kernels, dequant happens on host. + if (!use_native_q8_0_q1 && !use_native_q8_0 && + !use_native_q4_0_q1 && !use_native_q4_0) { + // for q4_0, q8_0 FA kernels that fail to compile + bool k_done = false; + bool v_done = false; + if (k->type == GGML_TYPE_Q8_0 || k->type == GGML_TYPE_Q4_0) { + k_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, k, kv_target_type, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, + temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + } + if (v->type == GGML_TYPE_Q8_0 || v->type == GGML_TYPE_Q4_0) { + v_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, v, kv_target_type, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, + temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } + if (!k_done) { + ggml_cl_flash_attn_prepare_quantized_tensor( + backend_ctx, k, kv_target_type, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + } + if (!v_done) { + ggml_cl_flash_attn_prepare_quantized_tensor( + backend_ctx, v, kv_target_type, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } + // Asymmetric KV on the F32 fallback path: convert the F16 side to F32 + // too. (Symmetric F16 / mixed paths handle F16 directly.) + if (kv_target_type == GGML_TYPE_F32 && !is_mixed && !is_f16) { + ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, k, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, v, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } } - { - size_t lws[3] = {(size_t) sched.softmax_reduce_lws0, 1, 1}; - size_t gws[3] = {ggml_cl_round_up((size_t) q_width, lws[0]), (size_t) kv_heads_total, 1}; - cl_kernel kernel = xstate.kernel_softmax_reduce_basic; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &softmax_input_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.softmax_stats_img2d)); - ggml_cl_set_arg_int4(kernel, 2, kv_heads_total, 1, q_width, n_kv); - ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, q_width, 0, 0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - } - { - size_t lws[3] = {(size_t) sched.softmax_apply_lws0, 1, (size_t) sched.softmax_apply_lws2}; - size_t gws[3] = { - ggml_cl_round_up((size_t) q_width, lws[0]), - (size_t) kv_heads_total, - ggml_cl_round_up((size_t) npack, lws[2]), - }; - cl_kernel kernel = xstate.kernel_softmax_apply_basic; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &softmax_output_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &softmax_input_img)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.softmax_stats_img2d)); - ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, npack, q_width, 1); - ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, q_width, n_kv, 0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - } - { - const size_t groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_v, 16); - const size_t packed_linear = (size_t) n_kv_padded * groups16; - const size_t lws0 = MIN((size_t) 1024, backend_ctx->max_workgroup_size); - size_t gws[3] = {ggml_cl_round_up(packed_linear, lws0), 1, 1}; - size_t lws[3] = {lws0, 1, 1}; - cl_kernel kernel = xstate.kernel_pack_v; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.v_packed_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.v_img)); - ggml_cl_set_arg_int4(kernel, 2, 8, (int) packed_linear, npack, n_kv_padded); - ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, kv_heads_total, opack, 0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - } + cl_mem k_pad_buffer = NULL; + cl_mem v_pad_buffer = NULL; + cl_mem mask_pad_buffer = NULL; + cl_mem blk_buffer = NULL; + cl_ulong mask_pad_nb1 = 0; + cl_ulong mask_pad_nb2 = 0; + cl_ulong mask_pad_nb3 = 0; + + // Flash-Decoding K-split decision. Resolved here, before the prefill + // prepass, because KV-pad and blk prepass are pure overhead when FD fires. + // Do not infer causality from tensor shapes: a NULL mask means full + // (bidirectional) attention, e.g. ViT encoders, where n_q == n_kv as well. + // Causal attention in llama.cpp always comes with an explicit KQ mask. + // Inferring is_causal here corrupted mmproj output on OpenCL (see #23800). + const int is_causal = 0; + const int fd_max_n_q = (d_head_q <= FD_MAX_DK_MULTI) ? FD_MAX_N_Q_MULTI : 1; + cl_kernel fd_k_split = NULL; + bool use_fd_mq = false; + bool use_fd_mq_decode = false; + int fd_head_sub = 1; + size_t fd_mq_wg = 256; // MQ_GQA=4 kernel: Q1_WG_SIZE(64) * MQ_NSG_SPLIT(4) + bool use_fa_k_img = false; // K bound as image1d_buffer_t instead of (buf, offset) { - size_t lws[3] = {(size_t) sched.pv_lws0, 1, (size_t) sched.pv_lws2}; - const int blocks = ggml_cl_round_up_div(opack, 8); - const size_t groups_z = (size_t) ggml_cl_round_up_div(blocks, sched.pv_lws2); - const size_t groups_x = (size_t) ggml_cl_round_up_div(q_width, sched.pv_lws0); - size_t gws[3] = { - lws[0] * groups_z, - groups_x, - (size_t) kv_heads_total * lws[2], - }; + const char * mq_env = getenv("GGML_OPENCL_FA_MQ"); + const bool mq_enabled = (mq_env == NULL) ? true : (mq_env[0] != '0'); + const bool mq_kv_ok = is_mixed || is_q8_0 || is_q4_0; - cl_kernel kernel = xstate.kernel_pv_gemm; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.v_packed_buf)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.xmem_pv)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &pv_prob_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &s.out_img)); - ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, opack, q_width, 32); - ggml_cl_set_arg_int4(kernel, 5, npack, 0, 0, kv_heads_total); - ggml_cl_set_arg_int4(kernel, 6, kv_heads_total * q_width, npack, q_width, 1); - ggml_cl_set_arg_int4(kernel, 7, 1, 0, 0, 0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); - } + const char * lmq_env = getenv("GGML_OPENCL_FA_LOCAL_MQ_SPLIT"); + const bool lmq_on = (lmq_env != NULL) && (lmq_env[0] != '0'); - { - size_t gws[3] = {ggml_cl_round_up((size_t) n_q, 8), (size_t) heads_total, (size_t) opack}; - size_t lws[3] = {8, 1, (size_t) ((opack <= 32) ? opack : 1)}; - cl_kernel kernel = xstate.kernel_img_to_f32; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_o->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_o)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.out_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_v)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &dst->nb[1])); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &dst->nb[2])); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &dst->nb[3])); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + static const char * vec_nq_env = getenv("GGML_OPENCL_FA_VEC_NQ"); + static const int N_MAX_VEC_NQ = (vec_nq_env != NULL && vec_nq_env[0] != '\0') + ? atoi(vec_nq_env) : 1; + + const bool nq_in_vec_range = (n_q >= 1) && (n_q <= N_MAX_VEC_NQ); + const bool nq1_only = (n_q == 1); + + // Cluster-parallel decode default on for Adreno X2E/X1E + static const int c8_env_state = []{ + const char * e = getenv("GGML_OPENCL_FA_C8"); + if (e == NULL || e[0] == '\0') { return -1; } + return (e[0] != '0') ? 1 : 0; + }(); + const bool c8_default_on = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E; + const bool c8_f16_on = (c8_env_state >= 0) ? (c8_env_state == 1) : c8_default_on; + // Quant-KV (q4_0/q8_0) GQA4 c8: default-on X2E + X1E + const bool c8_quant_on = (c8_env_state >= 0) ? (c8_env_state == 1) : c8_default_on; + if (mq_enabled && mq_kv_ok && nq_in_vec_range && !is_causal && + backend_ctx->gpu_family != INTEL && + !use_local_tile && + n_kv >= (mq_decode_shape ? 32 : FD_MIN_N_KV) && + backend_ctx->fa.f32_merge.count(dk_dv) > 0) { + if (mq_decode_shape && backend_ctx->fa.f32_f16_mq_decode.count(mq_decode_key) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_mq_decode.at(mq_decode_key); + fd_mq_wg = backend_ctx->fa.f32_f16_mq_decode_wg.at(mq_decode_key); + fd_head_sub = backend_ctx->fa.f32_f16_mq_decode_hs.at(mq_decode_key); + use_fd_mq = true; + use_fd_mq_decode = true; + } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && + gqa_ratio_dispatch == 8 && + backend_ctx->fa.f32_f16_q1_local_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 64; + } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && + gqa_ratio_dispatch == 4 && + backend_ctx->fa.f32_f16_q1_local_mq_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 64; + } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 4 && + ((d_head_q == 256 && d_head_v == 256) || + (d_head_q == 128 && d_head_v == 128)) && + backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { + const bool k_img_on = d_head_q == 128 && d_head_v == 128 && + getenv("GGML_OPENCL_FA_K_IMG") != NULL && + getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && + backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.count(dk_dv) > 0; + // Cluster-parallel decode + const bool c8_env = d_head_q == 128 && d_head_v == 128 && c8_f16_on; + if (c8_env && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.at(dk_dv); + use_fd_mq = true; + } else if (c8_env && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 128; + } else if (k_img_on) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.at(dk_dv); + use_fd_mq = true; + use_fa_k_img = true; + } else { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); + use_fd_mq = true; + } + // Cluster-parallel decode, DK=DV=256 GQA=8 + } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 256 && d_head_v == 256 && + c8_env_state == 1 && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 128; + // Cluster-parallel decode for the g8 + } else if (is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + c8_f16_on && + (backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count(dk_dv) > 0 || + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2.count(dk_dv) > 0)) { + if (backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.at(dk_dv); + fd_mq_wg = 192; + } else { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2.at(dk_dv); + fd_mq_wg = 128; + } + use_fd_mq = true; + } else if (is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + getenv("GGML_OPENCL_FA_K_IMG") != NULL && + getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + use_fa_k_img = true; + } else if (is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 4 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q8_0_q1_vec_mq_split.count(dk_dv) > 0) { + // Cluster-parallel q8_0 GQA4 + if (c8_quant_on && + backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8.at(dk_dv); + fd_mq_wg = 128; + } else { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split.at(dk_dv); + } + use_fd_mq = true; + } else if (nq1_only && is_q4_0) { + const char * q4_mq_env = getenv("GGML_OPENCL_FA_Q4_MQ"); + const bool q4_mq_on = (q4_mq_env != NULL) && (q4_mq_env[0] != '0'); + // Cluster-parallel q4_0 + const bool q4_c8_on = c8_env_state == 1 && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8.count(dk_dv) > 0; + if (q4_c8_on && gqa_ratio_dispatch == 8 && + d_head_q == 64 && d_head_v == 64) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 128; + } else if (q4_mq_on && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (gqa_ratio_dispatch == 4 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split.count(dk_dv) > 0) { + // Cluster-parallel q4_0 GQA4 + if (c8_quant_on && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8.at(dk_dv); + fd_mq_wg = 128; + } else { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split.at(dk_dv); + } + use_fd_mq = true; + } + } + } } -} - -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { - const ggml_tensor * v = dst->src[2]; - const ggml_tensor * mask = dst->src[3]; - const ggml_tensor * sinks = dst->src[4]; - GGML_ASSERT(q->extra); - GGML_ASSERT(k->extra); - GGML_ASSERT(v->extra); - GGML_ASSERT(dst->extra); - if (mask) { - GGML_ASSERT(mask->extra); + // Intel cluster-parallel decode FA + if (fd_k_split == NULL && backend_ctx->gpu_family == INTEL && n_q == 1 && !is_causal && + is_mixed && gqa_ratio_dispatch == 4 && d_head_q == 128 && d_head_v == 128 && + n_kv >= FD_MIN_N_KV && + getenv("GGML_OPENCL_FA_C8") != NULL && getenv("GGML_OPENCL_FA_C8")[0] != '0' && + backend_ctx->fa.f32_merge.count(dk_dv) > 0) { + if (backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 128; + } else if (backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 64; + } } - if (sinks) { - GGML_ASSERT(sinks->extra); + if (fd_k_split == NULL && + n_q >= 1 && n_q <= fd_max_n_q && n_kv >= FD_MIN_N_KV && !is_causal && + d_head_q <= FD_MAX_DK && + backend_ctx->fa.f32_merge.count(dk_dv) > 0) { + if (is_mixed && backend_ctx->fa.f32_f16_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_split.at(dk_dv); + } else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_split.at(dk_dv); + } else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_split.at(dk_dv); + } } + const bool use_fd = (fd_k_split != NULL); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - cl_context context = backend_ctx->context; + const int n_q_blocks = n_q > 1 ? (n_q + block_m - 1) / block_m : 0; + const int n_kv_blocks = (n_kv > 0 && block_n > 0) ? (n_kv + block_n - 1) / block_n : 0; + // KV pad + blk prepass are pure overhead when FD will fire — skip them. + const bool use_mixed_prepass = is_mixed && n_q > 1 && !use_fd; + // make sure prepass kernels are compiled + const bool have_kv_pad = backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0; + const bool have_blk = backend_ctx->fa.blk_f16.count(dk_dv) > 0; + const bool use_kv_pad = use_mixed_prepass && (n_kv % block_n != 0) && have_kv_pad; + // blk prepass: per-KV-tile mask class (0=masked, 1=mixed, 2=unmasked). + // Consumed identically by f32_f16, q8_0 and q4_0 prefill kernels. + const bool use_quant_prepass = (use_native_q8_0 || use_native_q4_0) && !use_fd; + const bool use_blk_mask = (use_mixed_prepass || use_quant_prepass) && mask_buffer != NULL && have_blk; - const int n_q = q->ne[1]; - const int n_kv = k->ne[1]; - const int d_head_q = q->ne[0]; - const int d_head_v = v->ne[0]; - const int n_head = q->ne[2]; - const int n_head_kv = k->ne[2]; - const int n_batch = q->ne[3]; + if (use_kv_pad) { + cl_int err; - const std::pair<int, int> dk_dv = {d_head_q, d_head_v}; - cl_kernel kernel = backend_ctx->fa.kernel_flash_attn_f32_f16_bin; - GGML_ASSERT(kernel != NULL); + const size_t k_pad_size = (size_t) k_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; + temp_k_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_pad_size, NULL, &err); + CL_CHECK(err); + k_pad_buffer = temp_k_pad.data; - ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; - ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *)k->extra; - ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; - ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; - ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; + const size_t v_pad_size = (size_t) v_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; + temp_v_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_pad_size, NULL, &err); + CL_CHECK(err); + v_pad_buffer = temp_v_pad.data; - cl_ulong offset_q = extra_q->offset + q->view_offs; - cl_ulong offset_o = extra_o->offset + dst->view_offs; + cl_kernel kernel_kv_pad = backend_ctx->fa.kv_pad_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 0, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 1, sizeof(cl_ulong), &offset_k)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 2, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 3, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 4, sizeof(cl_mem), &k_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 5, sizeof(cl_mem), &v_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 6, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 7, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 8, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 9, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 10, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 11, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 12, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 13, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 14, sizeof(cl_ulong), &v_nb3)); - cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; - cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; - cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; - cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; + size_t global_work_size[] = { (size_t) block_n, (size_t) n_head_kv, (size_t) n_batch }; + backend_ctx->enqueue_ndrange_kernel(kernel_kv_pad, 3, global_work_size, NULL, dst); - const cl_ulong q_nb1 = q->nb[1]; - const cl_ulong q_nb2 = q->nb[2]; - const cl_ulong q_nb3 = q->nb[3]; + if (mask_buffer != NULL) { + mask_pad_nb1 = (cl_ulong) block_n * (cl_ulong) sizeof(ggml_fp16_t); + mask_pad_nb2 = (cl_ulong) n_q * mask_pad_nb1; + mask_pad_nb3 = (cl_ulong) mask_ne2 * mask_pad_nb2; - cl_mem k_data_device = extra_k->data_device; - cl_ulong offset_k = extra_k->offset + k->view_offs; - cl_ulong k_nb1 = k->nb[1]; - cl_ulong k_nb2 = k->nb[2]; - cl_ulong k_nb3 = k->nb[3]; + const size_t mask_pad_size = (size_t) mask_ne3 * (size_t) mask_pad_nb3; + temp_mask_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, mask_pad_size, NULL, &err); + CL_CHECK(err); + mask_pad_buffer = temp_mask_pad.data; - cl_mem v_data_device = extra_v->data_device; - cl_ulong offset_v = extra_v->offset + v->view_offs; - cl_ulong v_nb1 = v->nb[1]; - cl_ulong v_nb2 = v->nb[2]; - cl_ulong v_nb3 = v->nb[3]; + cl_kernel kernel_mask_pad = backend_ctx->fa.mask_pad_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 0, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 1, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 2, sizeof(cl_mem), &mask_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 3, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 5, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 6, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 7, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 8, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 9, sizeof(int), &mask_ne3)); - const cl_ulong o_nb1 = dst->nb[1]; - const cl_ulong o_nb2 = dst->nb[2]; - const cl_ulong o_nb3 = dst->nb[3]; + size_t global_work_size_mask[] = { (size_t) block_n, (size_t) n_q, (size_t) (mask_ne2 * mask_ne3) }; + backend_ctx->enqueue_ndrange_kernel(kernel_mask_pad, 3, global_work_size_mask, NULL, dst); + } + } - const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; - const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; - const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; - const int mask_ne2 = mask ? mask->ne[2] : 0; - const int mask_ne3 = mask ? mask->ne[3] : 0; + if (use_blk_mask) { + cl_int err; + const size_t blk_size = (size_t) n_kv_blocks * (size_t) n_q_blocks * (size_t) mask_ne2 * (size_t) mask_ne3; + temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); + if (err != CL_SUCCESS) { + // Flush before retry — reclaim deferred driver deallocations. + CL_CHECK(clFinish(backend_ctx->queue)); + temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); + } + CL_CHECK(err); + blk_buffer = temp_blk.data; - float * params = (float *)dst->op_params; - float scale = params[0]; - float max_bias = params[1]; - float logit_softcap = params[2]; + cl_kernel kernel_blk = backend_ctx->fa.blk_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_blk, 0, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel_blk, 1, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel_blk, 2, sizeof(cl_mem), &blk_buffer)); + CL_CHECK(clSetKernelArg(kernel_blk, 3, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel_blk, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_blk, 5, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel_blk, 6, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel_blk, 7, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel_blk, 8, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel_blk, 9, sizeof(int), &mask_ne3)); - const int is_causal = (mask == NULL && n_q > 1 && n_q == n_kv); // redundant n_q > 1 check ? + size_t global_work_size_blk[] = { (size_t) n_kv_blocks, (size_t) n_q_blocks, (size_t) (mask_ne2 * mask_ne3) }; + backend_ctx->enqueue_ndrange_kernel(kernel_blk, 3, global_work_size_blk, NULL, dst); + } const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); - const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + if (use_fd) { + static const int fd_env_kv_per_split = []{ + const char * e = getenv("GGML_OPENCL_FD_KV_PER_SPLIT"); + return (e && e[0]) ? atoi(e) : 0; + }(); + static const int fd_env_max_splits = []{ + const char * e = getenv("GGML_OPENCL_FD_MAX_SPLITS"); + return (e && e[0]) ? atoi(e) : 0; + }(); - ggml_cl_flash_attn_temp_buffer temp_k; - ggml_cl_flash_attn_temp_buffer temp_v; - ggml_cl_flash_attn_temp_buffer temp_k_aos; - ggml_cl_flash_attn_temp_buffer temp_v_aos; + int fd_kv_per_split = use_fd_mq ? FD_MQ_KV_PER_SPLIT + : (is_mixed ? FD_KV_PER_SPLIT_F16 : FD_KV_PER_SPLIT); + int fd_max_splits = use_fd_mq ? FD_MQ_MAX_SPLITS : FD_MAX_SPLITS; + if (fd_env_kv_per_split > 0) { fd_kv_per_split = fd_env_kv_per_split; } + if (fd_env_max_splits > 0) { fd_max_splits = fd_env_max_splits; } + int n_splits = (n_kv + fd_kv_per_split - 1) / fd_kv_per_split; + if (n_splits < FD_MIN_SPLITS) { n_splits = FD_MIN_SPLITS; } + if (n_splits > fd_max_splits) { n_splits = fd_max_splits; } + if (use_fd_mq_decode) { + const size_t wg_per_split = (size_t) n_head_kv * n_batch; + const size_t wg_target = 4 * (size_t) backend_ctx->fa.compute_units; + while (wg_per_split * n_splits < wg_target && n_splits < fd_max_splits && + n_kv / (n_splits + 1) >= 32) { + n_splits++; + } + } + const int kv_per_split = (n_kv + n_splits - 1) / n_splits; - if (is_q8_0) { - ggml_cl_flash_attn_reconstruct_aos( - backend_ctx, k, temp_k_aos, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + const int fa_partial_floats = 2 + d_head_v; + const size_t partial_size_bytes = + (size_t) n_batch * n_head * n_q * n_splits * fa_partial_floats * sizeof(float); - ggml_cl_flash_attn_reconstruct_aos( - backend_ctx, v, temp_v_aos, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + ggml_cl_flash_attn_temp_buffer temp_partial; + cl_mem partial_buffer; + cl_int err; + if (use_fd_mq_decode) { + auto & pool = backend_ctx->fa.fd_partial; + if (partial_size_bytes > pool.size) { + CL_CHECK(clFinish(backend_ctx->queue)); + pool.allocate(backend_ctx->context, partial_size_bytes); + } + partial_buffer = pool.buffer; + } else { + temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, + partial_size_bytes, NULL, &err); + if (err != CL_SUCCESS) { + CL_CHECK(clFinish(backend_ctx->queue)); + temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, + partial_size_bytes, NULL, &err); + } + CL_CHECK(err); + partial_buffer = temp_partial.data; + } - bool k_done = ggml_cl_flash_attn_dequant_kv_gpu( - backend_ctx, k, GGML_TYPE_F16, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, - temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + cl_kernel k_split = fd_k_split; + int argi = 0; + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_q)); + if (use_fa_k_img) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + + (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + + (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + + (size_t)d_head_q * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; + const size_t k_pixels = k_bytes >> 3; + cl_mem k_img = nullptr; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + k_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); + } - bool v_done = ggml_cl_flash_attn_dequant_kv_gpu( - backend_ctx, v, GGML_TYPE_F16, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, - temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + // if image creation fails, fallback to buffer based kernels + if (k_img == nullptr) { + if (gqa_ratio_dispatch == 4 && + backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { + k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); + } else { + k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); + } + use_fa_k_img = false; + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); + } else { + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_img)); + } + } else { + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); + } + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_log2_val)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &logit_softcap)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &partial_buffer)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_splits)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &kv_per_split)); - GGML_ASSERT(k_done && v_done); + // MQ split kernel uses MQ_NSG_SPLIT subgroups and one WG per (kv_head, batch, split) + // matches Q1_WG_SIZE * NSG (MQ_GQA=4 -> 256; MQ_GQA=8 -> 192) + const size_t fd_wg = use_fd_mq ? fd_mq_wg : 64; + const size_t fd_head_dim = use_fd_mq + ? (size_t)(n_head_kv * fd_head_sub * n_batch) + : (size_t)(n_head * n_batch); + size_t fd_lws[3] = { fd_wg, 1, 1 }; + // gid(2) packs q_idx * n_splits + split_idx. + size_t fd_gws[3] = { fd_wg, fd_head_dim, (size_t)(n_splits * n_q) }; + backend_ctx->enqueue_ndrange_kernel(k_split, 3, fd_gws, fd_lws, dst); + + cl_kernel k_merge = backend_ctx->fa.f32_merge.at(dk_dv); + argi = 0; + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &partial_buffer)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &extra_o->data_device)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_o)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_splits)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb3)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &sinks_buffer)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_sinks)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_q)); + + const size_t merge_wg = (size_t) (d_head_v / 4); // one lane per float4 + size_t merge_lws[3] = { merge_wg, 1, 1 }; + size_t merge_gws[3] = { merge_wg, (size_t)(n_head * n_batch), (size_t) n_q }; + backend_ctx->enqueue_ndrange_kernel(k_merge, 3, merge_gws, merge_lws, dst); + return; + } + + cl_mem prefill_k_img = nullptr; + if (use_prefill_k_img) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + + (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + + (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + + (size_t)d_head_q * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; + const size_t k_pixels = k_bytes >> 3; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + prefill_k_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); + } + if (prefill_k_img == nullptr) { + kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); + use_prefill_k_img = false; + } + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), use_prefill_k_img ? &prefill_k_img : &k_data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_k)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_o->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_o)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &is_causal)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &o_nb3)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &n_head_log2_val)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(float), &logit_softcap)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel, 36, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel, 37, sizeof(int), &mask_ne3)); + CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_mem), &sinks_buffer)); + CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &offset_sinks)); + if (n_q > 1 && is_mixed) { + CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &k_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 41, sizeof(cl_mem), &v_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 42, sizeof(cl_mem), &mask_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 43, sizeof(cl_mem), &blk_buffer)); + CL_CHECK(clSetKernelArg(kernel, 44, sizeof(int), &n_kv_blocks)); + CL_CHECK(clSetKernelArg(kernel, 45, sizeof(cl_ulong), &mask_pad_nb1)); + CL_CHECK(clSetKernelArg(kernel, 46, sizeof(cl_ulong), &mask_pad_nb2)); + CL_CHECK(clSetKernelArg(kernel, 47, sizeof(cl_ulong), &mask_pad_nb3)); + } else if (use_native_q8_0 || use_native_q4_0) { + // arg 40 = blk classification buffer (NULL disables prepass opt). + CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &blk_buffer)); } - // Allocate input/output memory buffers - cl_mem mem_matrixQ; - cl_mem mem_matrixK; - cl_mem mem_matrixV; - cl_mem mem_matrixO; - cl_buffer_region region; - cl_int err; - - region.origin = offset_q; - region.size = ggml_nbytes(q); - mem_matrixQ = clCreateSubBuffer(extra_q->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - - region.origin = offset_k; - region.size = is_q8_0 ? (size_t) k_nb3 * (size_t) k->ne[3] : ggml_nbytes(k); - mem_matrixK = clCreateSubBuffer(k_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - - region.origin = offset_v; - region.size = is_q8_0 ? (size_t) v_nb3 * (size_t) v->ne[3] : ggml_nbytes(v); - mem_matrixV = clCreateSubBuffer(v_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - - region.origin = offset_o; - region.size = ggml_nbytes(dst); - mem_matrixO = clCreateSubBuffer(extra_o->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + if (n_q == 1) { + if (use_local_tile) { + const size_t lt_wg = 128; + size_t local_work_size[] = { lt_wg, 1, 1 }; + size_t global_work_size[] = { lt_wg, (size_t) n_head, (size_t) n_batch }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + // q1_vec dispatches with NSG subgroups + const size_t q1_wg = backend_ctx->gpu_family == INTEL ? 32 : 64; + const size_t wg_size = use_q1_vec ? 256 : q1_wg; + const size_t head_dim_global = use_q1_vec_mq + ? (size_t)(n_head_kv * n_batch) + : (size_t)(n_head * n_batch); + size_t local_work_size[] = { wg_size, 1 }; + size_t global_work_size[] = { wg_size, head_dim_global }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + } + } else if (use_native_q8_0 || use_native_q4_0) { + // Native quant prefill. The split variant may override BLOCK_M + // (e.g. DK=96 quant uses BM=16). + const bool use_split = use_native_q8_0 ? use_split_q8_0 : use_split_q4_0; + int bm; + size_t wg_size; + if (use_split) { + bm = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_bm.at(dk_dv) + : backend_ctx->fa.f32_q4_0_split_bm.at(dk_dv); + wg_size = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_wg_size.at(dk_dv) + : backend_ctx->fa.f32_q4_0_split_wg_size.at(dk_dv); + } else { + bm = backend_ctx->fa.bm.at(dk_dv); + wg_size = (size_t) bm; + } + size_t local_work_size[] = { wg_size, 1 }; + size_t global_work_size[] = { (size_t)((n_q + bm - 1) / bm) * wg_size, (size_t)(n_head * n_batch) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + } else { + const size_t wg_size = (size_t) wg_size_fa; + size_t local_work_size[] = { wg_size, 1 }; + size_t global_work_size[] = { (size_t)((n_q + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + } +} - cl_image_format img_fmt_1d = { CL_RGBA, CL_FLOAT}; - cl_image_desc img_desc_1d; +static void ggml_cl_mul_mat_f16_f32_tiled(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - // use image 1d buffer used as fallback when on mask is applied - cl_mem mem_tex_mask_fallback_1dbuf; - img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = 1; - img_desc_1d.buffer = mem_matrixK; - mem_tex_mask_fallback_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); - CL_CHECK(err); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_mem mem_tex_matrixO_1dbuf; - img_fmt_1d = { CL_RGBA, CL_FLOAT}; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = ggml_nbytes(dst) / 4 / 4; - img_desc_1d.buffer = mem_matrixO; - mem_tex_matrixO_1dbuf = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); - CL_CHECK(err); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // The bin kernel requires 2d (or 3d) buffers packed for data loading/multiplication. - // These repack kernels launch across all buffers to ensure compatibility - cl_mem mem_tex_matrixMask_1dbuf = NULL; - cl_mem mem_matrixMask = NULL; - cl_mem mem_matrixMask_padded = NULL; - cl_ulong mask_nb1_padded = mask_nb1, mask_nb2_padded = mask_nb2, mask_nb3_padded = mask_nb3; - if (extra_mask) { - // allocate mem_matrixMask w/ new padded size - size_t n_kv_padded = GGML_PAD(n_kv, 4); - size_t mask_nb_padded = n_kv_padded * sizeof(cl_half) * mask->ne[1] * mask->ne[2] * mask->ne[3]; + const int M = src0->ne[1]; + const int N = src1->ne[1]; + const int K = src0->ne[0]; - // apply offset and create subBuffer for mask - region.origin = offset_mask; - region.size = ggml_nbytes(mask); - mem_matrixMask = clCreateSubBuffer(extra_mask->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_tiled; - { - // create padded mask to contain all data - mem_matrixMask_padded = clCreateBuffer(context, CL_MEM_ALLOC_HOST_PTR, mask_nb_padded, NULL, &err); - CL_CHECK(err); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); - // pass extra_mask->data_device, mem_matrixMask to kernel for copying/padding - mask_nb1_padded = (cl_ulong)n_kv_padded * sizeof(cl_half); - mask_nb2_padded = mask_nb1_padded * (cl_ulong)mask->ne[1]; - mask_nb3_padded = mask_nb2_padded * (cl_ulong)mask->ne[2]; + // Tiling parameters. These need to be tuned for optimal performance. + // They must match the #defines in the kernel mul_mat_f16_f32.cl. + // + // OPWM / OPWN: Output tile size per Work-Group. A work-group computes a tile of size OPWM x OPWN. + // TPWM / TPWN: Threads per Work-group. This is the work-group size. + // OPTM / OPTN: Output elements per Thread. Each thread computes OPTM x OPTN elements. + // + // The following relationships must hold: + // OPWM = TPWM * OPTM + // OPWN = TPWN * OPTN + // + const int OPWM = 64; + const int OPWN = 64; + const int TPWM = 16; + const int TPWN = 8; - cl_kernel repack_mask = backend_ctx->fa.kernel_repack_mask_for_wmm; - CL_CHECK(clSetKernelArg(repack_mask, 0, sizeof(cl_mem), &mem_matrixMask)); - CL_CHECK(clSetKernelArg(repack_mask, 1, sizeof(cl_ulong), &mask_nb1)); - CL_CHECK(clSetKernelArg(repack_mask, 2, sizeof(cl_ulong), &mask_nb2)); - CL_CHECK(clSetKernelArg(repack_mask, 3, sizeof(cl_ulong), &mask_nb3)); - CL_CHECK(clSetKernelArg(repack_mask, 4, sizeof(int), &mask_ne2)); - CL_CHECK(clSetKernelArg(repack_mask, 5, sizeof(cl_mem), &mem_matrixMask_padded)); - CL_CHECK(clSetKernelArg(repack_mask, 6, sizeof(cl_ulong), &mask_nb1_padded)); - CL_CHECK(clSetKernelArg(repack_mask, 7, sizeof(cl_ulong), &mask_nb2_padded)); - CL_CHECK(clSetKernelArg(repack_mask, 8, sizeof(cl_ulong), &mask_nb3_padded)); + size_t local_work_size[2] = { TPWM, TPWN }; + size_t global_work_size[2] = { + (size_t) ((M + OPWM - 1) / OPWM) * TPWM, + (size_t) ((N + OPWN - 1) / OPWN) * TPWN, + }; - size_t repack_mask_gws[3] = {(size_t)n_kv, (size_t)mask->ne[1], (size_t)mask_ne2 * (size_t)mask->ne[3]}; - backend_ctx->enqueue_ndrange_kernel(repack_mask, 3, repack_mask_gws, NULL, dst); - } + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); +} - // use image 1d buffer for matrix Mask (padded row stride) - cl_image_format img_fmt_mask_1d = { CL_RGBA, CL_HALF_FLOAT}; - cl_image_desc img_desc_mask_1d; - memset(&img_desc_mask_1d, 0, sizeof(img_desc_mask_1d)); - img_desc_mask_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_mask_1d.image_width = mask_nb_padded / 2 / 4; - img_desc_mask_1d.buffer = mem_matrixMask_padded; - mem_tex_matrixMask_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_mask_1d, &img_desc_mask_1d, NULL, &err); - CL_CHECK(err); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_can_use_adreno_xmem_gemm_f16_f32( + const ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * src0, + const ggml_tensor * src1, + const ggml_tensor * dst) { + if (!backend_ctx->adreno_xmem_gemm_enabled) { + return false; } + if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { + return false; + } + if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16) || + src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { + return false; + } + if (src0->ne[2] != 1 || src0->ne[3] != 1 || + src1->ne[2] != 1 || src1->ne[3] != 1 || + dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + const int K = src0->ne[0]; + const int M = src0->ne[1]; + const int N = src1->ne[1]; + if (src1->ne[0] != K || dst->ne[0] != M || dst->ne[1] != N) { + return false; + } + if (N <= 1 || M < 64 || N < 16 || K < 64) { + return false; + } + if ((K % 8) != 0) { + return false; + } + const int kpack = K / 4; + const int npack = CEIL_DIV(M, 4); + if (static_cast<size_t>(N) > backend_ctx->image2d_max_width || + static_cast<size_t>(kpack) > backend_ctx->image2d_max_height) { + return false; + } + if (static_cast<size_t>(N) > backend_ctx->image2d_max_width || + static_cast<size_t>(npack) > backend_ctx->image2d_max_height) { + return false; + } + return true; +} - // WMM QK uses repacked 3D images. - // Q image: rows, heads, packed depth. - cl_image_format img_fmt_3d = { CL_RGBA, CL_HALF_FLOAT }; - cl_image_desc img_desc_3d; - - memset(&img_desc_3d, 0, sizeof(img_desc_3d)); - img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; - img_desc_3d.image_width = (size_t)n_q; - img_desc_3d.image_height = (size_t)n_batch * (size_t)n_head; - img_desc_3d.image_depth = (size_t)d_head_q / 4; - cl_mem img_q_wmm = NULL; - img_q_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); - CL_CHECK(err); +static void ggml_cl_mul_mat_f16_f32_adreno_xmem( + ggml_backend_t backend, + const ggml_tensor * src0, + const ggml_tensor * src1, + ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; - { - cl_kernel repack_q = backend_ctx->fa.kernel_repack_q_for_wmm; - CL_CHECK(clSetKernelArg(repack_q, 0, sizeof(cl_mem), &mem_matrixQ)); - CL_CHECK(clSetKernelArg(repack_q, 1, sizeof(cl_ulong), &q_nb1)); - CL_CHECK(clSetKernelArg(repack_q, 2, sizeof(cl_ulong), &q_nb2)); - CL_CHECK(clSetKernelArg(repack_q, 3, sizeof(cl_ulong), &q_nb3)); - CL_CHECK(clSetKernelArg(repack_q, 4, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(repack_q, 5, sizeof(cl_mem), &img_q_wmm)); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - size_t repack_q_gws[3] = {(size_t)d_head_q / 4, (size_t)n_q, (size_t)n_batch * (size_t)n_head}; - backend_ctx->enqueue_ndrange_kernel(repack_q, 3, repack_q_gws, NULL, dst); - } + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offset1 = extra1->offset + src1->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; - // K image: columns, row groups, KV heads. - const size_t n_kv_row4 = ((size_t)n_kv + 3) / 4; + const int K = src0->ne[0]; + const int M = src0->ne[1]; + const int N = src1->ne[1]; + const int kpack = K / 4; + const int npack = CEIL_DIV(M, 4); + const int os = 8; + // Pad weights to the 32-row tiles read by the xmem kernel. + const int npack_padded = CEIL_DIV(npack, os)*os; - memset(&img_desc_3d, 0, sizeof(img_desc_3d)); - img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; - img_desc_3d.image_width = (size_t)d_head_q; - img_desc_3d.image_height = n_kv_row4; - img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; - cl_mem img_k_wmm = NULL; - img_k_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); - CL_CHECK(err); + const size_t xmem_bytes = 6144; + const size_t weight_bytes = static_cast<size_t>(kpack) * static_cast<size_t>(npack_padded) * 4u * sizeof(cl_half4); - { - cl_kernel repack_k = backend_ctx->fa.kernel_repack_k_for_wmm; - CL_CHECK(clSetKernelArg(repack_k, 0, sizeof(cl_mem), &mem_matrixK)); - CL_CHECK(clSetKernelArg(repack_k, 1, sizeof(cl_ulong), &k_nb1)); - CL_CHECK(clSetKernelArg(repack_k, 2, sizeof(cl_ulong), &k_nb2)); - CL_CHECK(clSetKernelArg(repack_k, 3, sizeof(cl_ulong), &k_nb3)); - CL_CHECK(clSetKernelArg(repack_k, 4, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(repack_k, 5, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(repack_k, 6, sizeof(cl_mem), &img_k_wmm)); + backend_ctx->prealloc_adreno_xmem_const.allocate(backend_ctx->context, xmem_bytes); - size_t repack_k_gws[3] = {(size_t)d_head_q, n_kv_row4, (size_t)n_batch * (size_t)n_head_kv}; - backend_ctx->enqueue_ndrange_kernel(repack_k, 3, repack_k_gws, NULL, dst); - } + cl_int err = CL_SUCCESS; + cl_image_format fmt = {}; + fmt.image_channel_order = CL_RGBA; + fmt.image_channel_data_type = CL_HALF_FLOAT; - // V image: kv-rows (contracted), packed head-dim groups, KV heads. - memset(&img_desc_3d, 0, sizeof(img_desc_3d)); - img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; - img_desc_3d.image_width = (size_t)n_kv; - img_desc_3d.image_height = (size_t)d_head_v / 4; - img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; - cl_mem img_v_wmm = NULL; - img_v_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + cl_image_desc desc_src = {}; + desc_src.image_type = CL_MEM_OBJECT_IMAGE2D; + desc_src.image_width = static_cast<size_t>(N); + desc_src.image_height = static_cast<size_t>(kpack); + cl_mem src_img = clCreateImage(backend_ctx->context, CL_MEM_READ_WRITE, &fmt, &desc_src, nullptr, &err); CL_CHECK(err); - { - cl_kernel repack_v = backend_ctx->fa.kernel_repack_v_for_wmm; - CL_CHECK(clSetKernelArg(repack_v, 0, sizeof(cl_mem), &mem_matrixV)); - CL_CHECK(clSetKernelArg(repack_v, 1, sizeof(cl_ulong), &v_nb1)); - CL_CHECK(clSetKernelArg(repack_v, 2, sizeof(cl_ulong), &v_nb2)); - CL_CHECK(clSetKernelArg(repack_v, 3, sizeof(cl_ulong), &v_nb3)); - CL_CHECK(clSetKernelArg(repack_v, 4, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(repack_v, 5, sizeof(cl_mem), &img_v_wmm)); + cl_image_desc desc_dst = {}; + desc_dst.image_type = CL_MEM_OBJECT_IMAGE2D; + desc_dst.image_width = static_cast<size_t>(N); + desc_dst.image_height = static_cast<size_t>(npack); + cl_mem dst_img = clCreateImage(backend_ctx->context, CL_MEM_READ_WRITE, &fmt, &desc_dst, nullptr, &err); + CL_CHECK(err); - size_t repack_v_gws[3] = {(size_t)d_head_v / 4, (size_t)n_kv, (size_t)n_batch * (size_t)n_head_kv}; - backend_ctx->enqueue_ndrange_kernel(repack_v, 3, repack_v_gws, NULL, dst); - } + cl_mem weights = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, weight_bytes, nullptr, &err); + CL_CHECK(err); - cl_int enable_mask = (extra_mask) ? 1 : 0; - mask_buffer = extra_mask ? mem_tex_matrixMask_1dbuf : mem_tex_mask_fallback_1dbuf; + cl_kernel prepack = backend_ctx->kernel_adreno_xmem_prepack_weight_f16; + CL_CHECK(clSetKernelArg(prepack, 0, sizeof(cl_mem), &weights)); + CL_CHECK(clSetKernelArg(prepack, 1, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(prepack, 2, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(prepack, 3, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(prepack, 4, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(prepack, 5, sizeof(int), &kpack)); + CL_CHECK(clSetKernelArg(prepack, 6, sizeof(int), &npack_padded)); + CL_CHECK(clSetKernelArg(prepack, 7, sizeof(int), &os)); + size_t lws = 256; + size_t max_wg = backend_ctx->get_kernel_workgroup_size(prepack); + if (lws > max_wg) { + lws = max_wg; + } + size_t gws = CEIL_DIV(static_cast<size_t>(kpack) * static_cast<size_t>(npack_padded), lws) * lws; + backend_ctx->enqueue_ndrange_kernel(prepack, 1, &gws, &lws, dst); - cl_mem mem_sinksBuf = NULL; - cl_mem mem_tex_sinks_1dbuf = NULL; - cl_int enable_sinks = (sinks_buffer != NULL) ? 1 : 0; - if (enable_sinks) { - region.origin = offset_sinks; - region.size = ggml_nbytes(sinks); - mem_sinksBuf = clCreateSubBuffer(extra_sinks->data_device, CL_MEM_READ_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + cl_kernel pack_src = backend_ctx->kernel_adreno_xmem_pack_src_f32; + CL_CHECK(clSetKernelArg(pack_src, 0, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(pack_src, 1, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(pack_src, 2, sizeof(cl_mem), &src_img)); + CL_CHECK(clSetKernelArg(pack_src, 3, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(pack_src, 4, sizeof(int), &N)); + size_t pack_src_lws[2] = { 16, 16 }; + size_t pack_src_gws[2] = { + CEIL_DIV(static_cast<size_t>(N), pack_src_lws[0])*pack_src_lws[0], + CEIL_DIV(static_cast<size_t>(kpack), pack_src_lws[1])*pack_src_lws[1] + }; + backend_ctx->enqueue_ndrange_kernel(pack_src, 2, pack_src_gws, pack_src_lws, dst); - cl_image_format img_fmt_sinks_1d = { CL_R, CL_FLOAT }; - cl_image_desc img_desc_sinks_1d; - memset(&img_desc_sinks_1d, 0, sizeof(img_desc_sinks_1d)); - img_desc_sinks_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_sinks_1d.image_width = (size_t)n_head; - img_desc_sinks_1d.buffer = mem_sinksBuf; - mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_1d, &img_desc_sinks_1d, NULL, &err); - CL_CHECK(err); - } else { - // The image obj cannot be null so we back with buffer of size 1 and use matrixK to back because it always exists - cl_image_format img_fmt_sinks_fallback = { CL_R, CL_FLOAT }; - cl_image_desc img_desc_sinks_fallback; - memset(&img_desc_sinks_fallback, 0, sizeof(img_desc_sinks_fallback)); - img_desc_sinks_fallback.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_sinks_fallback.image_width = 1; - img_desc_sinks_fallback.buffer = mem_matrixK; - mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_fallback, &img_desc_sinks_fallback, NULL, &err); - CL_CHECK(err); - } + cl_kernel gemm = backend_ctx->kernel_gemm_xmem_f16_f32_os8; + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &weights)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &backend_ctx->prealloc_adreno_xmem_const.buffer)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &src_img)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &dst_img)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(int), &npack)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(int), &kpack)); + const size_t z_values = CEIL_DIV(static_cast<size_t>(npack), static_cast<size_t>(os)); + size_t gemm_lws[3] = { 64, 1, 1 }; + size_t gemm_gws[3] = { + z_values*gemm_lws[0], + CEIL_DIV(static_cast<size_t>(N), gemm_lws[0]), + 1 + }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, gemm_gws, gemm_lws, dst); - cl_uint arg = 0; + cl_kernel store_dst = backend_ctx->kernel_adreno_xmem_store_dst_f32; + CL_CHECK(clSetKernelArg(store_dst, 0, sizeof(cl_mem), &dst_img)); + CL_CHECK(clSetKernelArg(store_dst, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(store_dst, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(store_dst, 3, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(store_dst, 4, sizeof(int), &N)); + size_t store_lws[2] = { 16, 16 }; + size_t store_gws[2] = { + CEIL_DIV(static_cast<size_t>(N), store_lws[0])*store_lws[0], + CEIL_DIV(static_cast<size_t>(npack), store_lws[1])*store_lws[1] + }; + backend_ctx->enqueue_ndrange_kernel(store_dst, 2, store_gws, store_lws, dst); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_matrixO_1dbuf)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &is_causal)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb1)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb2)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb3)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb1)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb2)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb3)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb1)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb2)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb3)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb1)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb2)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb3)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &max_bias)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m0)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m1)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_log2_val)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &logit_softcap)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mask_buffer)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_mask)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb1_padded)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb2_padded)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb3_padded)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne2)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne3)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_sinks_1dbuf)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_sinks)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_q_wmm)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_k_wmm)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_v_wmm)); - CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &d_head_q)); + CL_CHECK(clReleaseMemObject(weights)); + CL_CHECK(clReleaseMemObject(dst_img)); + CL_CHECK(clReleaseMemObject(src_img)); +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - size_t global_work_size[3], local_work_size[3]; +static void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const int n_waves_v = d_head_q / 64; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - local_work_size[0] = 64; - local_work_size[1] = n_waves_v; - local_work_size[2] = 1; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - global_work_size[0] = 64; - global_work_size[1] = ((n_q + 64 - 1) / 64) * n_waves_v; - global_work_size[2] = n_batch * n_head; + const cl_uint Cout = ne03; + const cl_uint Cin = ne02; + const cl_uint N = ne13; + const cl_uint KW = ne00; + const cl_uint KH = ne01; + const cl_uint W = ne10; + const cl_uint H = ne11; + const cl_uint OW = ne0; + const cl_uint OH = ne1; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + const cl_uint s0 = dst->op_params[0]; + const cl_uint s1 = dst->op_params[1]; + const cl_uint p0 = dst->op_params[2]; + const cl_uint p1 = dst->op_params[3]; + const cl_uint d0 = dst->op_params[4]; + const cl_uint d1 = dst->op_params[5]; - CL_CHECK(clReleaseMemObject(mem_tex_matrixO_1dbuf)); - CL_CHECK(clReleaseMemObject(img_q_wmm)); - CL_CHECK(clReleaseMemObject(img_k_wmm)); - CL_CHECK(clReleaseMemObject(img_v_wmm)); + const cl_uint cl_nb00 = nb00/ggml_type_size(src0->type); + const cl_uint cl_nb01 = nb01/ggml_type_size(src0->type); + const cl_uint cl_nb02 = nb02/ggml_type_size(src0->type); + const cl_uint cl_nb03 = nb03/ggml_type_size(src0->type); + const cl_uint cl_nb10 = nb10/ggml_type_size(src1->type); + const cl_uint cl_nb11 = nb11/ggml_type_size(src1->type); + const cl_uint cl_nb12 = nb12/ggml_type_size(src1->type); + const cl_uint cl_nb13 = nb13/ggml_type_size(src1->type); + const cl_uint cl_nb1 = nb1/ggml_type_size(dst->type); + const cl_uint cl_nb2 = nb2/ggml_type_size(dst->type); + const cl_uint cl_nb3 = nb3/ggml_type_size(dst->type); - if (mem_tex_matrixMask_1dbuf) { - CL_CHECK(clReleaseMemObject(mem_tex_matrixMask_1dbuf)); - } - if (mem_matrixMask) { - CL_CHECK(clReleaseMemObject(mem_matrixMask)); - } - if (mem_matrixMask_padded) { - CL_CHECK(clReleaseMemObject(mem_matrixMask_padded)); - } - if (mem_tex_sinks_1dbuf) { - CL_CHECK(clReleaseMemObject(mem_tex_sinks_1dbuf)); - } - if (mem_sinksBuf) { - CL_CHECK(clReleaseMemObject(mem_sinksBuf)); - } - CL_CHECK(clReleaseMemObject(mem_matrixQ)); - CL_CHECK(clReleaseMemObject(mem_matrixK)); - CL_CHECK(clReleaseMemObject(mem_matrixV)); - CL_CHECK(clReleaseMemObject(mem_matrixO)); -} -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + const int64_t NPQ = (int64_t)N * OW * OH; -static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { - const ggml_tensor * v = dst->src[2]; - const ggml_tensor * mask = dst->src[3]; - const ggml_tensor * sinks = dst->src[4]; + const uint32_t BS_K = 64; + const uint32_t BS_NPQ = 64; + const uint32_t BS_CRS = 16; + const uint32_t VEC_SIZE = 4; - GGML_ASSERT(q->extra); - GGML_ASSERT(k->extra); - GGML_ASSERT(v->extra); - GGML_ASSERT(dst->extra); + const uint32_t TS_K = 4; + const uint32_t TS_NPQ = 8; - if (mask) { - GGML_ASSERT(mask->extra); - } - if (sinks) { - GGML_ASSERT(sinks->extra); - } + const uint32_t WG_K = BS_K / TS_K; + const uint32_t WG_NPQ = BS_NPQ / TS_NPQ; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + auto splitWork = [](uint32_t work_size, uint32_t block_size) { return (block_size + work_size - 1) / block_size; }; + const uint32_t NB_K = splitWork(Cout, BS_K); + const uint32_t NB_NPQ = splitWork(NPQ, BS_NPQ); - const int n_q = q->ne[1]; - const int n_kv = k->ne[1]; - const int d_head_q = q->ne[0]; - const int d_head_v = v->ne[0]; - const int n_head = q->ne[2]; - const int n_head_kv = k->ne[2]; - const int n_batch = q->ne[3]; + cl_kernel kernel; + size_t shmem_size; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (ggml_cl_adreno_xmem_attn_can_use(backend_ctx, q, k, dst)) { - ggml_cl_adreno_xmem_attn_run(backend, q, k, dst); - return; + if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { + kernel = backend_ctx->kernel_conv_2d_f16; + shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_half4)); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_conv_2d_f32; + shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_float) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_conv_2d_f16_f32; + shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); + } else { + GGML_ASSERT(false && "Unsupported data type combination for conv2d"); } -#endif - // DK=512 (Gemma-4 global layers) runs decode-only (q1 / q1_split) on - // Adreno - it never uses the BM-tile path, and the prepass + split-tile - // programs OOM the compiler at DK=512; supports_op only admits - // n_q==1 here and prefill goes to CPU - const bool fa_decode_only_512 = (d_head_q == 512); + cl_uint idx = 0; + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, idx++, shmem_size, NULL)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cout)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cin)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &N)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &W)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb00)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb01)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb02)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb03)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb10)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb11)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb12)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb13)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb2)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb3)); - // per-variant lazy compile for this (dk, dv) - // DK=512 decode (n_q==1) needs no prepass - // DK=512 prefill (n_q>1) does, so compile it only when needed - if (!fa_decode_only_512 || n_q > 1) { - ggml_opencl_ensure_fa_pre_kernels(backend_ctx, d_head_q, d_head_v); - } + size_t global_work_size[] = { (size_t)NB_K * WG_K, (size_t)NB_NPQ * WG_NPQ, 1 }; + size_t local_work_size[] = { (size_t)WG_K, (size_t)WG_NPQ, 1 }; - cl_kernel kernel = NULL; - bool use_prefill_k_img = false; // K is image1d_buffer_t for DK=512 prefill + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); +} - const bool is_f16 = q->type == GGML_TYPE_F16; - const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; - const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; - const bool is_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && v->type == GGML_TYPE_Q4_0; +static void ggml_cl_conv2_dw(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_fa_bin_kernels_prefill(backend_ctx, q, k, v)) { - // We support the prefill path of flash attn with a specialized d_head = 64/128/256 - ggml_cl_flash_attn_prefill_bin(backend, q, k, dst); - return; - } -#endif + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; - if (is_f16) { - ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F16); - } else if (is_mixed) { - ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16); - if (fa_decode_only_512) { - // DK=512: the BM-tile prefill kernels are specifically compiled from - // FA_PREFILL_ONLY - if (n_q > 1) { - ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false); - ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/true); - } - } else { - ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16_SPLIT); - } - } else if (is_q8_0) { - ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0); - if (d_head_q == 96 && d_head_v == 96) { - ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/true); - } else if (d_head_q == 256 && d_head_v == 256) { - ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/true); - } else { - ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0_SPLIT); - } - } else if (is_q4_0) { - ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0); - if (d_head_q == 96 && d_head_v == 96) { - ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/false); - } else if (d_head_q == 256 && d_head_v == 256) { - ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/false); - } else { - ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0_SPLIT); - } + const cl_long offset0 = extra0->offset + src0->view_offs; + const cl_long offset1 = extra1->offset + src1->view_offs; + const cl_long offsetd = extrad->offset + dst->view_offs; + + const cl_long C = ne12; + const cl_long N = ne13; + const cl_long KW = ne00; + const cl_long KH = ne01; + const cl_long W = ne10; + const cl_long H = ne11; + const cl_long OW = ne0; + const cl_long OH = ne1; + + const cl_int s0 = dst->op_params[0]; + const cl_int s1 = dst->op_params[1]; + const cl_int p0 = dst->op_params[2]; + const cl_int p1 = dst->op_params[3]; + const cl_int d0 = dst->op_params[4]; + const cl_int d1 = dst->op_params[5]; + + const cl_long cl_nb01 = nb01/ggml_type_size(src0->type); + const cl_long cl_nb03 = nb03/ggml_type_size(src0->type); + const cl_long cl_nb11 = nb11/ggml_type_size(src1->type); + const cl_long cl_nb12 = nb12/ggml_type_size(src1->type); + const cl_long cl_nb13 = nb13/ggml_type_size(src1->type); + const cl_long cl_nb1 = nb1/ggml_type_size(dst->type); + const cl_long cl_nb2 = nb2/ggml_type_size(dst->type); + const cl_long cl_nb3 = nb3/ggml_type_size(dst->type); + + cl_kernel kernel; + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_conv2d_dw_f32_f32; + } else if (src0->type == GGML_TYPE_F16) { + kernel = backend_ctx->kernel_conv2d_dwf16_f32; } else { - ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32); + GGML_ASSERT(false && "Unsupported data type combination for conv2_dw"); } - const std::pair<int, int> dk_dv = {d_head_q, d_head_v}; - const bool use_native_q8_0_q1 = is_q8_0 && n_q == 1 && - backend_ctx->fa.f32_q8_0_q1.count(dk_dv) > 0; - // Native q8_0 prefill — reads q8_0 directly, wg_size = cfg->bm. - const bool use_native_q8_0 = is_q8_0 && n_q > 1 && - backend_ctx->fa.f32_q8_0.count(dk_dv) > 0; - const bool use_native_q4_0_q1 = is_q4_0 && n_q == 1 && - backend_ctx->fa.f32_q4_0_q1.count(dk_dv) > 0; - const bool use_native_q4_0 = is_q4_0 && n_q > 1 && - backend_ctx->fa.f32_q4_0.count(dk_dv) > 0; - const int block_m = n_q > 1 - ? (is_mixed ? backend_ctx->fa.f32_f16_bm.at(dk_dv) : backend_ctx->fa.bm.at(dk_dv)) - : 0; - // block_n is only used by the n_q > 1 prefill path; its map is not - // populated for DK=512 decode, so do not read it for decode. - const int block_n = (n_q > 1) - ? (is_mixed ? backend_ctx->fa.f32_f16_bn.at(dk_dv) - : backend_ctx->fa.bn.at(dk_dv)) - : 0; - // Pick split variant only when n_kv crosses the per-(dk,dv) threshold. - // the N_SPLIT>1 prefill tile reduces DK partials via subgroup shuffle, - // on Intel it uses the non-split BM tile and does not depend on subgroup size - const bool use_split_kernel = (n_q > 1 && is_mixed && - backend_ctx->gpu_family != INTEL && - backend_ctx->fa.f32_f16_split.count(dk_dv) > 0 && - n_kv >= backend_ctx->fa.f32_f16_split_nkv_threshold.at(dk_dv)); - const bool use_split_q8_0 = (use_native_q8_0 && backend_ctx->gpu_family != INTEL && - backend_ctx->fa.f32_q8_0_split.count(dk_dv) > 0 && - n_kv >= backend_ctx->fa.f32_q8_0_split_nkv_threshold.at(dk_dv)); - const bool use_split_q4_0 = (use_native_q4_0 && backend_ctx->gpu_family != INTEL && - backend_ctx->fa.f32_q4_0_split.count(dk_dv) > 0 && - n_kv >= backend_ctx->fa.f32_q4_0_split_nkv_threshold.at(dk_dv)); - const int wg_size_fa = (n_q > 1 && is_mixed) - ? (use_split_kernel - ? backend_ctx->fa.f32_f16_split_wg_size.at(dk_dv) - : backend_ctx->fa.f32_f16_wg_size.at(dk_dv)) - : block_m; + cl_uint idx = 0; + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offset0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offset1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &C)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &N)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &KW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &KH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &W)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &H)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &OW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &OH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &s0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &s1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &p0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &p1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &d0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &d1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb01)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb03)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb11)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb12)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb13)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb2)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb3)); + + const size_t total = (size_t) N * C * OH * OW; + const size_t local_size = MIN((size_t) 256, backend_ctx->get_kernel_workgroup_size(kernel)); + size_t global_work_size[] = { CEIL_DIV(total, local_size) * local_size }; + size_t local_work_size[] = { local_size }; - ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; - ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; - ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); +} - // SoA q8_0/q4_0 K/V: data_device aliases the `q` subbuffer; reconstruct - // AoS into a temp buffer below. AoS tensors use extra_k/v->data_device. - const bool k_soa = ggml_cl_is_q8_0_soa(k) || ggml_cl_is_q4_0_soa(k); - const bool v_soa = ggml_cl_is_q8_0_soa(v) || ggml_cl_is_q4_0_soa(v); - ggml_tensor_extra_cl * extra_k = k_soa ? nullptr : (ggml_tensor_extra_cl *)k->extra; - ggml_tensor_extra_cl * extra_v = v_soa ? nullptr : (ggml_tensor_extra_cl *)v->extra; +// is_kq selects which of the two products this call is, and it is decided by the +// CALLER -- the two admission arms in ggml_cl_mul_mat, each of which knows which +// one it matched. It used to be re-derived here from nb01 > nb02, i.e. "K is +// head-major, V^T is not". That discriminator COLLAPSES at n_head_kv == 1, where +// the two strides are equal because there is only one head to order, so nothing +// here could tell a KQ from a KQV. Pass it in rather than infer it. +static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool is_kq) { + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - cl_ulong offset_q = extra_q->offset + q->view_offs; - cl_ulong offset_k = k_soa ? 0 : extra_k->offset + k->view_offs; - cl_ulong offset_v = v_soa ? 0 : extra_v->offset + v->view_offs; - cl_ulong offset_o = extra_o->offset + dst->view_offs; - cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; - cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; - cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; - cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - const cl_ulong q_nb1 = q->nb[1]; - const cl_ulong q_nb2 = q->nb[2]; - const cl_ulong q_nb3 = q->nb[3]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; - cl_ulong k_nb1 = k->nb[1]; - cl_ulong k_nb2 = k->nb[2]; - cl_ulong k_nb3 = k->nb[3]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + + const cl_ulong nb10 = src1->nb[0]; - cl_ulong v_nb1 = v->nb[1]; - cl_ulong v_nb2 = v->nb[2]; - cl_ulong v_nb3 = v->nb[3]; + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; - const cl_ulong o_nb1 = dst->nb[1]; - const cl_ulong o_nb2 = dst->nb[2]; - const cl_ulong o_nb3 = dst->nb[3]; + GGML_ASSERT(ne00 == ne10); - const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; - const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; - const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; - const int mask_ne2 = mask ? mask->ne[2] : 0; - const int mask_ne3 = mask ? mask->ne[3] : 0; + cl_kernel kernel; + cl_context context = backend_ctx->context; - float scale; - float max_bias; - float logit_softcap; + cl_int status; + cl_image_format img_fmt_1d; + cl_image_desc img_desc_1d; + cl_buffer_region region; + cl_mem A_image1d; + cl_mem A_sub_buffer; + cl_mem B_sub_buffer; + cl_mem D_image1d; + cl_mem D_sub_buffer; - const float * params = (const float *)dst->op_params; - scale = params[0]; - max_bias = params[1]; - logit_softcap = params[2]; + int M = ne01; + int N = ne1; + int K = ne00; - bool use_q1_vec = false; - bool use_q1_vec_mq = false; - bool use_local_tile = false; - // KV-head-coalesced gate: gqa_ratio == compile-time MQ_GQA - // restricts to DK=DV=256 for now due to local memory size - const int gqa_ratio_dispatch = n_head_kv > 0 ? (n_head / n_head_kv) : 0; - if (n_q == 1) { - if (use_native_q8_0_q1) { - if (d_head_v >= 256 && - backend_ctx->fa.f32_q8_0_q1_vec.count(dk_dv) > 0) { - kernel = backend_ctx->fa.f32_q8_0_q1_vec.at(dk_dv); - use_q1_vec = true; - } else { - kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); - } - } else if (use_native_q4_0_q1) { - // q4_0 vec kernel uses per-lane dp4a (cl_khr_integer_dot_product) - const char * q4vec_env = getenv("GGML_OPENCL_FA_Q4_VEC"); - const bool q4vec_off = (q4vec_env != NULL) && (q4vec_env[0] == '0'); - if (!q4vec_off && d_head_v >= 256 && - backend_ctx->fa.f32_q4_0_q1_vec.count(dk_dv) > 0) { - kernel = backend_ctx->fa.f32_q4_0_q1_vec.at(dk_dv); - use_q1_vec = true; - } else { - kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); - } - } else if (is_mixed) { - static const char * lt_env = getenv("GGML_OPENCL_FA_LOCAL_TILE"); - static const bool lt_on = (lt_env != NULL) && (lt_env[0] != '0'); - if (lt_on && d_head_q == 128 && d_head_v == 128 && - backend_ctx->fa.f32_f16_q1_local_tile.count(dk_dv) > 0) { - kernel = backend_ctx->fa.f32_f16_q1_local_tile.at(dk_dv); - use_local_tile = true; - } else { - static const char * f16_vec_dk128_env = getenv("GGML_OPENCL_FA_F16_VEC_DK128"); - static const bool f16_vec_dk128_off = (f16_vec_dk128_env != NULL) && (f16_vec_dk128_env[0] == '0'); - const int dv_gate = f16_vec_dk128_off ? 256 : 128; - if (d_head_v >= dv_gate && - backend_ctx->fa.f32_f16_q1_vec.count(dk_dv) > 0) { - kernel = backend_ctx->fa.f32_f16_q1_vec.at(dk_dv); - use_q1_vec = true; - } else { - kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); - } - } - } else if (is_f16) { - kernel = backend_ctx->fa.f16_q1.at(dk_dv); - } else { - kernel = backend_ctx->fa.f32_q1.at(dk_dv); - } + kernel = is_kq ? backend_ctx->kernel_mul_mm_f16_f32_kq + : backend_ctx->kernel_mul_mm_f16_f32_kqv; + // create sub-buffer for A + // <--------------------------------------------> // + extra0 = src0->view_src ? (ggml_tensor_extra_cl *)src0->view_src->extra : (ggml_tensor_extra_cl *)src0->extra; + + region.origin = (extra0->offset + src0->view_offs); + if (is_kq) { + // KQ + region.size = nb01 * ne01; } else { - if (use_native_q8_0) { - kernel = use_split_q8_0 - ? backend_ctx->fa.f32_q8_0_split.at(dk_dv) - : backend_ctx->fa.f32_q8_0.at(dk_dv); - } else if (use_native_q4_0) { - kernel = use_split_q4_0 - ? backend_ctx->fa.f32_q4_0_split.at(dk_dv) - : backend_ctx->fa.f32_q4_0.at(dk_dv); - } else if (is_mixed) { - if (use_split_kernel) { - // DK=512 prefill: opt-in texture-cache K reads (image1d_buffer_t). - static const char * pkimg_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); - const bool pkimg_on = (pkimg_env != NULL) && (pkimg_env[0] != '0'); - if (d_head_q == 512 && pkimg_on && - backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) > 0) { - kernel = backend_ctx->fa.f32_f16_split_k_img.at(dk_dv); - use_prefill_k_img = true; - } else { - kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); - } - } else { - kernel = backend_ctx->fa.f32_f16.at(dk_dv); - } - } else if (is_f16) { - kernel = backend_ctx->fa.f16.at(dk_dv); - } else { - kernel = backend_ctx->fa.f32.at(dk_dv); - } + // KQV + region.size = nb02 * ne02; } - // Intel goes to the basic q1 kernel - if (backend_ctx->gpu_family == INTEL && n_q == 1) { - use_q1_vec = use_q1_vec_mq = use_local_tile = false; - if (is_mixed && backend_ctx->fa.f32_f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); } - else if (is_f16 && backend_ctx->fa.f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f16_q1.at(dk_dv); } - else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); } - else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); } - else if (backend_ctx->fa.f32_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q1.at(dk_dv); } + A_sub_buffer = clCreateSubBuffer((extra0->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // <--------------------------------------------> // + + // create sub-buffer for B + // <--------------------------------------------> // + region.origin = (extra1->offset + src1->view_offs); + region.size = nb10 * ne10 * ne11 * ne12; + B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + // <--------------------------------------------> // + + img_fmt_1d = {CL_RGBA, CL_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + if (is_kq) { + img_desc_1d.image_width = (nb01 * ne01 / 4)/4; } - GGML_ASSERT(kernel != NULL); + else { + img_desc_1d.image_width = (nb02 * ne02 / 4)/4; + } + img_desc_1d.buffer = A_sub_buffer; + A_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); + CL_CHECK(status); - ggml_cl_flash_attn_temp_buffer temp_k; - ggml_cl_flash_attn_temp_buffer temp_v; - ggml_cl_flash_attn_temp_buffer temp_k_pad; - ggml_cl_flash_attn_temp_buffer temp_v_pad; - ggml_cl_flash_attn_temp_buffer temp_mask_pad; - ggml_cl_flash_attn_temp_buffer temp_blk; - const ggml_type kv_target_type = is_f16 ? GGML_TYPE_F16 : GGML_TYPE_F32; + // create sub-buffer for output C + // <--------------------------------------------> // + region.origin = (extrad->offset + dst->view_offs); + region.size = ne0 * ne1 * dst->ne[2] * dst->nb[0]; // size of C in bytes + D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + // <--------------------------------------------> // - cl_mem k_data_device = k_soa ? NULL : extra_k->data_device; - cl_mem v_data_device = v_soa ? NULL : extra_v->data_device; + // create image for C output + // <--------------------------------------------> // + img_fmt_1d = {CL_R, CL_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_1d.image_width = ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4; + img_desc_1d.buffer = D_sub_buffer; + D_image1d = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); + CL_CHECK(status); + // <--------------------------------------------> // - // SoA q8_0/q4_0 -> reconstruct AoS for downstream kernels that expect - // tight records (no-op when k/v is already AoS). - ggml_cl_flash_attn_temp_buffer temp_k_aos; - ggml_cl_flash_attn_temp_buffer temp_v_aos; - ggml_cl_flash_attn_reconstruct_aos(backend_ctx, k, temp_k_aos, - k_data_device, offset_k, k_nb1, k_nb2, k_nb3); - ggml_cl_flash_attn_reconstruct_aos(backend_ctx, v, temp_v_aos, - v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + int offset_src0 = 0; + int offset_src1 = 0; - // currently FA kernels support KV cache with f16, f32, q4_0 and q8_0. - // there two cases that these kernels cannot cover, - // 1. KV cache types are q4_0 or q8_0, but the FA kernels fail to compile - // 2. KV cache types not currently supported by an FA kernel, e.g., q4_1 - // these two cases are supported here by dequantizing to f32/f16 and this - // causes performance degradation. - // For q4_0 or q8_0 cases that fail kernel compilation, dequant happens in GPU; - // for types that do not have FA kernels, dequant happens on host. - if (!use_native_q8_0_q1 && !use_native_q8_0 && - !use_native_q4_0_q1 && !use_native_q4_0) { - // for q4_0, q8_0 FA kernels that fail to compile - bool k_done = false; - bool v_done = false; - if (k->type == GGML_TYPE_Q8_0 || k->type == GGML_TYPE_Q4_0) { - k_done = ggml_cl_flash_attn_dequant_kv_gpu( - backend_ctx, k, kv_target_type, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, - temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); - } - if (v->type == GGML_TYPE_Q8_0 || v->type == GGML_TYPE_Q4_0) { - v_done = ggml_cl_flash_attn_dequant_kv_gpu( - backend_ctx, v, kv_target_type, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, - temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); - } - if (!k_done) { - ggml_cl_flash_attn_prepare_quantized_tensor( - backend_ctx, k, kv_target_type, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); - } - if (!v_done) { - ggml_cl_flash_attn_prepare_quantized_tensor( - backend_ctx, v, kv_target_type, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); - } - // Asymmetric KV on the F32 fallback path: convert the F16 side to F32 - // too. (Symmetric F16 / mixed paths handle F16 directly.) - if (kv_target_type == GGML_TYPE_F32 && !is_mixed && !is_f16) { - ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, k, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); - ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, v, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); - } - } + // set kernel args + // <--------------------------------------------> // + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_sub_buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &D_image1d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &nb01)); + + size_t global_work_size[3] = {64, static_cast<size_t>(((M+63)/64)), static_cast<size_t>(((N+31)/32)*ne12)}; + size_t local_work_size[3] = {64, 1, 2}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + // deallocate sub buffers and images + // <--------------------------------------------> // + CL_CHECK(clReleaseMemObject(A_image1d)); + CL_CHECK(clReleaseMemObject(D_image1d)); + CL_CHECK(clReleaseMemObject(A_sub_buffer)); + CL_CHECK(clReleaseMemObject(B_sub_buffer)); + CL_CHECK(clReleaseMemObject(D_sub_buffer)); +} - cl_mem k_pad_buffer = NULL; - cl_mem v_pad_buffer = NULL; - cl_mem mask_pad_buffer = NULL; - cl_mem blk_buffer = NULL; - cl_ulong mask_pad_nb1 = 0; - cl_ulong mask_pad_nb2 = 0; - cl_ulong mask_pad_nb3 = 0; +static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - // Flash-Decoding K-split decision. Resolved here, before the prefill - // prepass, because KV-pad and blk prepass are pure overhead when FD fires. - // Do not infer causality from tensor shapes: a NULL mask means full - // (bidirectional) attention, e.g. ViT encoders, where n_q == n_kv as well. - // Causal attention in llama.cpp always comes with an explicit KQ mask. - // Inferring is_causal here corrupted mmproj output on OpenCL (see #23800). - const int is_causal = 0; - const int fd_max_n_q = (d_head_q <= FD_MAX_DK_MULTI) ? FD_MAX_N_Q_MULTI : 1; - cl_kernel fd_k_split = NULL; - bool use_fd_mq = false; - size_t fd_mq_wg = 256; // MQ_GQA=4 kernel: Q1_WG_SIZE(64) * MQ_NSG_SPLIT(4) - bool use_fa_k_img = false; // K bound as image1d_buffer_t instead of (buf, offset) + GGML_ASSERT(src0->type == GGML_TYPE_Q1_0); + GGML_ASSERT(src1->type == GGML_TYPE_F32); - { - const char * mq_env = getenv("GGML_OPENCL_FA_MQ"); - const bool mq_enabled = (mq_env == NULL) ? true : (mq_env[0] != '0'); - const bool mq_kv_ok = is_mixed || is_q8_0 || is_q4_0; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const char * lmq_env = getenv("GGML_OPENCL_FA_LOCAL_MQ_SPLIT"); - const bool lmq_on = (lmq_env != NULL) && (lmq_env[0] != '0'); + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; - static const char * vec_nq_env = getenv("GGML_OPENCL_FA_VEC_NQ"); - static const int N_MAX_VEC_NQ = (vec_nq_env != NULL && vec_nq_env[0] != '\0') - ? atoi(vec_nq_env) : 1; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - const bool nq_in_vec_range = (n_q >= 1) && (n_q <= N_MAX_VEC_NQ); - const bool nq1_only = (n_q == 1); + GGML_ASSERT(src1->view_offs == 0); + GGML_ASSERT(dst->view_offs == 0); - // Cluster-parallel decode default on for Adreno X2E/X1E - static const int c8_env_state = []{ - const char * e = getenv("GGML_OPENCL_FA_C8"); - if (e == NULL || e[0] == '\0') { return -1; } - return (e[0] != '0') ? 1 : 0; - }(); - const bool c8_default_on = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || - backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E; - const bool c8_f16_on = (c8_env_state >= 0) ? (c8_env_state == 1) : c8_default_on; - // Quant-KV (q4_0/q8_0) GQA4 c8: default-on X2E + X1E - const bool c8_quant_on = (c8_env_state >= 0) ? (c8_env_state == 1) : c8_default_on; - if (mq_enabled && mq_kv_ok && nq_in_vec_range && !is_causal && - backend_ctx->gpu_family != INTEL && - !use_local_tile && - n_kv >= FD_MIN_N_KV && - backend_ctx->fa.f32_merge.count(dk_dv) > 0) { - if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && - gqa_ratio_dispatch == 8 && - backend_ctx->fa.f32_f16_q1_local_mq_split_g8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split_g8.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 64; - } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && - gqa_ratio_dispatch == 4 && - backend_ctx->fa.f32_f16_q1_local_mq_split.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 64; - } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 4 && - ((d_head_q == 256 && d_head_v == 256) || - (d_head_q == 128 && d_head_v == 128)) && - backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { - const bool k_img_on = d_head_q == 128 && d_head_v == 128 && - getenv("GGML_OPENCL_FA_K_IMG") != NULL && - getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && - backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.count(dk_dv) > 0; - // Cluster-parallel decode - const bool c8_env = d_head_q == 128 && d_head_v == 128 && c8_f16_on; - if (c8_env && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.at(dk_dv); - use_fd_mq = true; - } else if (c8_env && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 128; - } else if (k_img_on) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.at(dk_dv); - use_fd_mq = true; - use_fa_k_img = true; - } else { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); - use_fd_mq = true; - } - // Cluster-parallel decode, DK=DV=256 GQA=8 - } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 8 && - d_head_q == 256 && d_head_v == 256 && - c8_env_state == 1 && - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 128; - // Cluster-parallel decode for the g8 - } else if (is_mixed && gqa_ratio_dispatch == 8 && - d_head_q == 128 && d_head_v == 128 && - c8_f16_on && - (backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count(dk_dv) > 0 || - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2.count(dk_dv) > 0)) { - if (backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.at(dk_dv); - fd_mq_wg = 192; - } else { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2.at(dk_dv); - fd_mq_wg = 128; - } - use_fd_mq = true; - } else if (is_mixed && gqa_ratio_dispatch == 8 && - d_head_q == 128 && d_head_v == 128 && - getenv("GGML_OPENCL_FA_K_IMG") != NULL && - getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 192; - use_fa_k_img = true; - } else if (is_mixed && gqa_ratio_dispatch == 8 && - d_head_q == 128 && d_head_v == 128 && - backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 192; - } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 8 && - d_head_q == 128 && d_head_v == 128 && - backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 192; - } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 4 && - d_head_q == 128 && d_head_v == 128 && - backend_ctx->fa.f32_q8_0_q1_vec_mq_split.count(dk_dv) > 0) { - // Cluster-parallel q8_0 GQA4 - if (c8_quant_on && - backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8.at(dk_dv); - fd_mq_wg = 128; - } else { - fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split.at(dk_dv); - } - use_fd_mq = true; - } else if (nq1_only && is_q4_0) { - const char * q4_mq_env = getenv("GGML_OPENCL_FA_Q4_MQ"); - const bool q4_mq_on = (q4_mq_env != NULL) && (q4_mq_env[0] != '0'); - // Cluster-parallel q4_0 - const bool q4_c8_on = c8_env_state == 1 && - backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8.count(dk_dv) > 0; - if (q4_c8_on && gqa_ratio_dispatch == 8 && - d_head_q == 64 && d_head_v == 64) { - fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 128; - } else if (q4_mq_on && gqa_ratio_dispatch == 8 && - d_head_q == 128 && d_head_v == 128 && - backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 192; - } else if (gqa_ratio_dispatch == 4 && - d_head_q == 128 && d_head_v == 128 && - backend_ctx->fa.f32_q4_0_q1_vec_mq_split.count(dk_dv) > 0) { - // Cluster-parallel q4_0 GQA4 - if (c8_quant_on && - backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8.at(dk_dv); - fd_mq_wg = 128; - } else { - fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split.at(dk_dv); - } - use_fd_mq = true; - } - } - } - } - // Intel cluster-parallel decode FA - if (fd_k_split == NULL && backend_ctx->gpu_family == INTEL && n_q == 1 && !is_causal && - is_mixed && gqa_ratio_dispatch == 4 && d_head_q == 128 && d_head_v == 128 && - n_kv >= FD_MIN_N_KV && - getenv("GGML_OPENCL_FA_C8") != NULL && getenv("GGML_OPENCL_FA_C8")[0] != '0' && - backend_ctx->fa.f32_merge.count(dk_dv) > 0) { - if (backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 128; - } else if (backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.at(dk_dv); - use_fd_mq = true; - fd_mq_wg = 64; - } - } - if (fd_k_split == NULL && - n_q >= 1 && n_q <= fd_max_n_q && n_kv >= FD_MIN_N_KV && !is_causal && - d_head_q <= FD_MAX_DK && - backend_ctx->fa.f32_merge.count(dk_dv) > 0) { - if (is_mixed && backend_ctx->fa.f32_f16_q1_split.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_f16_q1_split.at(dk_dv); - } else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1_split.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_q8_0_q1_split.at(dk_dv); - } else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1_split.count(dk_dv) > 0) { - fd_k_split = backend_ctx->fa.f32_q4_0_q1_split.at(dk_dv); - } - } - const bool use_fd = (fd_k_split != NULL); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + + const int ne10 = src1->ne[0]; + const int ne12 = src1->ne[2]; + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 == ne10); + GGML_ASSERT((ne00 % 128) == 0); + GGML_ASSERT(ne0 == ne01); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + // image for q (uint32: each texel packs 32 sign bits) + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 32; + img_desc.buffer = extra0_q1_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - const int n_q_blocks = n_q > 1 ? (n_q + block_m - 1) / block_m : 0; - const int n_kv_blocks = (n_kv > 0 && block_n > 0) ? (n_kv + block_n - 1) / block_n : 0; - // KV pad + blk prepass are pure overhead when FD will fire — skip them. - const bool use_mixed_prepass = is_mixed && n_q > 1 && !use_fd; - // make sure prepass kernels are compiled - const bool have_kv_pad = backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0; - const bool have_blk = backend_ctx->fa.blk_f16.count(dk_dv) > 0; - const bool use_kv_pad = use_mixed_prepass && (n_kv % block_n != 0) && have_kv_pad; - // blk prepass: per-KV-tile mask class (0=masked, 1=mixed, 2=unmasked). - // Consumed identically by f32_f16, q8_0 and q4_0 prefill kernels. - const bool use_quant_prepass = (use_native_q8_0 || use_native_q4_0) && !use_fd; - const bool use_blk_mask = (use_mixed_prepass || use_quant_prepass) && mask_buffer != NULL && have_blk; + // create a sub_buffer for B + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - if (use_kv_pad) { - cl_int err; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - const size_t k_pad_size = (size_t) k_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; - temp_k_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_pad_size, NULL, &err); - CL_CHECK(err); - k_pad_buffer = temp_k_pad.data; + // Tall M has enough row parallelism to skip the K split and its reduction. + const bool use_nsg1 = backend_ctx->min_m_gemv_nsg1_q1_0 > 0 && M >= backend_ctx->min_m_gemv_nsg1_q1_0; + kernel = use_nsg1 ? backend_ctx->kernel_gemv_noshuffle_q1_0_f32_nsg1 : backend_ctx->kernel_gemv_noshuffle_q1_0_f32; - const size_t v_pad_size = (size_t) v_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; - temp_v_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_pad_size, NULL, &err); - CL_CHECK(err); - v_pad_buffer = temp_v_pad.data; + int r2 = 1; + int r3 = 1; - cl_kernel kernel_kv_pad = backend_ctx->fa.kv_pad_f16.at(dk_dv); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 0, sizeof(cl_mem), &k_data_device)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 1, sizeof(cl_ulong), &offset_k)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 2, sizeof(cl_mem), &v_data_device)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 3, sizeof(cl_ulong), &offset_v)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 4, sizeof(cl_mem), &k_pad_buffer)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 5, sizeof(cl_mem), &v_pad_buffer)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 6, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 7, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 8, sizeof(int), &n_batch)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 9, sizeof(cl_ulong), &k_nb1)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 10, sizeof(cl_ulong), &k_nb2)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 11, sizeof(cl_ulong), &k_nb3)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 12, sizeof(cl_ulong), &v_nb1)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 13, sizeof(cl_ulong), &v_nb2)); - CL_CHECK(clSetKernelArg(kernel_kv_pad, 14, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &extra1->offset)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); - size_t global_work_size[] = { (size_t) block_n, (size_t) n_head_kv, (size_t) n_batch }; - backend_ctx->enqueue_ndrange_kernel(kernel_kv_pad, 3, global_work_size, NULL, dst); + size_t wavesize = backend_ctx->adreno_wave_size; + const size_t nsg = use_nsg1 ? 1 : 4; + size_t local_work_size[] = { wavesize, nsg, 1 }; + size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, nsg, 1 }; - if (mask_buffer != NULL) { - mask_pad_nb1 = (cl_ulong) block_n * (cl_ulong) sizeof(ggml_fp16_t); - mask_pad_nb2 = (cl_ulong) n_q * mask_pad_nb1; - mask_pad_nb3 = (cl_ulong) mask_ne2 * mask_pad_nb2; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - const size_t mask_pad_size = (size_t) mask_ne3 * (size_t) mask_pad_nb3; - temp_mask_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, mask_pad_size, NULL, &err); - CL_CHECK(err); - mask_pad_buffer = temp_mask_pad.data; + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; - cl_kernel kernel_mask_pad = backend_ctx->fa.mask_pad_f16.at(dk_dv); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 0, sizeof(cl_mem), &mask_buffer)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 1, sizeof(cl_ulong), &offset_mask)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 2, sizeof(cl_mem), &mask_pad_buffer)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 3, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 4, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 5, sizeof(cl_ulong), &mask_nb1)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 6, sizeof(cl_ulong), &mask_nb2)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 7, sizeof(cl_ulong), &mask_nb3)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 8, sizeof(int), &mask_ne2)); - CL_CHECK(clSetKernelArg(kernel_mask_pad, 9, sizeof(int), &mask_ne3)); + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - size_t global_work_size_mask[] = { (size_t) block_n, (size_t) n_q, (size_t) (mask_ne2 * mask_ne3) }; - backend_ctx->enqueue_ndrange_kernel(kernel_mask_pad, 3, global_work_size_mask, NULL, dst); + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; } - } - if (use_blk_mask) { - cl_int err; - const size_t blk_size = (size_t) n_kv_blocks * (size_t) n_q_blocks * (size_t) mask_ne2 * (size_t) mask_ne3; - temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); - if (err != CL_SUCCESS) { - // Flush before retry — reclaim deferred driver deallocations. - CL_CHECK(clFinish(backend_ctx->queue)); - temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; } - CL_CHECK(err); - blk_buffer = temp_blk.data; + int width_B = K/4; + int padded_height_B = (N + padding)/4; - cl_kernel kernel_blk = backend_ctx->fa.blk_f16.at(dk_dv); - CL_CHECK(clSetKernelArg(kernel_blk, 0, sizeof(cl_mem), &mask_buffer)); - CL_CHECK(clSetKernelArg(kernel_blk, 1, sizeof(cl_ulong), &offset_mask)); - CL_CHECK(clSetKernelArg(kernel_blk, 2, sizeof(cl_mem), &blk_buffer)); - CL_CHECK(clSetKernelArg(kernel_blk, 3, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel_blk, 4, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel_blk, 5, sizeof(cl_ulong), &mask_nb1)); - CL_CHECK(clSetKernelArg(kernel_blk, 6, sizeof(cl_ulong), &mask_nb2)); - CL_CHECK(clSetKernelArg(kernel_blk, 7, sizeof(cl_ulong), &mask_nb3)); - CL_CHECK(clSetKernelArg(kernel_blk, 8, sizeof(int), &mask_ne2)); - CL_CHECK(clSetKernelArg(kernel_blk, 9, sizeof(int), &mask_ne3)); + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - size_t global_work_size_blk[] = { (size_t) n_kv_blocks, (size_t) n_q_blocks, (size_t) (mask_ne2 * mask_ne3) }; - backend_ctx->enqueue_ndrange_kernel(kernel_blk, 3, global_work_size_blk, NULL, dst); + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q1_0_f32; + int padded_N = N + padding; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; + size_t local_work_size[] = { 2, 128, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif +} - const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; - const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; - const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_mul_mat_q2_0_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q2_0 * extra0 = (ggml_tensor_extra_cl_q2_0 *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - if (use_fd) { - static const int fd_env_kv_per_split = []{ - const char * e = getenv("GGML_OPENCL_FD_KV_PER_SPLIT"); - return (e && e[0]) ? atoi(e) : 0; - }(); - static const int fd_env_max_splits = []{ - const char * e = getenv("GGML_OPENCL_FD_MAX_SPLITS"); - return (e && e[0]) ? atoi(e) : 0; - }(); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_q2_0_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_q2_0_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_Q2_0_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + +static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q2_0 * extra0 = (ggml_tensor_extra_cl_q2_0 *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + const bool wide = m >= 16384 && m % 8 == 0; + cl_kernel gemv = wide ? backend_ctx->kernel_gemv_noshuffle_q2_0_f32_wide : backend_ctx->kernel_gemv_noshuffle_q2_0_f32; + const size_t workgroup = wide ? backend_ctx->q2_0_mv_workgroup_size_wide : backend_ctx->q2_0_mv_workgroup_size; + const size_t rows = wide ? 8 : 4; + size_t nsg = std::min((size_t)backend_ctx->q2_0_mv_nsg, wide ? size_t(8) : m >= 4096 ? size_t(16) : size_t(32)); + while (nsg > (size_t)k/64 || nsg*16 > workgroup) { + nsg /= 2; + } + const size_t lanes = workgroup/nsg; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_d)); + size_t global[] = { (size_t)CEIL_DIV(m, rows*lanes)*lanes, (size_t)n*nsg, 1 }; + size_t local[] = { lanes, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_q2_0_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_q2_0_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} - int fd_kv_per_split = use_fd_mq ? FD_MQ_KV_PER_SPLIT - : (is_mixed ? FD_KV_PER_SPLIT_F16 : FD_KV_PER_SPLIT); - int fd_max_splits = use_fd_mq ? FD_MQ_MAX_SPLITS : FD_MAX_SPLITS; - if (fd_env_kv_per_split > 0) { fd_kv_per_split = fd_env_kv_per_split; } - if (fd_env_max_splits > 0) { fd_max_splits = fd_env_max_splits; } - int n_splits = (n_kv + fd_kv_per_split - 1) / fd_kv_per_split; - if (n_splits < FD_MIN_SPLITS) { n_splits = FD_MIN_SPLITS; } - if (n_splits > fd_max_splits) { n_splits = fd_max_splits; } - const int kv_per_split = (n_kv + n_splits - 1) / n_splits; +static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - const int fa_partial_floats = 2 + d_head_v; - const size_t partial_size_bytes = - (size_t) n_batch * n_head * n_q * n_splits * fa_partial_floats * sizeof(float); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_cl_flash_attn_temp_buffer temp_partial; - cl_int err; - temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, - partial_size_bytes, NULL, &err); - if (err != CL_SUCCESS) { - CL_CHECK(clFinish(backend_ctx->queue)); - temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, - partial_size_bytes, NULL, &err); - } - CL_CHECK(err); + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; - cl_kernel k_split = fd_k_split; - int argi = 0; - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &extra_q->data_device)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_q)); - if (use_fa_k_img) { - const size_t nb00_bytes = sizeof(uint16_t); - const size_t k_bytes_span = - (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + - (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + - (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + - (size_t)d_head_q * nb00_bytes; - const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; - const size_t k_pixels = k_bytes >> 3; - cl_mem k_img = nullptr; - if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { - k_img = ggml_cl_img_pool_get_or_create( - backend_ctx, backend_ctx->kq_img_pool, - k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); - } + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // if image creation fails, fallback to buffer based kernels - if (k_img == nullptr) { - if (gqa_ratio_dispatch == 4 && - backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { - k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); - } else { - k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); - } - use_fa_k_img = false; - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); - } else { - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_img)); - } - } else { - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); - } - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &v_data_device)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_v)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &scale)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb1)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb2)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb3)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb1)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb2)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb3)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb1)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb2)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb3)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &max_bias)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m0)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m1)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_log2_val)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &logit_softcap)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &mask_buffer)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_mask)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb1)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb2)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb3)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne2)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne3)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &temp_partial.data)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_splits)); - CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &kv_per_split)); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - // MQ split kernel uses MQ_NSG_SPLIT subgroups and one WG per (kv_head, batch, split) - // matches Q1_WG_SIZE * NSG (MQ_GQA=4 -> 256; MQ_GQA=8 -> 192) - const size_t fd_wg = use_fd_mq ? fd_mq_wg : 64; - const size_t fd_head_dim = use_fd_mq - ? (size_t)(n_head_kv * n_batch) - : (size_t)(n_head * n_batch); - size_t fd_lws[3] = { fd_wg, 1, 1 }; - // gid(2) packs q_idx * n_splits + split_idx. - size_t fd_gws[3] = { fd_wg, fd_head_dim, (size_t)(n_splits * n_q) }; - backend_ctx->enqueue_ndrange_kernel(k_split, 3, fd_gws, fd_lws, dst); + const int ne1 = dst->ne[1]; - cl_kernel k_merge = backend_ctx->fa.f32_merge.at(dk_dv); - argi = 0; - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &temp_partial.data)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &extra_o->data_device)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_o)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_splits)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb1)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb2)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb3)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &sinks_buffer)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_sinks)); - CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_q)); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - const size_t merge_wg = (size_t) (d_head_v / 4); // one lane per float4 - size_t merge_lws[3] = { merge_wg, 1, 1 }; - size_t merge_gws[3] = { merge_wg, (size_t)(n_head * n_batch), (size_t) n_q }; - backend_ctx->enqueue_ndrange_kernel(k_merge, 3, merge_gws, merge_lws, dst); - return; - } + cl_context context = backend_ctx->context; + cl_kernel kernel; - cl_mem prefill_k_img = nullptr; - if (use_prefill_k_img) { - const size_t nb00_bytes = sizeof(uint16_t); - const size_t k_bytes_span = - (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + - (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + - (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + - (size_t)d_head_q * nb00_bytes; - const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; - const size_t k_pixels = k_bytes >> 3; - if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { - prefill_k_img = ggml_cl_img_pool_get_or_create( - backend_ctx, backend_ctx->kq_img_pool, - k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); - } - if (prefill_k_img == nullptr) { - kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); - use_prefill_k_img = false; - } - } + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), use_prefill_k_img ? &prefill_k_img : &k_data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_k)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &v_data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_v)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_o->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_o)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(float), &scale)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &is_causal)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &q_nb1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &q_nb2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &q_nb3)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &k_nb1)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &k_nb2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &k_nb3)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &v_nb1)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &v_nb2)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &v_nb3)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &o_nb1)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &o_nb2)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &o_nb3)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &max_bias)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(float), &m0)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &m1)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &n_head_log2_val)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(float), &logit_softcap)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_mem), &mask_buffer)); - CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &offset_mask)); - CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &mask_nb1)); - CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &mask_nb2)); - CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &mask_nb3)); - CL_CHECK(clSetKernelArg(kernel, 36, sizeof(int), &mask_ne2)); - CL_CHECK(clSetKernelArg(kernel, 37, sizeof(int), &mask_ne3)); - CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_mem), &sinks_buffer)); - CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &offset_sinks)); - if (n_q > 1 && is_mixed) { - CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &k_pad_buffer)); - CL_CHECK(clSetKernelArg(kernel, 41, sizeof(cl_mem), &v_pad_buffer)); - CL_CHECK(clSetKernelArg(kernel, 42, sizeof(cl_mem), &mask_pad_buffer)); - CL_CHECK(clSetKernelArg(kernel, 43, sizeof(cl_mem), &blk_buffer)); - CL_CHECK(clSetKernelArg(kernel, 44, sizeof(int), &n_kv_blocks)); - CL_CHECK(clSetKernelArg(kernel, 45, sizeof(cl_ulong), &mask_pad_nb1)); - CL_CHECK(clSetKernelArg(kernel, 46, sizeof(cl_ulong), &mask_pad_nb2)); - CL_CHECK(clSetKernelArg(kernel, 47, sizeof(cl_ulong), &mask_pad_nb3)); - } else if (use_native_q8_0 || use_native_q4_0) { - // arg 40 = blk classification buffer (NULL disables prepass opt). - CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &blk_buffer)); - } + int M = ne01; + int N = ne1; + int K = ne00; - if (n_q == 1) { - if (use_local_tile) { - const size_t lt_wg = 128; - size_t local_work_size[] = { lt_wg, 1, 1 }; - size_t global_work_size[] = { lt_wg, (size_t) n_head, (size_t) n_batch }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - // q1_vec dispatches with NSG subgroups - const size_t q1_wg = backend_ctx->gpu_family == INTEL ? 32 : 64; - const size_t wg_size = use_q1_vec ? 256 : q1_wg; - const size_t head_dim_global = use_q1_vec_mq - ? (size_t)(n_head_kv * n_batch) - : (size_t)(n_head * n_batch); - size_t local_work_size[] = { wg_size, 1 }; - size_t global_work_size[] = { wg_size, head_dim_global }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); - } - } else if (use_native_q8_0 || use_native_q4_0) { - // Native quant prefill. The split variant may override BLOCK_M - // (e.g. DK=96 quant uses BM=16). - const bool use_split = use_native_q8_0 ? use_split_q8_0 : use_split_q4_0; - int bm; - size_t wg_size; - if (use_split) { - bm = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_bm.at(dk_dv) - : backend_ctx->fa.f32_q4_0_split_bm.at(dk_dv); - wg_size = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_wg_size.at(dk_dv) - : backend_ctx->fa.f32_q4_0_split_wg_size.at(dk_dv); - } else { - bm = backend_ctx->fa.bm.at(dk_dv); - wg_size = (size_t) bm; - } - size_t local_work_size[] = { wg_size, 1 }; - size_t global_work_size[] = { (size_t)((n_q + bm - 1) / bm) * wg_size, (size_t)(n_head * n_batch) }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &M)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[3] = { wavesize, 4, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M, 64) * 64, 4, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); } else { - const size_t wg_size = (size_t) wg_size_fa; - size_t local_work_size[] = { wg_size, 1 }; - size_t global_work_size[] = { (size_t)((n_q + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); - } -} + const bool have_a8 = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin != nullptr; + const int gemm_tile_n = have_a8 ? 64 : 32; + int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); -static void ggml_cl_mul_mat_f16_f32_tiled(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_mem a_img = extra0_q4_0->q_img; + cl_mem s_img = extra0_q4_0->d_img; + GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them"); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + static const char * q4_0_bin_dp4a_env = getenv("GGML_OPENCL_Q4_0_BIN_DP4A"); + bool q4_0_bin_dp4a_on = q4_0_bin_dp4a_env + ? (atoi(q4_0_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q4_0_bin_dp4a_on = backend_ctx->has_integer_dot && q4_0_bin_dp4a_on; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + if (q4_0_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); - const int M = src0->ne[1]; - const int N = src1->ne[1]; - const int K = src0->ne[0]; + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); - cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_tiled; + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - // Tiling parameters. These need to be tuned for optimal performance. - // They must match the #defines in the kernel mul_mat_f16_f32.cl. - // - // OPWM / OPWN: Output tile size per Work-Group. A work-group computes a tile of size OPWM x OPWN. - // TPWM / TPWN: Threads per Work-group. This is the work-group size. - // OPTM / OPTN: Output elements per Thread. Each thread computes OPTM x OPTN elements. - // - // The following relationships must hold: - // OPWM = TPWM * OPTM - // OPWN = TPWN * OPTN - // - const int OPWM = 64; - const int OPWN = 64; - const int TPWM = 16; - const int TPWN = 8; + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - size_t local_work_size[2] = { TPWM, TPWN }; - size_t global_work_size[2] = { - (size_t) ((M + OPWM - 1) / OPWM) * TPWM, - (size_t) ((N + OPWN - 1) / OPWN) * TPWN, - }; + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); -} + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static bool ggml_cl_can_use_adreno_xmem_gemm_f16_f32( - const ggml_backend_opencl_context * backend_ctx, - const ggml_tensor * src0, - const ggml_tensor * src1, - const ggml_tensor * dst) { - if (!backend_ctx->adreno_xmem_gemm_enabled) { - return false; - } - if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { - return false; - } - if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16) || - src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { - return false; - } - if (src0->ne[2] != 1 || src0->ne[3] != 1 || - src1->ne[2] != 1 || src1->ne[3] != 1 || - dst->ne[2] != 1 || dst->ne[3] != 1) { - return false; - } - const int K = src0->ne[0]; - const int M = src0->ne[1]; - const int N = src1->ne[1]; - if (src1->ne[0] != K || dst->ne[0] != M || dst->ne[1] != N) { - return false; - } - if (N <= 1 || M < 64 || N < 16 || K < 64) { - return false; - } - if ((K % 8) != 0) { - return false; - } - const int kpack = K / 4; - const int npack = CEIL_DIV(M, 4); - if (static_cast<size_t>(N) > backend_ctx->image2d_max_width || - static_cast<size_t>(kpack) > backend_ctx->image2d_max_height) { - return false; - } - if (static_cast<size_t>(N) > backend_ctx->image2d_max_width || - static_cast<size_t>(npack) > backend_ctx->image2d_max_height) { - return false; - } - return true; -} + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); -static void ggml_cl_mul_mat_f16_f32_adreno_xmem( - ggml_backend_t backend, - const ggml_tensor * src0, - const ggml_tensor * src1, - ggml_tensor * dst) { - ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } - const cl_ulong offset0 = extra0->offset + src0->view_offs; - const cl_ulong offset1 = extra1->offset + src1->view_offs; - const cl_ulong offsetd = extrad->offset + dst->view_offs; + // Pad B through a zero-filled scratch buffer when N needs + // padding, since the GEMM kernel always reads a full N-tile. + const bool need_pad = N_pad > N; + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + if (need_pad) { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, + (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), + 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, + offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + } else { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + } - const int K = src0->ne[0]; - const int M = src0->ne[1]; - const int N = src1->ne[1]; - const int kpack = K / 4; - const int npack = CEIL_DIV(M, 4); - const int os = 8; + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = need_pad ? (size_t)K * N_pad : (size_t)K * N; + img_desc.buffer = need_pad ? b_padded : b_sub_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - const size_t xmem_bytes = 6144; - const size_t weight_bytes = static_cast<size_t>(kpack) * static_cast<size_t>(npack) * 4u * sizeof(cl_half4); + int line_stride_matrix_A_in_bytes = M * 4; + int line_stride_matrix_S_in_bytes = M * 2; + int line_stride_matrix_B_in_bytes = K * 4; + int line_stride_matrix_C_in_bytes = M * 4; - backend_ctx->prealloc_adreno_xmem_const.allocate(backend_ctx->context, xmem_bytes); + const int b_offset_for_kernel = 0; - cl_int err = CL_SUCCESS; - cl_image_format fmt = {}; - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_HALF_FLOAT; + if (!have_a8) { + const int c_offset_for_kernel = (int)offsetd; - cl_image_desc desc_src = {}; - desc_src.image_type = CL_MEM_OBJECT_IMAGE2D; - desc_src.image_width = static_cast<size_t>(N); - desc_src.image_height = static_cast<size_t>(kpack); - cl_mem src_img = clCreateImage(backend_ctx->context, CL_MEM_READ_WRITE, &fmt, &desc_src, nullptr, &err); - CL_CHECK(err); + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + if ((m_tiles % 2) != 0) { + // keep the work-group count in y even so every work-group is full + m_tiles += 1; + } + + size_t local_work_size[3] = { 64, 2, 4 }; + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + return; + } - cl_image_desc desc_dst = {}; - desc_dst.image_type = CL_MEM_OBJECT_IMAGE2D; - desc_dst.image_width = static_cast<size_t>(N); - desc_dst.image_height = static_cast<size_t>(npack); - cl_mem dst_img = clCreateImage(backend_ctx->context, CL_MEM_READ_WRITE, &fmt, &desc_dst, nullptr, &err); - CL_CHECK(err); + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_mem weights = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, weight_bytes, nullptr, &err); - CL_CHECK(err); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - cl_kernel prepack = backend_ctx->kernel_adreno_xmem_prepack_weight_f16; - CL_CHECK(clSetKernelArg(prepack, 0, sizeof(cl_mem), &weights)); - CL_CHECK(clSetKernelArg(prepack, 1, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(prepack, 2, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(prepack, 3, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(prepack, 4, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(prepack, 5, sizeof(int), &kpack)); - CL_CHECK(clSetKernelArg(prepack, 6, sizeof(int), &npack)); - CL_CHECK(clSetKernelArg(prepack, 7, sizeof(int), &os)); - size_t lws = 256; - size_t max_wg = backend_ctx->get_kernel_workgroup_size(prepack); - if (lws > max_wg) { - lws = max_wg; - } - size_t gws = CEIL_DIV(static_cast<size_t>(kpack) * static_cast<size_t>(npack), lws) * lws; - backend_ctx->enqueue_ndrange_kernel(prepack, 1, &gws, &lws, dst); + const int c_offset_for_kernel = 0; - cl_kernel pack_src = backend_ctx->kernel_adreno_xmem_pack_src_f32; - CL_CHECK(clSetKernelArg(pack_src, 0, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(pack_src, 1, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(pack_src, 2, sizeof(cl_mem), &src_img)); - CL_CHECK(clSetKernelArg(pack_src, 3, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(pack_src, 4, sizeof(int), &N)); - size_t pack_src_lws[2] = { 16, 16 }; - size_t pack_src_gws[2] = { - CEIL_DIV(static_cast<size_t>(N), pack_src_lws[0])*pack_src_lws[0], - CEIL_DIV(static_cast<size_t>(kpack), pack_src_lws[1])*pack_src_lws[1] - }; - backend_ctx->enqueue_ndrange_kernel(pack_src, 2, pack_src_gws, pack_src_lws, dst); + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; - cl_kernel gemm = backend_ctx->kernel_gemm_xmem_f16_f32_os8; - CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &weights)); - CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &backend_ctx->prealloc_adreno_xmem_const.buffer)); - CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &src_img)); - CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &dst_img)); - CL_CHECK(clSetKernelArg(gemm, 4, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(gemm, 5, sizeof(int), &npack)); - CL_CHECK(clSetKernelArg(gemm, 6, sizeof(int), &kpack)); - const size_t z_values = CEIL_DIV(static_cast<size_t>(npack), static_cast<size_t>(os)); - size_t gemm_lws[3] = { 64, 1, 1 }; - size_t gemm_gws[3] = { - z_values*gemm_lws[0], - CEIL_DIV(static_cast<size_t>(N), gemm_lws[0]), - 1 - }; - backend_ctx->enqueue_ndrange_kernel(gemm, 3, gemm_gws, gemm_lws, dst); + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); - cl_kernel store_dst = backend_ctx->kernel_adreno_xmem_store_dst_f32; - CL_CHECK(clSetKernelArg(store_dst, 0, sizeof(cl_mem), &dst_img)); - CL_CHECK(clSetKernelArg(store_dst, 1, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(store_dst, 2, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(store_dst, 3, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(store_dst, 4, sizeof(int), &N)); - size_t store_lws[2] = { 16, 16 }; - size_t store_gws[2] = { - CEIL_DIV(static_cast<size_t>(N), store_lws[0])*store_lws[0], - CEIL_DIV(static_cast<size_t>(npack), store_lws[1])*store_lws[1] - }; - backend_ctx->enqueue_ndrange_kernel(store_dst, 2, store_gws, store_lws, dst); + size_t local_work_size[3] = { 64, 2, 2 }; + size_t global_work_size[3] = { 64, (size_t)CEIL_DIV(M, 64), (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(weights)); - CL_CHECK(clReleaseMemObject(dst_img)); - CL_CHECK(clReleaseMemObject(src_img)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_TENSOR_BINARY_OP_LOCALS; +static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const cl_uint Cout = ne03; - const cl_uint Cin = ne02; - const cl_uint N = ne13; - const cl_uint KW = ne00; - const cl_uint KH = ne01; - const cl_uint W = ne10; - const cl_uint H = ne11; - const cl_uint OW = ne0; - const cl_uint OH = ne1; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; - const cl_uint s0 = dst->op_params[0]; - const cl_uint s1 = dst->op_params[1]; - const cl_uint p0 = dst->op_params[2]; - const cl_uint p1 = dst->op_params[3]; - const cl_uint d0 = dst->op_params[4]; - const cl_uint d1 = dst->op_params[5]; + const int ne10 = src1->ne[0]; + const int ne12 = src1->ne[2]; - const cl_uint cl_nb00 = nb00/ggml_type_size(src0->type); - const cl_uint cl_nb01 = nb01/ggml_type_size(src0->type); - const cl_uint cl_nb02 = nb02/ggml_type_size(src0->type); - const cl_uint cl_nb03 = nb03/ggml_type_size(src0->type); - const cl_uint cl_nb10 = nb10/ggml_type_size(src1->type); - const cl_uint cl_nb11 = nb11/ggml_type_size(src1->type); - const cl_uint cl_nb12 = nb12/ggml_type_size(src1->type); - const cl_uint cl_nb13 = nb13/ggml_type_size(src1->type); - const cl_uint cl_nb1 = nb1/ggml_type_size(dst->type); - const cl_uint cl_nb2 = nb2/ggml_type_size(dst->type); - const cl_uint cl_nb3 = nb3/ggml_type_size(dst->type); + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; - const int64_t NPQ = (int64_t)N * OW * OH; + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - const uint32_t BS_K = 64; - const uint32_t BS_NPQ = 64; - const uint32_t BS_CRS = 16; - const uint32_t VEC_SIZE = 4; + cl_context context = backend_ctx->context; + cl_kernel kernel; - const uint32_t TS_K = 4; - const uint32_t TS_NPQ = 8; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - const uint32_t WG_K = BS_K / TS_K; - const uint32_t WG_NPQ = BS_NPQ / TS_NPQ; + int M = ne01; + int N = ne1; + int K = ne00; - auto splitWork = [](uint32_t work_size, uint32_t block_size) { return (block_size + work_size - 1) / block_size; }; - const uint32_t NB_K = splitWork(Cout, BS_K); - const uint32_t NB_NPQ = splitWork(NPQ, BS_NPQ); + // Multi-column (N=3) verify GEMV for q4_0 + static const bool q40_mc3 = (getenv("GGML_OPENCL_Q4_0_MC3") != nullptr); + const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); - cl_kernel kernel; - size_t shmem_size; + const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); - if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { - kernel = backend_ctx->kernel_conv_2d_f16; - shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_half4)); - } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_conv_2d_f32; - shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_float) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); - } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_conv_2d_f16_f32; - shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); - } else { - GGML_ASSERT(false && "Unsupported data type combination for conv2d"); + if (use_bin) { + if (use_q40_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4_0_MC3 is bypassed by Q4_0 binary kernels\n"); + warned = true; + } + } + // This bin layout is not supported by CoopK. + ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst); + return; } - cl_uint idx = 0; - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, idx++, shmem_size, NULL)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cout)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cin)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &N)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KW)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KH)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &W)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OW)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OH)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s0)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p0)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d0)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb00)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb01)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb02)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb03)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb10)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb11)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb12)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb13)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb2)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb3)); + if (ne1 == 1 || use_q40_mc3) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + // image for q + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - size_t global_work_size[] = { (size_t)NB_K * WG_K, (size_t)NB_NPQ * WG_NPQ, 1 }; - size_t local_work_size[] = { (size_t)WG_K, (size_t)WG_NPQ, 1 }; + if (use_q40_mc3) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); + } else { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32; + if (M == 4096 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; + } else if (M == 4096 && K == 11008) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; + } else if (M == 11008 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; + } else if (M == 32000 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); -} + int r2 = 1; + int r3 = 1; -// is_kq selects which of the two products this call is, and it is decided by the -// CALLER -- the two admission arms in ggml_cl_mul_mat, each of which knows which -// one it matched. It used to be re-derived here from nb01 > nb02, i.e. "K is -// head-major, V^T is not". That discriminator COLLAPSES at n_head_kv == 1, where -// the two strides are equal because there is only one head to order, so nothing -// here could tell a KQ from a KQV. Pass it in rather than infer it. -static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool is_kq) { - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + } - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // Small-M mc3 verify is occupancy/latency-bound (too few WGs at small M, so + // its bandwidth falls well short of the FFN matmuls'). Use 8 subgroups (512-WI WGs, half the + // per-lane K-walk) for small M. Layout stride is fixed (4 uints/block), so only + // the K-split count changes; the mc3 kernel reads it via get_local_size(1). The + // ne1==1 base kernel hardcodes N_SIMDGROUP=4, so it always stays at 4. + int mc3_nsg = (use_q40_mc3 && ne01 < 4096) ? 8 : 4; + if (use_q40_mc3) { + // Register pressure can lower the kernel limit below the device limit. + const size_t max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (mc3_nsg > 1 && (size_t)(64 * mc3_nsg) > max_workgroup_size) { + mc3_nsg /= 2; + } + } + size_t local_work_size[3] = {64, (size_t)mc3_nsg, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, (size_t)mc3_nsg, 1}; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + // CoopK uses the same untiled weight layout as gemm_noshuffle_q4_0_f32, with K-major Q8_1 activations (K contiguous). + if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_0) + && backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 != nullptr + && ((N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4) != nullptr) { + cl_mem b_sub_buf = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; + const int cols = (N <= 2) ? 2 : 4; + const size_t n_blocks_padded = (size_t)cols * (K / 32); + const size_t size_qa = (size_t)cols * K * sizeof(cl_char); + const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size; + const size_t size_da_prev = backend_ctx->prealloc_moe_da.size; + backend_ctx->prealloc_moe_qa.allocate(context, size_qa); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half)); + if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) { + const cl_char z8 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer, + &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL)); + } + if (backend_ctx->prealloc_moe_da.size != size_da_prev) { + const cl_half z16 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL)); + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL)); + } + + cl_int n_blocks = (cl_int)((size_t)N * (K / 32)); + cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks)); + size_t local_work_size_quant[1] = { 64 }; + size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst); + + cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4; + const cl_int n_padded = cols; + CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_int), &n_padded)); + CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &N)); + + const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q4_0; + size_t local_work_size[3] = { 64, nsg, 1 }; + size_t global_work_size[3] = { (size_t)(M / backend_ctx->rows_coopk_dp4a_q4_0), nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst); - const cl_ulong nb10 = src1->nb[0]; + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; + static const char * q4_0_act_vload4_env = getenv("GGML_OPENCL_Q4_0_DP4A_ACT_VLOAD4"); + const bool q4_0_act_vload4_on = q4_0_act_vload4_env == nullptr || atoi(q4_0_act_vload4_env) != 0; + // Enable dense prefill by default on X2E when the uint4 activation tile is enabled. + static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A"); + bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env + ? (atoi(q4_0_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && q4_0_act_vload4_on); + // dot prod has to be available + q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on; - GGML_ASSERT(ne00 == ne10); + if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_kernel kernel; - cl_context context = backend_ctx->context; + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); - cl_int status; - cl_image_format img_fmt_1d; - cl_image_desc img_desc_1d; - cl_buffer_region region; - cl_mem A_image1d; - cl_mem A_sub_buffer; - cl_mem B_sub_buffer; - cl_mem D_image1d; - cl_mem D_sub_buffer; + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - int M = ne01; - int N = ne1; - int K = ne00; + cl_kernel dk = q4_0_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - kernel = is_kq ? backend_ctx->kernel_mul_mm_f16_f32_kq - : backend_ctx->kernel_mul_mm_f16_f32_kqv; - // create sub-buffer for A - // <--------------------------------------------> // - extra0 = src0->view_src ? (ggml_tensor_extra_cl *)src0->view_src->extra : (ggml_tensor_extra_cl *)src0->extra; + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } - region.origin = (extra0->offset + src0->view_offs); - if (is_kq) { - // KQ - region.size = nb01 * ne01; - } else { - // KQV - region.size = nb02 * ne02; - } + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; + cl_mem d_sub_buf = nullptr; - A_sub_buffer = clCreateSubBuffer((extra0->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - CL_CHECK(status); + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // <--------------------------------------------> // + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // create sub-buffer for B - // <--------------------------------------------> // - region.origin = (extra1->offset + src1->view_offs); - region.size = nb10 * ne10 * ne11 * ne12; - B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - CL_CHECK(status); - // <--------------------------------------------> // + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } - img_fmt_1d = {CL_RGBA, CL_FLOAT}; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - if (is_kq) { - img_desc_1d.image_width = (nb01 * ne01 / 4)/4; - } - else { - img_desc_1d.image_width = (nb02 * ne02 / 4)/4; - } - img_desc_1d.buffer = A_sub_buffer; - A_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); - CL_CHECK(status); + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // create sub-buffer for output C - // <--------------------------------------------> // - region.origin = (extrad->offset + dst->view_offs); - region.size = ne0 * ne1 * dst->ne[2] * dst->nb[0]; // size of C in bytes - D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - CL_CHECK(status); - // <--------------------------------------------> // + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - // create image for C output - // <--------------------------------------------> // - img_fmt_1d = {CL_R, CL_FLOAT}; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4; - img_desc_1d.buffer = D_sub_buffer; - D_image1d = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); - CL_CHECK(status); - // <--------------------------------------------> // + // subbuffer for output + region.origin = extrad->offset; // Specify the starting offset (in bytes) + region.size = M * N * sizeof(float); // Specify the size of the sub-buffer + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = K/4; + int padded_height_B = (N + padding)/4; + + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { + local_work_size_t[0]=4; + local_work_size_t[1]=8; + } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { + local_work_size_t[0]=2; + local_work_size_t[1]=8; + } else if(ne0 == 4096 && ne1 == 128 && ne10 == 11008) { + local_work_size_t[0]=1; + local_work_size_t[1]=8; + } else if(ne0 == 32000 && ne1 == 128 && ne10 == 4096) { + local_work_size_t[0]=2; + local_work_size_t[1]=8; + } + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - int offset_src0 = 0; - int offset_src1 = 0; + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32; + int padded_N = N + padding; - // set kernel args - // <--------------------------------------------> // - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src0)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_sub_buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &D_image1d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &extrad->offset)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d_sub_buf)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne1)); - size_t global_work_size[3] = {64, static_cast<size_t>(((M+63)/64)), static_cast<size_t>(((N+31)/32)*ne12)}; - size_t local_work_size[3] = {64, 1, 2}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; + if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { + local_work_size[0] = 1; + local_work_size[1] = 128; + } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { + local_work_size[0] = 2; + local_work_size[1] = 64; + } else if (ne0 == 4096 && ne1 == 128 && ne10 == 11008) { + local_work_size[0] = 2; + local_work_size[1] = 64; + } else if (ne0 == 32000 && ne1 == 128 && ne10 == 4096) { + local_work_size[0] = 2; + local_work_size[1] = 64; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - // deallocate sub buffers and images - // <--------------------------------------------> // - CL_CHECK(clReleaseMemObject(A_image1d)); - CL_CHECK(clReleaseMemObject(D_image1d)); - CL_CHECK(clReleaseMemObject(A_sub_buffer)); - CL_CHECK(clReleaseMemObject(B_sub_buffer)); - CL_CHECK(clReleaseMemObject(D_sub_buffer)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -19223,34 +25950,21 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_Q1_0); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; + ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - GGML_ASSERT(src1->view_offs == 0); - GGML_ASSERT(dst->view_offs == 0); - const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - - const int ne10 = src1->ne[0]; - const int ne12 = src1->ne[2]; - const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; - GGML_ASSERT(ne00 == ne10); - GGML_ASSERT((ne00 % 128) == 0); - GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; @@ -19264,23 +25978,27 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - if (ne1 == 1) { + // Multi-column (N=3) verify GEMV for q4_1 + static const bool q41_mc3 = (getenv("GGML_OPENCL_Q4_1_MC3") != nullptr); + const bool use_q41_mc3 = q41_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + + if (ne1 == 1 || use_q41_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; - // image for q (uint32: each texel packs 32 sign bits) + // image for q img_fmt = { CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 32; - img_desc.buffer = extra0_q1_0->q; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_1->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // create a sub_buffer for B + // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; @@ -19290,36 +26008,29 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q1_0_f32; - - int r2 = 1; - int r3 = 1; + kernel = use_q41_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 + : backend_ctx->kernel_gemv_noshuffle_q4_1_f32; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &extra1->offset)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &extrad->offset)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + if (use_q41_mc3) { + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); // n_cols + } - size_t wavesize = backend_ctx->adreno_wave_size; - size_t local_work_size[] = { wavesize, 4, 1 }; - size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); } else { cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; @@ -19380,28 +26091,29 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q1_0_f32; + kernel = backend_ctx->kernel_gemm_noshuffle_q4_1_f32; int padded_N = N + padding; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); - size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; - size_t local_work_size[] = { 2, 128, 1 }; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img_trans)); } #else GGML_UNUSED(backend); @@ -19411,9 +26123,8 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); @@ -19425,7 +26136,7 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; + ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; @@ -19450,194 +26161,230 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg int K = ne00; if (ne1 == 1) { + cl_mem qs_img = nullptr; cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + cl_mem b_img = nullptr; + + // image for qs + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q5_0->qs; + CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // subbuffer for activations region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); + region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - img_fmt = { CL_RGBA, CL_FLOAT }; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)K * N / 4; - img_desc.buffer = b_sub_buf; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &K)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &M)); + kernel = backend_ctx->kernel_gemv_noshuffle_q5_0_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; - size_t wavesize = backend_ctx->adreno_wave_size; - size_t local_work_size[3] = { wavesize, 4, 1 }; - size_t global_work_size[3] = { (size_t)CEIL_DIV(M, 64) * 64, 4, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(qs_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { - const int gemm_tile_n = 64; - int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); - - cl_mem a_img = extra0_q4_0->q_img; - cl_mem s_img = extra0_q4_0->d_img; - GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them"); - - static const char * q4_0_bin_dp4a_env = getenv("GGML_OPENCL_Q4_0_BIN_DP4A"); - bool q4_0_bin_dp4a_on = q4_0_bin_dp4a_env - ? (atoi(q4_0_bin_dp4a_env) != 0) - : true; + // dp4a (int8) dense q5_0 prefill GEMM, default off + static const char * q5_dense_dp4a_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A"); + static const char * q5_dense_wimg_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A_WIMG"); + const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0); + bool q5_dense_dp4a_on = q5_dense_wimg_on + ? true + : (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0)); // dot prod has to be available - q4_0_bin_dp4a_on = backend_ctx->has_integer_dot && q4_0_bin_dp4a_on; - - if (q4_0_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin) { - const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; - const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); - - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); - backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); - backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + q5_dense_dp4a_on = backend_ctx->has_integer_dot && q5_dense_dp4a_on; - cl_mem b_sub = nullptr; + if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); - CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_int tb = (cl_int)((size_t)N * (K / 32)); + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - cl_mem d_sub = nullptr; - cl_mem d_img = nullptr; - region.origin = offsetd; - region.size = (size_t)M * N * sizeof(float); - CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - img_fmt = { CL_R, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * N; - img_desc.buffer = d_sub; - CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; - - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + // optional qs texture (image1d_buffer over the nibble plane; the same + // CL_R/UINT32 view, width M*K/8, the GEMV path builds). + cl_mem q5_qs_img = nullptr; + bool use_wimg = q5_dense_wimg_on; + if (use_wimg) { + const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels (2 ushorts/texel) + if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { + use_wimg = false; + } else { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = tex; + img_desc.buffer = extra0_q5_0->qs; + q5_qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q5_qs_img == nullptr) { use_wimg = false; q5_qs_img = nullptr; } + } + } - size_t local_work_size[3] = { 64, 1, 1 }; - size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg + : backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a; + int ai = 0; + if (use_wimg) { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q5_qs_img)); + } else { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qs)); + } + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - CL_CHECK(clReleaseMemObject(b_sub)); - CL_CHECK(clReleaseMemObject(d_img)); - CL_CHECK(clReleaseMemObject(d_sub)); + if (q5_qs_img != nullptr) { + CL_CHECK(clReleaseMemObject(q5_qs_img)); + } + CL_CHECK(clReleaseMemObject(a_sub)); return; } - // Pad B through a zero-filled scratch buffer when N needs - // padding, since the GEMM kernel always reads a full N-tile. - const bool need_pad = N_pad > N; cl_mem b_sub_buf = nullptr; - cl_mem b_padded = nullptr; - if (need_pad) { - CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, - (size_t)K * N_pad * sizeof(float), NULL, &err), err)); - const float zero = 0.0f; - CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), - 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, - offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); - } else { - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - } + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; + cl_mem d_sub_buf = nullptr; - img_fmt = { CL_R, CL_FLOAT }; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = need_pad ? (size_t)K * N_pad : (size_t)K * N; - img_desc.buffer = need_pad ? b_padded : b_sub_buf; - cl_mem b_img; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - region.origin = offsetd; - region.size = (size_t)M * N * sizeof(float); - cl_mem d_sub_buf; - CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } - img_fmt = { CL_R, CL_FLOAT }; + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * N; - img_desc.buffer = d_sub_buf; - cl_mem d_img; - CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - int line_stride_matrix_A_in_bytes = M * 4; - int line_stride_matrix_S_in_bytes = M * 2; - int line_stride_matrix_B_in_bytes = K * 4; - int line_stride_matrix_C_in_bytes = M * 4; + // subbuffer for output + region.origin = extrad->offset; + region.size = M * N * sizeof(float); + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - int c_offset_for_kernel = 0; - int b_offset_for_kernel = 0; + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &s_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q5_0_f32; + int padded_N = N + padding; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_sub_buf)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); + + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; - size_t local_work_size[3] = { 64, 2, 2 }; - size_t m_tiles = (size_t)CEIL_DIV(M, 64); - size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); - if (b_sub_buf) { - CL_CHECK(clReleaseMemObject(b_sub_buf)); - } - if (b_padded) { - CL_CHECK(clReleaseMemObject(b_padded)); - } - CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); CL_CHECK(clReleaseMemObject(d_sub_buf)); } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q5_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -19650,19 +26397,14 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; + ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne10 = src1->ne[0]; - const int ne12 = src1->ne[2]; - - const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); @@ -19679,42 +26421,18 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - // Multi-column (N=3) verify GEMV for q4_0: route the spec/MTP verify batch - // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- - // zone (gemm_noshuffle_q4_0 is ~50% of MTP decode on a Q4_0 model since q4_0 - // weights have no cok/mc3, unlike q4_K). Reuses the ne1==1 GEMV image setup - // (activation image already sized by N=ne1). Byte-identical. Opt-in via - // GGML_OPENCL_Q40_MC3=1. Per-layer only (ne01 < 32768); q4_0 lm_head doesn't - // occur (token_embd/output stay Q6_K), guard kept for parity with q4_K mc3. - static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); - const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); - - const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); - - if (use_bin) { - if (use_q40_mc3) { - static bool warned = false; - if (!warned) { - GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q40_MC3 is bypassed by Q4_0 binary kernels\n"); - warned = true; - } - } - ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst); - return; - } - - if (ne1 == 1 || use_q40_mc3) { - cl_mem q_img = nullptr; + if (ne1 == 1) { + cl_mem qs_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; - // image for q - img_fmt = { CL_R, CL_UNSIGNED_INT32}; + // image for qs + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q4_0->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_desc.buffer = extra0_q5_1->qs; + CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for activations region.origin = offset1; @@ -19729,114 +26447,27 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - if (use_q40_mc3) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); - } else { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32; - if (M == 4096 && K == 4096) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; - } else if (M == 4096 && K == 11008) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; - } else if (M == 11008 && K == 4096) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; - } else if (M == 32000 && K == 4096) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; - } - - int r2 = 1; - int r3 = 1; + kernel = backend_ctx->kernel_gemv_noshuffle_q5_1_f32; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - // Small-M mc3 verify is occupancy/latency-bound (too few WGs at small M, so - // its bandwidth falls well short of the FFN matmuls'). Use 8 subgroups (512-WI WGs, half the - // per-lane K-walk) for small M. Layout stride is fixed (4 uints/block), so only - // the K-split count changes; the mc3 kernel reads it via get_local_size(1). The - // ne1==1 base kernel hardcodes N_SIMDGROUP=4, so it always stays at 4. - const int mc3_nsg = (use_q40_mc3 && ne01 < 4096) ? 8 : 4; - size_t local_work_size[3] = {64, (size_t)mc3_nsg, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, (size_t)mc3_nsg, 1}; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(qs_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { - // dp4a (int8) dense prefill GEMM, default off - static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A"); - bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env - ? (atoi(q4_0_dense_dp4a_env) != 0) - : false; - // dot prod has to be available - q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on; - - if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a - && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { - cl_mem a_sub = nullptr; - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - const size_t n_blocks = (size_t)N * (K / 32); - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); - backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); - backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); - - cl_int tb = (cl_int)n_blocks; - cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); - CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); - size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; - backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - - cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a; - int ai = 0; - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); - size_t d_local[3] = { 64, 1, 1 }; - size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; - backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - - CL_CHECK(clReleaseMemObject(a_sub)); - return; - } - cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; @@ -19878,8 +26509,8 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for output - region.origin = extrad->offset; // Specify the starting offset (in bytes) - region.size = M * N * sizeof(float); // Specify the size of the sub-buffer + region.origin = extrad->offset; + region.size = M * N * sizeof(float); CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // transpose activations @@ -19899,49 +26530,25 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { - local_work_size_t[0]=4; - local_work_size_t[1]=8; - } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { - local_work_size_t[0]=2; - local_work_size_t[1]=8; - } else if(ne0 == 4096 && ne1 == 128 && ne10 == 11008) { - local_work_size_t[0]=1; - local_work_size_t[1]=8; - } else if(ne0 == 32000 && ne1 == 128 && ne10 == 4096) { - local_work_size_t[0]=2; - local_work_size_t[1]=8; - } backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32; + kernel = backend_ctx->kernel_gemm_noshuffle_q5_1_f32; int padded_N = N + padding; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d_sub_buf)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_sub_buf)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; size_t local_work_size[3] = {1, 128, 1}; - if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { - local_work_size[0] = 1; - local_work_size[1] = 128; - } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { - local_work_size[0] = 2; - local_work_size[1] = 64; - } else if (ne0 == 4096 && ne1 == 128 && ne10 == 11008) { - local_work_size[0] = 2; - local_work_size[1] = 64; - } else if (ne0 == 32000 && ne1 == 128 && ne10 == 4096) { - local_work_size[0] = 2; - local_work_size[1] = 64; - } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); @@ -19959,7 +26566,7 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } -static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -19972,7 +26579,7 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra; + ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; @@ -19982,7 +26589,7 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t const int ne1 = dst->ne[1]; - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + GGML_ASSERT(ne00 % 32 == 0); cl_context context = backend_ctx->context; cl_kernel kernel; @@ -19996,14 +26603,7 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - // Multi-column (N=3) verify GEMV for q4_1: route the spec/MTP verify batch - // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- - // zone (gemm_noshuffle_q4_1). Reuses the ne1==1 GEMV image setup. Opt-in via - // GGML_OPENCL_Q41_MC3=1. Per-layer only (ne01 < 32768). - static const bool q41_mc3 = (getenv("GGML_OPENCL_Q41_MC3") != nullptr); - const bool use_q41_mc3 = q41_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); - - if (ne1 == 1 || use_q41_mc3) { + if (ne1 == 1) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; @@ -20013,7 +26613,7 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q4_1->q; + img_desc.buffer = extra0_iq4_nl->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for activations @@ -20029,30 +26629,77 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = use_q41_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 - : backend_ctx->kernel_gemv_noshuffle_q4_1_f32; + kernel = backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); - if (use_q41_mc3) { - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); // n_cols - } + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + // dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to + // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch + // (ne1>8) only + static const char * iq4nl_dense_dp4a_env = getenv("GGML_OPENCL_IQ4NL_DENSE_DP4A"); + bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env + ? (atoi(iq4nl_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // dot prod has to be available + iq4nl_dense_dp4a_on = backend_ctx->has_integer_dot && iq4nl_dense_dp4a_on; + + if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } + cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; @@ -20112,19 +26759,18 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q4_1_f32; + kernel = backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32; int padded_N = N + padding; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; size_t local_work_size[3] = {1, 128, 1}; @@ -20144,7 +26790,170 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t #endif } -static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_mul_mat_iq4_xs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq4_xs * extra0 = (ggml_tensor_extra_cl_iq4_xs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ4_XS_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + +static void ggml_cl_mul_mat_iq4_xs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq4_xs * extra0 = (ggml_tensor_extra_cl_iq4_xs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq4_xs_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq4_xs_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_iq4_xs_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq4_xs_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} + +#endif + +static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -20153,21 +26962,33 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)src0->extra; + // SoA extra lives on view_src (view->extra is pre-SoA). + const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; + ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; - const int ne1 = dst->ne[1]; + const int ne10 = src1->ne[0]; + const int ne12 = src1->ne[2]; - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 == ne10); + GGML_ASSERT((ne00 % 32) == 0); + GGML_ASSERT(ne0 == ne01); cl_context context = backend_ctx->context; cl_kernel kernel; @@ -20181,63 +27002,246 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - if (ne1 == 1) { - cl_mem qs_img = nullptr; + // Multi-column verify GEMV for ne1 2..4, reuses the ne1 == 1 image setup. + // Default on for X2E, GGML_OPENCL_Q8_0_MC=0/1 forces it. + static const char * q8_mc_env = getenv("GGML_OPENCL_Q8_0_MC"); + const bool q8_mc = q8_mc_env ? atoi(q8_mc_env) != 0 : backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E; + const bool use_q8_mc = q8_mc && ne1 >= 2 && ne1 <= 4; + + if (ne1 == 1 || use_q8_mc) { + cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; - // image for qs - img_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q5_0->qs; - CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // image for q + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 4; + img_desc.buffer = extra0_q8_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // create a sub_buffer for B + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + if (use_q8_mc) { + kernel = backend_ctx->kernel_gemv_noshuffle_q8_0_f32_mc; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[] = { wavesize, 4, 1 }; + size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } + + // Split-K for small-M decode GEMVs. The base kernel puts one output row + // per lane and splits K only inside one workgroup, so M is the sole source + // of workgroup parallelism: gpt-oss's K and V projections are M=512 = 8 + // workgroups on a 16-CU X2, and the kernel measures 48 GB/s where the + // M=2880/4096 projections in the same decode graph reach 122-123. Mirrors + // the q4_0/q4_K split-K above and reuses their reduce kernel. + // + // Enabled where it is measured to win, like the q4_K gate: X2-90 +2.8% + // tg32 @d4096 on gpt-oss; Adreno 840 (12 CU) NEUTRAL on Llama-3.2-3B-Q8_0 + // (0.0% @d4096 -- its K/V proj is M=1024 = 16 workgroups, which already + // fills 12 CUs). Unmeasured on X1E/A7X/A6X and the q4_K split-K measured + // -0.7% on X1E, so the default is not widened on absence of evidence. + static const bool q8_splitk_env_set = []{ + const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); + return e && e[0] != '\0'; + }(); + static const bool q8_splitk_env_on = []{ + const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); + return !(e && e[0] == '0'); + }(); + const bool q8_splitk_on = q8_splitk_env_set + ? q8_splitk_env_on + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q8_splitk_on && backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk && + ne01 <= 1024 && ne01 % 64 == 0) { + const int nsg = 8; + const int ksplit = 8; // -> 8 * M/64 workgroups + const size_t gx = (size_t) CEIL_DIV(ne01, 64) * 64; + + backend_ctx->prealloc_splitk_partial.allocate( + backend_ctx->context, (size_t) ksplit * ne01 * sizeof(float)); + cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; + + cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk; + CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_int), &ne01)); + size_t lsk[3] = { 64, (size_t) nsg, 1 }; + size_t gsk[3] = { gx, (size_t) (nsg * ksplit), 1 }; + backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); + + cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; + CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); + size_t lr[3] = { 64, 1, 1 }; + size_t gr[3] = { (size_t) CEIL_DIV(ne01, 64) * 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); + + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } + + kernel = backend_ctx->kernel_gemv_noshuffle_q8_0_f32; + + int r2 = 1; + int r3 = 1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[] = { wavesize, 4, 1 }; + size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + static const char * q8_bin_dp4a_env = getenv("GGML_OPENCL_Q8_0_BIN_DP4A"); + const bool q8_bin_dp4a_on = backend_ctx->has_integer_dot && + (q8_bin_dp4a_env ? (atoi(q8_bin_dp4a_env) != 0) : true); + + if (q8_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin + && M % 64 == 0) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + + const size_t N_pad = ((size_t)N + 31) / 32 * 32; + const size_t n_blocks_pad = N_pad * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_pad * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_pad * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem q_img = nullptr; + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * (size_t)K / 4; + img_desc.buffer = extra0_q8_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + cl_mem d_sub_buf = nullptr; + cl_mem d_img = nullptr; + region.origin = extrad->offset; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q5_0_f32; + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin; + cl_uint ne00_u = (cl_uint)K; + cl_uint ne01_u = (cl_uint)M; + CL_CHECK(clSetKernelArg(dk, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(dk, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(dk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, 4, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(dk, 5, sizeof(cl_uint), &ne00_u)); + CL_CHECK(clSetKernelArg(dk, 6, sizeof(cl_uint), &ne01_u)); + CL_CHECK(clSetKernelArg(dk, 7, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + // dp4a dense q8_0 prefill GEMM. Quantizes the [N,K] activations to + // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch + // (ne1>8) only; q8_0 weights are already int8 (no requant) and symmetric + // (no min term) + static const char * q8_dense_dp4a_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A"); + static const char * q8_dense_wimg_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A_WIMG"); + const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0); - CL_CHECK(clReleaseMemObject(qs_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { - // dp4a (int8) dense q5_0 prefill GEMM, default off - static const char * q5_dense_dp4a_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A"); - static const char * q5_dense_wimg_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A_WIMG"); - const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0); - bool q5_dense_dp4a_on = q5_dense_wimg_on + const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr); + // bin kernel takes precedence + bool q8_dense_dp4a_on = q8_dense_wimg_on ? true - : (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0)); + : q8_dense_dp4a_env + ? (atoi(q8_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded); // dot prod has to be available - q5_dense_dp4a_on = backend_ctx->has_integer_dot && q5_dense_dp4a_on; + q8_dense_dp4a_on = backend_ctx->has_integer_dot && q8_dense_dp4a_on; - if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a + if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { cl_mem a_sub = nullptr; region.origin = offset1; @@ -20260,12 +27264,11 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - // optional qs texture (image1d_buffer over the nibble plane; the same - // CL_R/UINT32 view, width M*K/8, the GEMV path builds). - cl_mem q5_qs_img = nullptr; - bool use_wimg = q5_dense_wimg_on; + // optional weight texture, the same CL_R/UINT32 view, width M*K/4 + cl_mem q8_q_img = nullptr; + bool use_wimg = q8_dense_wimg_on; if (use_wimg) { - const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels (2 ushorts/texel) + const size_t tex = (size_t)M * (size_t)K / 4; // uint32 texels if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { use_wimg = false; } else { @@ -20273,25 +27276,28 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = tex; - img_desc.buffer = extra0_q5_0->qs; - q5_qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); - if (err != CL_SUCCESS || q5_qs_img == nullptr) { use_wimg = false; q5_qs_img = nullptr; } + img_desc.buffer = extra0_q8_0->q; + q8_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q8_q_img == nullptr) { use_wimg = false; q8_q_img = nullptr; } } } - cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg - : backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a; + static const char * q8_0_act_vload4_env = getenv("GGML_OPENCL_Q8_0_DP4A_ACT_VLOAD4"); + const bool q8_0_act_vload4_on = q8_0_act_vload4_env == nullptr || atoi(q8_0_act_vload4_env) != 0; + cl_kernel dk = use_wimg + ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg + : q8_0_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a; int ai = 0; if (use_wimg) { - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q5_qs_img)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img)); } else { - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->q)); } - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qh)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); @@ -20301,199 +27307,110 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - if (q5_qs_img != nullptr) { - CL_CHECK(clReleaseMemObject(q5_qs_img)); + if (q8_q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q8_q_img)); } CL_CHECK(clReleaseMemObject(a_sub)); return; } - cl_mem b_sub_buf = nullptr; - cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; - cl_mem d_sub_buf = nullptr; - - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - // pad N to multiple of 8 - int extra_elements = N % 8; - int padding = 0; - if (extra_elements > 0){ - padding = 8 - extra_elements; - } - - // subbuffer for transposed activations - region.origin = 0; - region.size = K * (N + padding) * sizeof(float)/2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - // image for transposed activations - img_fmt = {CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - - // subbuffer for output - region.origin = extrad->offset; - region.size = M * N * sizeof(float); - CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - // transpose activations - int height_B = N/4; - if (height_B == 0) { - height_B = 1; - } - int width_B = K/4; - int padded_height_B = (N + padding)/4; - - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - - size_t local_work_size_t[2] = { 1, 16 }; - size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q5_0_f32; - int padded_N = N + padding; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_sub_buf)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); - - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {1, 128, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); - CL_CHECK(clReleaseMemObject(d_sub_buf)); - } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); -#endif -} - -static void ggml_cl_mul_mat_q5_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)src0->extra; + // use bin kernel if available + if (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin) { + int K_pad = K; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_mem b_sub_buf = nullptr; + cl_mem d_sub_buf = nullptr; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; + cl_mem a_img = nullptr; + cl_mem s_img = nullptr; + cl_mem b_img = nullptr; + cl_mem d_img = nullptr; - const int ne1 = dst->ne[1]; + // subbuffer for activations + region.origin = offset1; + region.size = K_pad * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + // Create subbuffer and image1d_buffer for dst + region.origin = (extrad->offset); // + dst->view_offs; + region.size = M * N * sizeof(float); + CL_CHECK((d_sub_buf = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_context context = backend_ctx->context; - cl_kernel kernel; + // create an image for A + img_fmt = { CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 4; // Divide by 4 for char -> float + img_desc.buffer = extra0_q8_0->q; + CL_CHECK((a_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; + // create an image for Scale + img_fmt = { CL_R, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 32; // Block size is 32 + img_desc.buffer = extra0_q8_0->d; + CL_CHECK((s_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - int M = ne01; - int N = ne1; - int K = ne00; + // create an image for B from sub_buffer + img_fmt = {CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K_pad * N; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - if (ne1 == 1) { - cl_mem qs_img = nullptr; - cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + // img for d + img_fmt = {CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * N; + img_desc.buffer = d_sub_buf; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // image for qs - img_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q5_1->qs; - CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin; - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + bool layoutA_Mfirst = true; + bool layoutS_Mfirst = true; + bool layoutB_Nfirst = false; + bool layoutC_Mfirst = true; - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + cl_uint lineStrideMatrixAinBytes = layoutA_Mfirst ? M * 4 : K; // int8 + cl_uint lineStrideMatrixSinBytes = layoutS_Mfirst ? M * 2 : (K / 32) * 2; // fp16 + cl_uint lineStrideMatrixBinBytes = layoutB_Nfirst ? N * 4 : K_pad * 4; // fp32 + cl_uint lineStrideMatrixCinBytes = layoutC_Mfirst ? M * 4 : N * 4; // fp32 - kernel = backend_ctx->kernel_gemv_noshuffle_q5_1_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &extra1->offset)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &lineStrideMatrixAinBytes)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &lineStrideMatrixSinBytes)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &lineStrideMatrixBinBytes)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &lineStrideMatrixCinBytes)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + size_t global_work_size[] = { 64, (size_t)CEIL_DIV(M, 64), (size_t)CEIL_DIV(N, 64)}; + size_t local_work_size[] = { 64, 2, 2 }; - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + CL_CHECK(clReleaseMemObject(a_img)); + CL_CHECK(clReleaseMemObject(s_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(d_img)); + return; + } - CL_CHECK(clReleaseMemObject(qs_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; - cl_mem d_sub_buf = nullptr; // subbuffer for activations region.origin = offset1; @@ -20529,11 +27446,6 @@ static void ggml_cl_mul_mat_q5_1_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - // subbuffer for output - region.origin = extrad->offset; - region.size = M * N * sizeof(float); - CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // transpose activations int height_B = N/4; if (height_B == 0) { @@ -20554,30 +27466,28 @@ static void ggml_cl_mul_mat_q5_1_f32_adreno(ggml_backend_t backend, const ggml_t backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q5_1_f32; + kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32; int padded_N = N + padding; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_sub_buf)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {1, 128, 1}; + size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; + size_t local_work_size[] = { 2, 128, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img_trans)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); - CL_CHECK(clReleaseMemObject(d_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); } #else GGML_UNUSED(backend); @@ -20587,8 +27497,9 @@ static void ggml_cl_mul_mat_q5_1_f32_adreno(ggml_backend_t backend, const ggml_t #endif } -static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); @@ -20600,17 +27511,17 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)src0->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - const int ne1 = dst->ne[1]; + const int ne1 = dst->ne[1]; - GGML_ASSERT(ne00 % 32 == 0); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; @@ -20624,194 +27535,257 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml int N = ne1; int K = ne00; - if (ne1 == 1) { - cl_mem q_img = nullptr; + // CoopK handles with _bin use the bin weight layout (uint-transposed q), with K-major Q8_1 activations (K contiguous). + // This switch disables only this bin-layout path; weight conversion is unchanged. + static const char * env_coopk_bin = getenv("GGML_OPENCL_COOPK_Q4_K_BIN"); + const bool use_coopk_bin = !env_coopk_bin || !*env_coopk_bin || atoi(env_coopk_bin) != 0; + if (use_coopk_bin && ne1 >= 2 && ne1 <= 4 && + ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k) && + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 != nullptr && + ((ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 + : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4) != nullptr) { cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for q - img_fmt = { CL_R, CL_UNSIGNED_INT32}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_iq4_nl->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // Allocate all kernel columns; padded columns have independent accumulators and are not stored. + const int cols = (ne1 <= 2) ? 2 : 4; + const size_t size_qa = (size_t)cols * K * sizeof(cl_char); + const size_t n_blocks_padded = (size_t)cols * (K / 32); + const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size; + const size_t size_da_prev = backend_ctx->prealloc_moe_da.size; + backend_ctx->prealloc_moe_qa.allocate(context, size_qa); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half)); + if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) { + const cl_char z8 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer, + &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL)); + } + if (backend_ctx->prealloc_moe_da.size != size_da_prev) { + const cl_half z16 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL)); + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL)); + } + + cl_int n_blocks = (cl_int)((size_t)N * (K / 32)); + cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks)); + size_t local_work_size_quant[1] = { 64 }; + size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst); + + cl_kernel kernel_coopk = (ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 + : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4; + const int nsg_selected = (ne1 <= 2) ? backend_ctx->nsg_coopk_dp4a_q4_k_bin_c2 + : backend_ctx->nsg_coopk_dp4a_q4_k_bin_c4; + const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; + const cl_int n_padded = cols; + CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_k->q)); + CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &n_padded)); + CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel_coopk, 14, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel_coopk, 15, sizeof(cl_uchar), &mask_hi2)); + + // Each width can compile with a different subgroup count. + const size_t nsg = (size_t)nsg_selected; + size_t local_work_size[3] = { 64, nsg, 1 }; + size_t global_work_size[3] = { (size_t)(ne01 / backend_ctx->rows_coopk_dp4a_q4_k), nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - // subbuffer for activations region.origin = offset1; - region.size = K * N * sizeof(float); + region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; + img_fmt = { CL_RGBA, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); - - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { - // dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to - // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch - // (ne1>8) only - static const char * iq4nl_dense_dp4a_env = getenv("GGML_OPENCL_IQ4NL_DENSE_DP4A"); - bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env - ? (atoi(iq4nl_dense_dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + + static const char * q4_k_bin_dp4a_env = getenv("GGML_OPENCL_Q4_K_BIN_DP4A"); + bool q4_k_bin_dp4a_on = q4_k_bin_dp4a_env + ? (atoi(q4_k_bin_dp4a_env) != 0) + : true; // dot prod has to be available - iq4nl_dense_dp4a_on = backend_ctx->has_integer_dot && iq4nl_dense_dp4a_on; + q4_k_bin_dp4a_on = backend_ctx->has_integer_dot && q4_k_bin_dp4a_on; - if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a - && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { - cl_mem a_sub = nullptr; - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + if (q4_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); - const size_t n_blocks = (size_t)N * (K / 32); - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); - cl_int tb = (cl_int)n_blocks; + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a; - int ai = 0; - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->q)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->d)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); - size_t d_local[3] = { 64, 1, 1 }; - size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; - backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - CL_CHECK(clReleaseMemObject(a_sub)); - return; - } + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - cl_mem b_sub_buf = nullptr; - cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; + kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - // pad N to multiple of 8 - int extra_elements = N % 8; - int padding = 0; - if (extra_elements > 0){ - padding = 8 - extra_elements; + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; } - // subbuffer for transposed activations - region.origin = 0; - region.size = K * (N + padding) * sizeof(float)/2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - // image for transposed activations - img_fmt = {CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - - // transpose activations - int height_B = N/4; - if (height_B == 0) { - height_B = 1; + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; + } else { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; } - int width_B = K/4; - int padded_height_B = (N + padding)/4; - - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - - size_t local_work_size_t[2] = { 1, 16 }; - size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32; - int padded_N = N + padding; + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {1, 128, 1}; + kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); -#endif } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -20820,33 +27794,21 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - // SoA extra lives on view_src (view->extra is pre-SoA). - const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; - ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - - const int ne10 = src1->ne[0]; - const int ne12 = src1->ne[2]; - const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; - GGML_ASSERT(ne00 == ne10); - GGML_ASSERT((ne00 % 32) == 0); - GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; @@ -20860,23 +27822,49 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - if (ne1 == 1) { + cl_uchar mask_d6 = 0x3F; + cl_uchar mask_d4 = 0x0F; + cl_uchar mask_hi2 = 0xC0; + + // Multi-column verify GEMV + static const bool q4k_mc3 = (getenv("GGML_OPENCL_Q4_K_MC3") != nullptr); + const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); + + const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4_K_MC3 is bypassed by Q4_K binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_k_f32_adreno_ila(backend, src0, src1, dst); + return; + } + + if (ne1 == 1 || use_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; - // image for q - img_fmt = { CL_R, CL_UNSIGNED_INT32}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 4; - img_desc.buffer = extra0_q8_0->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + const bool use_tiled = !use_mc3 && use_q4k_tiled(backend_ctx, src0); - // create a sub_buffer for B + // image for q (not needed for the tiled path, which reads __global) + if (!use_tiled) { + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_k->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + } + + // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; @@ -20886,48 +27874,91 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // Split-K for small-M decode GEMVs. The base kernel puts one output row - // per lane and splits K only inside one workgroup, so M is the sole source - // of workgroup parallelism: gpt-oss's K and V projections are M=512 = 8 - // workgroups on a 16-CU X2, and the kernel measures 48 GB/s where the - // M=2880/4096 projections in the same decode graph reach 122-123. Mirrors - // the q4_0/q4_K split-K above and reuses their reduce kernel. + // 4-output-per-WI o4 variant for the long-vocab lm_head/embed GEMV + // (ne01 = vocab ~256K on Gemma): shares one activation read across 4 + // output rows. Gated to large ne01 (lm_head/embed). Default on; opt-out + // GGML_OPENCL_Q4K_GEMV_O4=0. (Skipped when mc3 handles the ne1==3 verify.) + static const bool q4k_o4_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_O4"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool use_q4k_o4 = !use_tiled && !use_mc3 && q4k_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); + // Split-K across workgroups for small-M decode GEMVs. A single-token GEMV + // makes only CEIL_DIV(M/2,64) workgroups; even with the wide intra-WG split + // (16 subgroups) those all land on ONE CU, so small-M matmuls under-fill the + // 16 CUs and their bandwidth falls well short of what the large-M FFN matmuls + // reach. Adding a `ksplit` second grid dim that spreads K across WGs (+ a + // reduce pass) fills the CUs. Gate is M<=2560: the tiny M<=1024 ones only + // break even (the reduce dispatch eats the kernel win), but the big-K M=2560 + // cases (ffn_down, attn_output) make the per-call win dwarf the reduce, and + // are byte-identical. ffn_gate/up (large M) fill the CUs already and are excluded. // - // Enabled where it is measured to win, like the q4_K gate: X2-90 +2.8% - // tg32 @d4096 on gpt-oss; Adreno 840 (12 CU) NEUTRAL on Llama-3.2-3B-Q8_0 - // (0.0% @d4096 -- its K/V proj is M=1024 = 16 workgroups, which already - // fills 12 CUs). Unmeasured on X1E/A7X/A6X and the q4_K split-K measured - // -0.7% on X1E, so the default is not widened on absence of evidence. - static const bool q8_splitk_env_set = []{ - const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); + // DEVICE-GATED. Split-K buys GPU time by spending an extra kernel LAUNCH (the + // reduce), so it only pays where launches are cheap. That is a per-device + // property and it does not travel from the X2-90 this was tuned on. Measured + // with one binary, env A/B (tg32, GGML_OPENCL_Q4K_GEMV_SPLITK=0/1): + // + // X2-90 +3.36% gemma-4 E4B (the number this gate was built on) + // 840 -1.3% Qwen3.5-4B-Q4_K_M 14.00 -> 13.85 + // 850 -20.0% Qwen3-1.7B-Q4_K_M 6.97 -> 5.58 (6 interleaved reps) + // + // The kernel is not the problem. On the 850 split-K makes the GPU strictly + // faster -- total busy 537 -> 485 ms, this GEMV 43.7 -> 34.0 us/call (-22%) -- + // and still costs a fifth of decode, because the +3696 reduce dispatches cost + // ~550 us of HOST round-trip each against 2.7 us of GPU work (~200x; that part + // is ~95% host-bound at decode). The 840 pays the same tax at ~42 us/dispatch. + // Break-even needs launch cost below the ~9.7 us/call the split actually saves, + // so this is not a "the 850 is slow" adjustment that a faster part would fix -- + // the 840 is 13x cheaper per launch and still loses. + // + // Enabled where it is measured to win, i.e. X2E only. The X1-85 was measured + // afterwards and is NOT a win either: Qwen3.5-4B-Q4_K_M tg32, split-K off + // 17.98/18.10/18.19 vs on 18.03/17.91/17.97 = -0.7%, so X1E stays excluded on + // evidence rather than on absence of it. Do not widen this without a NEW + // measurement. The env still forces either way so every device stays measurable. + static const bool splitk_env_set = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); return e && e[0] != '\0'; }(); - static const bool q8_splitk_env_on = []{ - const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); + static const bool splitk_env_on = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); return !(e && e[0] == '0'); }(); - const bool q8_splitk_on = q8_splitk_env_set - ? q8_splitk_env_on + const bool splitk_wg_env = splitk_env_set + ? splitk_env_on : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); - if (q8_splitk_on && backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk && - ne01 <= 1024 && ne01 % 64 == 0) { + // Gate: small-M decode GEMVs that under-fill the 16 CUs even with the wide + // intra-WG split (all 16 subgroups land on one CU). M<=2560 covers Kcur/Vcur + // (M=1024), Qcur (2048), attn_output + ffn_down (2560). The tiny ones + // (M<=1024) only break even (reduce dispatch eats the kernel win), but the + // big-K M=2560 cases (ffn_down K=10240 @182us, attn_output @42us) have a + // large per-call win that dwarfs the ~5us reduce, so extending to 2560 nets + // positive end-to-end. ffn_gate/up (M=10240) already fill the CUs -> excluded. + const bool use_splitk = splitk_wg_env && !use_tiled && !use_q4k_o4 && !use_mc3 && ne01 <= 2560; + + if (use_splitk) { const int nsg = 8; - const int ksplit = 8; // -> 8 * M/64 workgroups - const size_t gx = (size_t) CEIL_DIV(ne01, 64) * 64; + const int ksplit = (ne01 <= 512) ? 8 : 4; // -> ~32 total WGs + const size_t gx = (size_t)CEIL_DIV(ne01/2, 64) * 64; backend_ctx->prealloc_splitk_partial.allocate( - backend_ctx->context, (size_t) ksplit * ne01 * sizeof(float)); + backend_ctx->context, (size_t)ksplit * ne01 * sizeof(float)); cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; - cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk; - CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &partial)); - CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_int), &ne01)); - size_t lsk[3] = { 64, (size_t) nsg, 1 }; - size_t gsk[3] = { gx, (size_t) (nsg * ksplit), 1 }; + cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk; + CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(ks, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(ks, 7, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(ks, 8, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(ks, 9, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(ks, 10, sizeof(cl_uchar), &mask_hi2)); + size_t lsk[3] = {64, (size_t)nsg, 1}; + size_t gsk[3] = {gx, (size_t)(nsg * ksplit), 1}; backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; @@ -20936,310 +27967,204 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); - size_t lr[3] = { 64, 1, 1 }; - size_t gr[3] = { (size_t) CEIL_DIV(ne01, 64) * 64, 1, 1 }; + size_t lr[3] = {64, 1, 1}; + size_t gr[3] = {(size_t)CEIL_DIV(ne01, 64) * 64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); - CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_img)); + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); return; } - kernel = backend_ctx->kernel_gemv_noshuffle_q8_0_f32; - - int r2 = 1; - int r3 = 1; + kernel = use_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 + : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled + : use_q4k_o4 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 + : backend_ctx->kernel_gemv_noshuffle_q4_k_f32; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + if (use_tiled) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2)); + } - size_t wavesize = backend_ctx->adreno_wave_size; - size_t local_work_size[] = { wavesize, 4, 1 }; - size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; + // Wide K-split for the decode GEMV: the default 4-subgroup K-split leaves + // each Adreno SP with only ~4 waves, too few to hide LPDDR weight-load + // latency, so even the large FFN matmuls run well below the achievable + // bandwidth. Widen to 16 subgroups/WG (= the 1024-lane Adreno WG max) so + // each SP holds enough in-flight memory requests. Prefill is unaffected (the + // GEMM path is separate) and coherence-identical (greedy output unchanged). + // Applies to the plain base + // GEMV only; tiled/o4/mc3 keep 4 (their reductions are hard-coded to 4). + // Layout-safe: the base kernel derives its K-split from get_local_size(1) + // and the packed block stride is a physical constant (independent of it). + // Opt-out: GGML_OPENCL_Q4K_GEMV_WIDE=0. + static const bool splitk_wide_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_WIDE"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool splitk_wide = splitk_wide_env && !use_tiled && !use_q4k_o4 && !use_mc3; + size_t nsg_y = splitk_wide ? 16 : 4; + // Cap the wide K-split by the kernel's real max WG. X1-class drivers cap + // this GEMV at 768 (< 64*16 = 1024), so an uncapped lws aborts the + // dispatch with CL_INVALID_WORK_GROUP_SIZE (-54) and breaks ALL q4_K + // decode for M>2560. nsg_y is a pure K-split (the base kernel reads it + // from get_local_size(1); the packed block stride is a physical constant), + // so halving it stays coherent — just a narrower split. X2 keeps 16 + // (maxwg 1024); X1 falls to 8. + if (splitk_wide) { + const size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); + while (nsg_y > 4 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } + } + size_t local_work_size[3] = {64, nsg_y, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(use_tiled ? ne01 : (use_q4k_o4 ? ne01/4 : ne01/2), 64)*64, nsg_y, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - } else { - static const char * q8_bin_dp4a_env = getenv("GGML_OPENCL_Q8_0_BIN_DP4A"); - const bool q8_bin_dp4a_on = backend_ctx->has_integer_dot && - (q8_bin_dp4a_env ? (atoi(q8_bin_dp4a_env) != 0) : true); - - if (q8_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin - && M % 64 == 0) { - cl_mem a_sub = nullptr; - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - const size_t n_blocks = (size_t)N * (K / 32); - - const size_t N_pad = ((size_t)N + 31) / 32 * 32; - const size_t n_blocks_pad = N_pad * (K / 32); - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N_pad * K * sizeof(cl_char)); - backend_ctx->prealloc_moe_da.allocate(context, n_blocks_pad * sizeof(cl_half)); - backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_pad * sizeof(cl_half)); - - cl_int tb = (cl_int)n_blocks; - cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); - CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); - size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; - backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - - cl_mem q_img = nullptr; - img_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * (size_t)K / 4; - img_desc.buffer = extra0_q8_0->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - cl_mem d_sub_buf = nullptr; - cl_mem d_img = nullptr; - region.origin = extrad->offset; - region.size = (size_t)M * N * sizeof(float); - CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - img_fmt = { CL_R, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * N; - img_desc.buffer = d_sub_buf; - CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin; - cl_uint ne00_u = (cl_uint)K; - cl_uint ne01_u = (cl_uint)M; - CL_CHECK(clSetKernelArg(dk, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(dk, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(dk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(dk, 4, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(dk, 5, sizeof(cl_uint), &ne00_u)); - CL_CHECK(clSetKernelArg(dk, 6, sizeof(cl_uint), &ne01_u)); - CL_CHECK(clSetKernelArg(dk, 7, sizeof(int), &N)); - - size_t d_local[3] = { 64, 1, 1 }; - size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; - backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - - CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(d_img)); - CL_CHECK(clReleaseMemObject(d_sub_buf)); - CL_CHECK(clReleaseMemObject(a_sub)); - return; - } - - // dp4a dense q8_0 prefill GEMM. Quantizes the [N,K] activations to - // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch - // (ne1>8) only; q8_0 weights are already int8 (no requant) and symmetric - // (no min term) - static const char * q8_dense_dp4a_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A"); - static const char * q8_dense_wimg_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A_WIMG"); - const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0); + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { - const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr); - // bin kernel takes precedence - bool q8_dense_dp4a_on = q8_dense_wimg_on - ? true - : q8_dense_dp4a_env - ? (atoi(q8_dense_dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded); - // dot prod has to be available - q8_dense_dp4a_on = backend_ctx->has_integer_dot && q8_dense_dp4a_on; + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; - if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a - && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { - cl_mem a_sub = nullptr; - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // Only the tiled kernels can read the tiled layout (use_q4k_tiled). + if (use_q4k_tiled(backend_ctx, src0)) { const size_t n_blocks = (size_t)N * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); - cl_int tb = (cl_int)n_blocks; + cl_int tbq = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tbq)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - // optional weight texture, the same CL_R/UINT32 view, width M*K/4 - cl_mem q8_q_img = nullptr; - bool use_wimg = q8_dense_wimg_on; - if (use_wimg) { - const size_t tex = (size_t)M * (size_t)K / 4; // uint32 texels - if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { - use_wimg = false; - } else { - img_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = tex; - img_desc.buffer = extra0_q8_0->q; - q8_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); - if (err != CL_SUCCESS || q8_q_img == nullptr) { use_wimg = false; q8_q_img = nullptr; } - } - } - - cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg - : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a; - int ai = 0; - if (use_wimg) { - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img)); - } else { - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->q)); - } - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); - size_t d_local[3] = { 64, 1, 1 }; - size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; - backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a; + CL_CHECK(clSetKernelArg(dk, 0, sizeof(cl_mem), &extra0_q4_k->q)); + CL_CHECK(clSetKernelArg(dk, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(dk, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(dk, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(dk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, 9, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(dk, 10, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(dk, 11, sizeof(cl_int), &ne1)); + + const int WROWS_D = 4 * 64; + const int BN_DP = 8; + size_t dlocal[3] = {64, 4, 1}; + size_t dglobal[3] = {(size_t)CEIL_DIV(ne01, WROWS_D) * 64, 4, (size_t)CEIL_DIV(ne1, BN_DP)}; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dglobal, dlocal, dst); - if (q8_q_img != nullptr) { - CL_CHECK(clReleaseMemObject(q8_q_img)); - } - CL_CHECK(clReleaseMemObject(a_sub)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); return; } - // use bin kernel if available - if (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin) { - int K_pad = K; - - cl_mem b_sub_buf = nullptr; - cl_mem d_sub_buf = nullptr; - - cl_mem a_img = nullptr; - cl_mem s_img = nullptr; - cl_mem b_img = nullptr; - cl_mem d_img = nullptr; - - // subbuffer for activations - region.origin = offset1; - region.size = K_pad * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - // Create subbuffer and image1d_buffer for dst - region.origin = (extrad->offset); // + dst->view_offs; - region.size = M * N * sizeof(float); - CL_CHECK((d_sub_buf = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - // create an image for A - img_fmt = { CL_R, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 4; // Divide by 4 for char -> float - img_desc.buffer = extra0_q8_0->q; - CL_CHECK((a_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - // create an image for Scale - img_fmt = { CL_R, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 32; // Block size is 32 - img_desc.buffer = extra0_q8_0->d; - CL_CHECK((s_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - // create an image for B from sub_buffer - img_fmt = {CL_R, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K_pad * N; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - // img for d - img_fmt = {CL_R, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * N; - img_desc.buffer = d_sub_buf; - CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin; - - bool layoutA_Mfirst = true; - bool layoutS_Mfirst = true; - bool layoutB_Nfirst = false; - bool layoutC_Mfirst = true; - - cl_uint lineStrideMatrixAinBytes = layoutA_Mfirst ? M * 4 : K; // int8 - cl_uint lineStrideMatrixSinBytes = layoutS_Mfirst ? M * 2 : (K / 32) * 2; // fp16 - cl_uint lineStrideMatrixBinBytes = layoutB_Nfirst ? N * 4 : K_pad * 4; // fp32 - cl_uint lineStrideMatrixCinBytes = layoutC_Mfirst ? M * 4 : N * 4; // fp32 - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &a_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s_img)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &extra1->offset)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &extrad->offset)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &lineStrideMatrixAinBytes)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &lineStrideMatrixSinBytes)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &lineStrideMatrixBinBytes)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &lineStrideMatrixCinBytes)); - - size_t global_work_size[] = { 64, (size_t)CEIL_DIV(M, 64), (size_t)CEIL_DIV(N, 64)}; - size_t local_work_size[] = { 64, 2, 2 }; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + // CoopK handles without _bin use the same untiled weight layout as gemm_noshuffle_q4_k_f32. + // Activations use K-major Q8_1 SoA (K contiguous). + if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k) + && backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4 != nullptr + && ((N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4) != nullptr) { + const int cols = (N <= 2) ? 2 : 4; + const size_t size_qa = (size_t)cols * K * sizeof(cl_char); + const size_t n_blocks_padded = (size_t)cols * (K / 32); + const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size; + const size_t size_da_prev = backend_ctx->prealloc_moe_da.size; + backend_ctx->prealloc_moe_qa.allocate(context, size_qa); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half)); + if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) { + const cl_char z8 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer, + &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL)); + } + if (backend_ctx->prealloc_moe_da.size != size_da_prev) { + const cl_half z16 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL)); + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL)); + } + + cl_int n_blocks = (cl_int)((size_t)N * (K / 32)); + cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks)); + size_t local_work_size_quant[1] = { 64 }; + size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst); + + cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4; + const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; + const cl_int n_padded = cols; + CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_k->q)); + CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &n_padded)); + CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel_coopk, 14, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel_coopk, 15, sizeof(cl_uchar), &mask_hi2)); + + const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q4_k; + size_t local_work_size[3] = { 64, nsg, 1 }; + size_t global_work_size[3] = { (size_t)(M / backend_ctx->rows_coopk_dp4a_q4_k), nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(d_sub_buf)); - CL_CHECK(clReleaseMemObject(a_img)); - CL_CHECK(clReleaseMemObject(s_img)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(d_img)); return; } - cl_mem b_sub_buf = nullptr; - cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; - - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); @@ -21261,56 +28186,232 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for transposed activations - img_fmt = {CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = K/4; + int padded_height_B = (N + padding)/4; + + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + + // dp4a (int8) dense prefill GEMM and weight via texture + static const char * q4k_dense_dp4a_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A"); + static const char * q4k_dense_wimg_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A_WIMG"); + + const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0); + bool q4k_dense_dp4a_on = q4k_dense_wimg_on + ? true + : q4k_dense_dp4a_env + ? (atoi(q4k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + + // dp4 has to be available + q4k_dense_dp4a_on = backend_ctx->has_integer_dot && q4k_dense_dp4a_on; + + // Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8 + static const char * q4k_dp4a_minn_env = getenv("GGML_OPENCL_Q4K_DP4A_MINN"); + const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9; + + if (q4k_dense_dp4a_on && N >= q4k_dp4a_minn && (K % 32 == 0) && (M % 64 == 0)) { + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + // check if weights go through texture + cl_mem q4k_q_img = nullptr; + bool use_wimg = q4k_dense_wimg_on; + if (use_wimg) { + const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels = bytes/4 + if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { + use_wimg = false; + } else { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = tex; + img_desc.buffer = extra0_q4_k->q; + q4k_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q4k_q_img == nullptr) { + use_wimg = false; + q4k_q_img = nullptr; + } + } + } + + const bool use_narrow = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow != nullptr + && N <= backend_ctx->max_n_dp4a_narrow_q4_k; + static const char * q4_k_act_vload4_env = getenv("GGML_OPENCL_Q4_K_DP4A_ACT_VLOAD4"); + const bool q4_k_act_vload4_on = q4_k_act_vload4_env == nullptr || atoi(q4_k_act_vload4_env) != 0; + cl_kernel dk = use_narrow + ? (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg + : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow) + : use_wimg + ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg + : q4_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a; + int ai = 0; + if (use_wimg) { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img)); + } else { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->q)); + } + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); + const int tile_n = use_narrow ? backend_ctx->tile_n_dp4a_narrow_q4_k : backend_ctx->tile_n_dp4a_q4_k; + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, tile_n) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - // transpose activations - int height_B = N/4; - if (height_B == 0) { - height_B = 1; + if (q4k_q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q4k_q_img)); + } + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + return; } - int width_B = K/4; - int padded_height_B = (N + padding)/4; - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + // gemm + // Small-batch (medium n_q) occupancy fix: at ne1<=8 the 2x8 grid is + // (1, ceil(M/2)) -> ~M/256 workgroups, which under-occupies the SP and + // makes the GEMM much slower than the ne1==1 GEMV at the same weight + // traffic. The _r1 (1-row) kernel doubles the M-axis workgroup count + // and removes the accumulator spill. Opt-in via env while validating. + static const bool q4k_gemm_r1 = (getenv("GGML_OPENCL_Q4K_GEMM_R1") != nullptr); + static const bool q4k_gemm_kimg = (getenv("GGML_OPENCL_Q4K_GEMM_KIMG") != nullptr); + // Cooperative-K (intra-WG K-split + reduction) for the small-batch + // (n_q in [2..8]) path: DEFAULT ON, opt out with GGML_OPENCL_Q4K_GEMM_COK=0. + // Byte-identical greedy output; large-batch (ne1>8) untouched. + static const char * q4k_cok_env = getenv("GGML_OPENCL_Q4K_GEMM_COK"); + static const bool q4k_gemm_cok = (q4k_cok_env == nullptr) || (atoi(q4k_cok_env) != 0); + const bool use_cok = q4k_gemm_cok && (ne1 <= 8); + const bool use_r1 = !use_cok && q4k_gemm_r1 && (ne1 <= 8); + // Weights-as-image (L1/TPL1) for the small-batch weight-read-bound path. + const bool use_kimg = !use_cok && !use_r1 && q4k_gemm_kimg && (ne1 <= 8); - size_t local_work_size_t[2] = { 1, 16 }; - size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + cl_mem q_img = nullptr; + if (use_kimg) { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_k->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + } - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32; + kernel = use_cok ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok + : use_r1 ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 + : use_kimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg + : backend_ctx->kernel_gemm_noshuffle_q4_k_f32; int padded_N = N + padding; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + if (use_kimg) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); + } + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_hi2)); - size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; - size_t local_work_size[] = { 2, 128, 1 }; + size_t global_work_size[3]; + size_t local_work_size[3]; + if (use_cok) { + // (COK_SG lanes x COK_NSG subgroups): one row per lane, K split + // across the COK_NSG subgroups. ne01 is a multiple of 64. + global_work_size[0] = (size_t)ne01; // rows + global_work_size[1] = 8; // COK_NSG + global_work_size[2] = 1; + local_work_size[0] = 64; // COK_SG + local_work_size[1] = 8; // COK_NSG + local_work_size[2] = 1; + } else if (use_r1) { + // 1 row per WI (opt-in occupancy experiment). + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)ne01; + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } else if (use_kimg) { + // kimg is a 2-row tile (opt-in weights-as-image experiment). + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 2); + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } else { + // Default: x2-unified base kernel is the 4-row (gx<<2) tile. + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - - CL_CHECK(clReleaseMemObject(b_img_trans)); + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img_trans)); } #else GGML_UNUSED(backend); @@ -21321,7 +28422,7 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, +static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -21332,9 +28433,9 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; @@ -21349,18 +28450,18 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg cl_context context = backend_ctx->context; cl_kernel kernel; - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; + cl_int err; + cl_buffer_region region; + cl_image_format img_fmt; + cl_image_desc img_desc; - int M = ne01; - int N = ne1; - int K = ne00; + const int M = ne01; + const int N = ne1; + const int K = ne00; if (ne1 == 1) { - cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); @@ -21373,11 +28474,11 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + kernel = backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); @@ -21388,20 +28489,20 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); } else { const int gemm_tile_n = 64; int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; - static const char * q4_k_bin_dp4a_env = getenv("GGML_OPENCL_Q4_K_BIN_DP4A"); - bool q4_k_bin_dp4a_on = q4_k_bin_dp4a_env - ? (atoi(q4_k_bin_dp4a_env) != 0) + static const char * q6_k_bin_dp4a_env = getenv("GGML_OPENCL_Q6_K_BIN_DP4A"); + bool q6_k_bin_dp4a_on = q6_k_bin_dp4a_env + ? (atoi(q6_k_bin_dp4a_env) != 0) : true; // dot prod has to be available - q4_k_bin_dp4a_on = backend_ctx->has_integer_dot && q4_k_bin_dp4a_on; + q6_k_bin_dp4a_on = backend_ctx->has_integer_dot && q6_k_bin_dp4a_on; - if (q4_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin) { + if (q6_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin) { const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); @@ -21438,20 +28539,19 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg img_desc.buffer = d_sub; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; + kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->q_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); size_t local_work_size[3] = { 64, 1, 1 }; size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; @@ -21499,11 +28599,11 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg cl_mem d_img; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); @@ -21528,7 +28628,7 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg } #endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -21539,524 +28639,660 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - const int ne1 = dst->ne[1]; + const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - cl_context context = backend_ctx->context; - cl_kernel kernel; - - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; - - int M = ne01; - int N = ne1; - int K = ne00; + cl_context context = backend_ctx->context; + cl_kernel kernel; - cl_uchar mask_d6 = 0x3F; - cl_uchar mask_d4 = 0x0F; - cl_uchar mask_hi2 = 0xC0; + cl_int err; + cl_buffer_region region; + cl_image_format img_fmt; + cl_image_desc img_desc; - // Multi-column verify GEMV: route the spec/MTP verify batch (ne1==3 = 2 - // drafts + 1 bonus) onto the efficient GEMV path (subgroup-broadcast, no - // transpose) instead of the transposed-GEMM dead-zone. Reuses the ne1==1 - // GEMV setup (the activation image is already sized by N=ne1). Byte- - // identical. Opt-in via GGML_OPENCL_Q4K_MC3=1 while validating. - static const bool q4k_mc3 = (getenv("GGML_OPENCL_Q4K_MC3") != nullptr); - // Per-layer only (ne01 < 32768): the batched large-vocab lm_head at ne1==3 - // is left to the existing routing (corrupts on the Adreno GEMV path; x2- - // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical. - const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); + // Multi-column verify GEMV + static const bool q6k_mc3 = (getenv("GGML_OPENCL_Q6_K_MC3") != nullptr); + const bool use_q6k_mc3 = q6k_mc3 && (ne1 == 3) && (ne01 < 32768); + // Multi-column tiled vocab GEMV for verify + // streams the large lm_head weight once across the verify columns + static const char * q6k_tiled_mc_env = getenv("GGML_OPENCL_Q6_K_TILED_MC"); + const bool q6k_tiled_mc = q6k_tiled_mc_env ? atoi(q6k_tiled_mc_env) != 0 + : (q6k_mc3 || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const bool use_q6k_tiled_mc = q6k_tiled_mc && (ne1 >= 2 && ne1 <= 4) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0); - const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); + const bool use_bin = use_q6_k_bin_kernels(backend_ctx, src0); if (use_bin) { - if (use_mc3) { + if (use_q6k_mc3 || use_q6k_tiled_mc) { static bool warned = false; if (!warned) { - GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4K_MC3 is bypassed by Q4_K binary kernels\n"); + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q6_K_MC3 is bypassed by Q6_K binary kernels\n"); warned = true; } } - ggml_cl_mul_mat_q4_k_f32_adreno_ila(backend, src0, src1, dst); + // This bin layout is not supported by CoopK. + ggml_cl_mul_mat_q6_K_f32_adreno_ila(backend, src0, src1, dst); return; } - if (ne1 == 1 || use_mc3) { - cl_mem q_img = nullptr; - cl_mem b_sub_buf = nullptr; + if (ne1 == 1 || use_q6k_mc3 || use_q6k_tiled_mc) { + cl_mem ql_img = nullptr; + cl_mem qh_img = nullptr; + cl_mem b_sub_buffer = nullptr; cl_mem b_img = nullptr; - const bool use_tiled = !use_mc3 && use_q4k_tiled(backend_ctx, src0); + // o4 = 4-output-per-WI variant for long-vocab lm_head/embed; gated to + // ne01 >= 32768 so per-layer q6_K (ne01=hidden 2-8K) keeps the 2-output + // kernel (o4 regresses there). o4_global reads the weights from __global + // coalesced instead of image1d_buffer -- the texture cache caps the + // read-once-per-token lm_head bandwidth, while __global reaches the higher + // rate the rest of the model gets. Both default ON; opt out via + // GGML_OPENCL_Q6K_GEMV_O4 / GGML_OPENCL_Q6K_GEMV_O4_GLOBAL = 0. + static const bool gemv_o4_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + static const bool o4_global_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4_GLOBAL"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool use_tiled = !use_q6k_mc3 && use_q6k_tiled(backend_ctx, src0); + const bool use_o4 = !use_tiled && !use_q6k_mc3 && gemv_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); + const bool use_o4_global = use_o4 && o4_global_env; - // image for q (not needed for the tiled path, which reads __global) - if (!use_tiled) { - img_fmt = { CL_R, CL_UNSIGNED_INT32}; + // ql/qh image views are only needed when NOT reading weights from global. + if (!use_o4_global && !use_tiled) { + // image for ql + img_fmt.image_channel_order = CL_R; + img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q4_k->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_desc.image_width = ne01 * ne00 / 8; + img_desc.buffer = extra0_q6_K->ql; + CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // image for qh + img_fmt.image_channel_order = CL_R; + img_fmt.image_channel_data_type = CL_HALF_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne01 * ne00 / 8; + img_desc.buffer = extra0_q6_K->qh; + CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); } - // subbuffer for activations region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + region.size = ne00 * ne1 * sizeof(float); + CL_CHECK((b_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; + img_desc.image_width = ne00 * ne1 / 4; + img_desc.buffer = b_sub_buffer; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // 4-output-per-WI o4 variant for the long-vocab lm_head/embed GEMV - // (ne01 = vocab ~256K on Gemma): shares one activation read across 4 - // output rows. Gated to large ne01 (lm_head/embed). Default on; opt-out - // GGML_OPENCL_Q4K_GEMV_O4=0. (Skipped when mc3 handles the ne1==3 verify.) - static const bool q4k_o4_env = []{ - const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_O4"); - return !e || e[0] == '\0' || e[0] != '0'; - }(); - const bool use_q4k_o4 = !use_tiled && !use_mc3 && q4k_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); - // Split-K across workgroups for small-M decode GEMVs. A single-token GEMV - // makes only CEIL_DIV(M/2,64) workgroups; even with the wide intra-WG split - // (16 subgroups) those all land on ONE CU, so small-M matmuls under-fill the - // 16 CUs and their bandwidth falls well short of what the large-M FFN matmuls - // reach. Adding a `ksplit` second grid dim that spreads K across WGs (+ a - // reduce pass) fills the CUs. Gate is M<=2560: the tiny M<=1024 ones only - // break even (the reduce dispatch eats the kernel win), but the big-K M=2560 - // cases (ffn_down, attn_output) make the per-call win dwarf the reduce, and - // are byte-identical. ffn_gate/up (large M) fill the CUs already and are excluded. - // - // DEVICE-GATED. Split-K buys GPU time by spending an extra kernel LAUNCH (the - // reduce), so it only pays where launches are cheap. That is a per-device - // property and it does not travel from the X2-90 this was tuned on. Measured - // with one binary, env A/B (tg32, GGML_OPENCL_Q4K_GEMV_SPLITK=0/1): - // - // X2-90 +3.36% gemma-4 E4B (the number this gate was built on) - // 840 -1.3% Qwen3.5-4B-Q4_K_M 14.00 -> 13.85 - // 850 -20.0% Qwen3-1.7B-Q4_K_M 6.97 -> 5.58 (6 interleaved reps) - // - // The kernel is not the problem. On the 850 split-K makes the GPU strictly - // faster -- total busy 537 -> 485 ms, this GEMV 43.7 -> 34.0 us/call (-22%) -- - // and still costs a fifth of decode, because the +3696 reduce dispatches cost - // ~550 us of HOST round-trip each against 2.7 us of GPU work (~200x; that part - // is ~95% host-bound at decode). The 840 pays the same tax at ~42 us/dispatch. - // Break-even needs launch cost below the ~9.7 us/call the split actually saves, - // so this is not a "the 850 is slow" adjustment that a faster part would fix -- - // the 840 is 13x cheaper per launch and still loses. - // - // Enabled where it is measured to win, i.e. X2E only. The X1-85 was measured - // afterwards and is NOT a win either: Qwen3.5-4B-Q4_K_M tg32, split-K off - // 17.98/18.10/18.19 vs on 18.03/17.91/17.97 = -0.7%, so X1E stays excluded on - // evidence rather than on absence of it. Do not widen this without a NEW - // measurement. The env still forces either way so every device stays measurable. - static const bool splitk_env_set = []{ - const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); - return e && e[0] != '\0'; - }(); - static const bool splitk_env_on = []{ - const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); - return !(e && e[0] == '0'); - }(); - const bool splitk_wg_env = splitk_env_set - ? splitk_env_on - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); - // Gate: small-M decode GEMVs that under-fill the 16 CUs even with the wide - // intra-WG split (all 16 subgroups land on one CU). M<=2560 covers Kcur/Vcur - // (M=1024), Qcur (2048), attn_output + ffn_down (2560). The tiny ones - // (M<=1024) only break even (reduce dispatch eats the kernel win), but the - // big-K M=2560 cases (ffn_down K=10240 @182us, attn_output @42us) have a - // large per-call win that dwarfs the ~5us reduce, so extending to 2560 nets - // positive end-to-end. ffn_gate/up (M=10240) already fill the CUs -> excluded. - const bool use_splitk = splitk_wg_env && !use_tiled && !use_q4k_o4 && !use_mc3 && ne01 <= 2560; + kernel = use_q6k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 + : use_q6k_tiled_mc ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 + : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled + : use_o4_global ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global + : use_o4 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 + : backend_ctx->kernel_gemv_noshuffle_q6_K_f32; - if (use_splitk) { - const int nsg = 8; - const int ksplit = (ne01 <= 512) ? 8 : 4; // -> ~32 total WGs - const size_t gx = (size_t)CEIL_DIV(ne01/2, 64) * 64; + if (use_o4_global || use_tiled) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); + } + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + if (use_q6k_tiled_mc) { + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); + } - backend_ctx->prealloc_splitk_partial.allocate( - backend_ctx->context, (size_t)ksplit * ne01 * sizeof(float)); - cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; + const size_t gws_x = use_tiled + ? (size_t) CEIL_DIV(ne01, 64) * 64 + : use_o4 + ? (size_t) CEIL_DIV(ne01/4, 64) * 64 + : (size_t) CEIL_DIV(ne01/2, 64) * 64; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {gws_x, 4, 1}; - cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk; - CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_mem), &partial)); - CL_CHECK(clSetKernelArg(ks, 6, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(ks, 7, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(ks, 8, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(ks, 9, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(ks, 10, sizeof(cl_uchar), &mask_hi2)); - size_t lsk[3] = {64, (size_t)nsg, 1}; - size_t gsk[3] = {gx, (size_t)(nsg * ksplit), 1}; - backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; - CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial)); - CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); - size_t lr[3] = {64, 1, 1}; - size_t gr[3] = {(size_t)CEIL_DIV(ne01, 64) * 64, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); + if (ql_img) CL_CHECK(clReleaseMemObject(ql_img)); + if (qh_img) CL_CHECK(clReleaseMemObject(qh_img)); + CL_CHECK(clReleaseMemObject(b_sub_buffer)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + // Tiled-layout batched GEMM. When the weight was converted to the 64-row + // tiled canonical layout (use_q6k_tiled — the default for lm_head/embed), + // the plain noshuffle GEMM below reads it as plain-transposed and produces + // garbage. Use the batched GEMM that matches the decode tiled GEMV's + // layout; it reads the f32 activation directly (column-major, no transpose). + if (use_q6k_tiled(backend_ctx, src0)) { + cl_mem b_sub_buf_t = nullptr; + cl_mem b_img_t = nullptr; + + region.origin = offset1; + region.size = ne00 * ne1 * sizeof(float); + CL_CHECK((b_sub_buf_t = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne00 * ne1 / 4; + img_desc.buffer = b_sub_buf_t; + CL_CHECK((b_img_t = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + cl_kernel kt = backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled; + CL_CHECK(clSetKernelArg(kt, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kt, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kt, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kt, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kt, 4, sizeof(cl_mem), &b_img_t)); + CL_CHECK(clSetKernelArg(kt, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kt, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kt, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kt, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kt, 9, sizeof(int), &ne1)); + + // Must match the kernel: NTILES=4 64-row tiles per work-group (256 rows), + // BN=8 output columns per work-group. + const int BN_T = 16; + const int WROWS = 4 * 64; // NTILES * TILE_ROWS + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01, WROWS) * 64, 4, (size_t)CEIL_DIV(ne1, BN_T)}; + backend_ctx->enqueue_ndrange_kernel(kt, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img_t)); + CL_CHECK(clReleaseMemObject(b_sub_buf_t)); + return; + } + + cl_mem b_sub_buf; + cl_mem b_buf_trans; + cl_mem b_img; + cl_mem b_img_trans; + + // subbuffer for activation + region.origin = offset1; + region.size = ne00 * ne1 * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // CoopK uses the same untiled weight layout as gemm_noshuffle_q6_K_f32, with K-major Q8_1 activations (K contiguous). + // Keep output and embedding weights on the existing path for logit stability. + if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q6_k) + && backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4 != nullptr + && strncmp(src0->name, "output", 6) != 0 + && strncmp(src0->name, "token_embd", 10) != 0 + && ((ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4) != nullptr) { + const int N = (int)ne1, K = (int)ne00; + const int cols = (N <= 2) ? 2 : 4; + const size_t n_blocks_padded = (size_t)cols * (K / 32); + const size_t size_qa = (size_t)cols * K * sizeof(cl_char); + const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size; + const size_t size_da_prev = backend_ctx->prealloc_moe_da.size; + backend_ctx->prealloc_moe_qa.allocate(context, size_qa); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half)); + if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) { + const cl_char z8 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer, + &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL)); + } + if (backend_ctx->prealloc_moe_da.size != size_da_prev) { + const cl_half z16 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL)); + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL)); + } + + cl_int n_blocks = (cl_int)((size_t)N * (K / 32)); + cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks)); + size_t local_work_size_quant[1] = { 64 }; + size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst); + + cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4; + const cl_ushort mask_f000 = 0xF000; + const cl_uchar mask_c0 = 0xC0; + const cl_int n_padded = cols; + const cl_int m = (cl_int)ne01, k = (cl_int)ne00, n = (cl_int)ne1; + CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_int), &m)); + CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &n_padded)); + CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &k)); + CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &n)); + CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_ushort), &mask_f000)); + CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_c0)); + + const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q6_k; + size_t local_work_size[3] = { 64, nsg, 1 }; + size_t global_work_size[3] = { (size_t)(ne01 / backend_ctx->rows_coopk_dp4a_q6_k), nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst); - if (q_img) CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); return; } - kernel = use_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 - : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled - : use_q4k_o4 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 - : backend_ctx->kernel_gemv_noshuffle_q4_k_f32; - - if (use_tiled) { - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - } else { - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2)); - } + // dp4a (int8) dense q6_K prefill GEMM + static const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A"); + bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr) + ? (atoi(q6k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E); + // dot prod has to be available + q6k_dense_dp4a_on = backend_ctx->has_integer_dot && q6k_dense_dp4a_on; - // Wide K-split for the decode GEMV: the default 4-subgroup K-split leaves - // each Adreno SP with only ~4 waves, too few to hide LPDDR weight-load - // latency, so even the large FFN matmuls run well below the achievable - // bandwidth. Widen to 16 subgroups/WG (= the 1024-lane Adreno WG max) so - // each SP holds enough in-flight memory requests. Prefill is unaffected (the - // GEMM path is separate) and coherence-identical (greedy output unchanged). - // Applies to the plain base - // GEMV only; tiled/o4/mc3 keep 4 (their reductions are hard-coded to 4). - // Layout-safe: the base kernel derives its K-split from get_local_size(1) - // and the packed block stride is a physical constant (independent of it). - // Opt-out: GGML_OPENCL_Q4K_GEMV_WIDE=0. - static const bool splitk_wide_env = []{ - const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_WIDE"); - return !e || e[0] == '\0' || e[0] != '0'; - }(); - const bool splitk_wide = splitk_wide_env && !use_tiled && !use_q4k_o4 && !use_mc3; - size_t nsg_y = splitk_wide ? 16 : 4; - // Cap the wide K-split by the kernel's real max WG. X1-class drivers cap - // this GEMV at 768 (< 64*16 = 1024), so an uncapped lws aborts the - // dispatch with CL_INVALID_WORK_GROUP_SIZE (-54) and breaks ALL q4_K - // decode for M>2560. nsg_y is a pure K-split (the base kernel reads it - // from get_local_size(1); the packed block stride is a physical constant), - // so halving it stays coherent — just a narrower split. X2 keeps 16 - // (maxwg 1024); X1 falls to 8. - if (splitk_wide) { - const size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); - while (nsg_y > 4 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } - } - size_t local_work_size[3] = {64, nsg_y, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(use_tiled ? ne01 : (use_q4k_o4 ? ne01/4 : ne01/2), 64)*64, nsg_y, 1}; + const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 || + strncmp(src0->name, "token_embd", 10) == 0; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + if (q6k_dense_dp4a_on && !is_output_w_dp4a && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { + const int M = ne01, N = ne1, K = ne00; + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); - if (q_img) CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - cl_mem b_sub_buf = nullptr; - cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; + static const char * q6_k_act_vload4_env = getenv("GGML_OPENCL_Q6_K_DP4A_ACT_VLOAD4"); + const bool q6_k_act_vload4_on = q6_k_act_vload4_env == nullptr || atoi(q6_k_act_vload4_env) != 0; + cl_kernel dk = q6_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; + // image for activation + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; + img_desc.image_width = ne00 * ne1 / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 - int extra_elements = N % 8; + int extra_elements = ne1 % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } - // subbuffer for transposed activations - region.origin = 0; - region.size = K * (N + padding) * sizeof(float)/2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // subbuffer for transposed activation + region.origin = 0; + region.size = ne00 * (ne1 + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for transposed activation + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_HALF_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne00 * (ne1 + padding) / 4; + img_desc.buffer = b_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + + // transpose activation + int height_B = ne1/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = ne00/4; + int padded_height_B = (ne1 + padding) / 4; + + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + + size_t local_size_t[2] = { 1, 16 }; + size_t global_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst); + + // gemm + // Cooperative-K small-batch (n_q in [2..8]) path: intra-WG K-split, + // mirrors the q4_K _cok path (batched serving). OPT-IN + // (GGML_OPENCL_Q6K_GEMM_COK=1), DEFAULT OFF: q6_K is the tied lm_head/ + // output projection, so the K-reassociation perturbs final logits and + // greedy is NOT byte-identical (op-tests pass, output coherent, but not + // bit-exact). It is also NEUTRAL on end-to-end MTP (q4_K cok already + // captured that; the MTP bottleneck moved off the GEMMs). Keep opt-in + // for batched serving until PPL-validated on a non-GDN q6_K model. + static const char * q6k_cok_env = getenv("GGML_OPENCL_Q6K_GEMM_COK"); + static const bool q6k_gemm_cok = (q6k_cok_env != nullptr) && (atoi(q6k_cok_env) != 0); + const bool use_q6k_cok = q6k_gemm_cok && (ne1 <= 8); + kernel = use_q6k_cok ? backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok + : backend_ctx->kernel_gemm_noshuffle_q6_K_f32; + int padded_N = ne1 + padding; + + cl_ushort mask_f000 = 0xF000; + cl_uchar mask_c0 = 0xC0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ushort),&mask_f000)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_c0)); + + size_t global_work_size[3]; + size_t local_work_size[3]; + if (use_q6k_cok) { + global_work_size[0] = (size_t)ne01; // rows (1 per lane) + global_work_size[1] = 8; // COK_NSG + global_work_size[2] = 1; + local_work_size[0] = 64; // COK_SG + local_work_size[1] = 8; // COK_NSG + local_work_size[2] = 1; + } else { + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); + global_work_size[2] = 1; + local_work_size[0] = 2; + local_work_size[1] = 128; + local_work_size[2] = 1; + } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif +} + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q5_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; - // image for transposed activations - img_fmt = {CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + cl_int err; + cl_buffer_region region; + cl_image_format img_fmt; + cl_image_desc img_desc; - // transpose activations - int height_B = N/4; - if (height_B == 0) { - height_B = 1; - } - int width_B = K/4; - int padded_height_B = (N + padding)/4; + const int M = ne01; + const int N = ne1; + const int K = ne00; - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - size_t local_work_size_t[2] = { 1, 16 }; - size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // dp4a (int8) dense prefill GEMM and weight via texture - static const char * q4k_dense_dp4a_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A"); - static const char * q4k_dense_wimg_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A_WIMG"); + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0); - bool q4k_dense_dp4a_on = q4k_dense_wimg_on - ? true - : q4k_dense_dp4a_env - ? (atoi(q4k_dense_dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + kernel = backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); - // dp4 has to be available - q4k_dense_dp4a_on = backend_ctx->has_integer_dot && q4k_dense_dp4a_on; + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - // Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8 - static const char * q4k_dp4a_minn_env = getenv("GGML_OPENCL_Q4K_DP4A_MINN"); - const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9; + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + static const char * q5_k_bin_dp4a_env = getenv("GGML_OPENCL_Q5_K_BIN_DP4A"); + bool q5_k_bin_dp4a_on = q5_k_bin_dp4a_env + ? (atoi(q5_k_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q5_k_bin_dp4a_on = backend_ctx->has_integer_dot && q5_k_bin_dp4a_on; - if (q4k_dense_dp4a_on && N >= q4k_dp4a_minn && (K % 32 == 0) && (M % 64 == 0)) { - const size_t n_blocks = (size_t)N * (K / 32); - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + if (q5_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); - cl_int tb = (cl_int)n_blocks; + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - // check if weights go through texture - cl_mem q4k_q_img = nullptr; - bool use_wimg = q4k_dense_wimg_on; - if (use_wimg) { - const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels = bytes/4 - if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { - use_wimg = false; - } else { - img_fmt = { CL_R, CL_UNSIGNED_INT32 }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = tex; - img_desc.buffer = extra0_q4_k->q; - q4k_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); - if (err != CL_SUCCESS || q4k_q_img == nullptr) { - use_wimg = false; - q4k_q_img = nullptr; - } - } - } - - cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg - : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a; - int ai = 0; - if (use_wimg) { - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img)); - } else { - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->q)); - } - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); - // Must match the compile-time TILESIZE_N chosen at program build (per-device, - // X1E=8 else 32; env override). Same inputs -> same value. - int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32; - if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) q4k_dp4a_ts = atoi(e); - size_t d_local[3] = { 64, 1, 1 }; - size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, q4k_dp4a_ts) }; - backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - - if (q4k_q_img != nullptr) { - CL_CHECK(clReleaseMemObject(q4k_q_img)); - } - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); - return; - } - - // gemm - // Small-batch (medium n_q) occupancy fix: at ne1<=8 the 2x8 grid is - // (1, ceil(M/2)) -> ~M/256 workgroups, which under-occupies the SP and - // makes the GEMM much slower than the ne1==1 GEMV at the same weight - // traffic. The _r1 (1-row) kernel doubles the M-axis workgroup count - // and removes the accumulator spill. Opt-in via env while validating. - static const bool q4k_gemm_r1 = (getenv("GGML_OPENCL_Q4K_GEMM_R1") != nullptr); - static const bool q4k_gemm_kimg = (getenv("GGML_OPENCL_Q4K_GEMM_KIMG") != nullptr); - // Cooperative-K (intra-WG K-split + reduction) for the small-batch - // (n_q in [2..8]) path: DEFAULT ON, opt out with GGML_OPENCL_Q4K_GEMM_COK=0. - // Byte-identical greedy output; large-batch (ne1>8) untouched. - static const char * q4k_cok_env = getenv("GGML_OPENCL_Q4K_GEMM_COK"); - static const bool q4k_gemm_cok = (q4k_cok_env == nullptr) || (atoi(q4k_cok_env) != 0); - const bool use_cok = q4k_gemm_cok && (ne1 <= 8); - const bool use_r1 = !use_cok && q4k_gemm_r1 && (ne1 <= 8); - // Weights-as-image (L1/TPL1) for the small-batch weight-read-bound path. - const bool use_kimg = !use_cok && !use_r1 && q4k_gemm_kimg && (ne1 <= 8); + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_mem q_img = nullptr; - if (use_kimg) { - img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q4_k->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - } - - kernel = use_cok ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok - : use_r1 ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 - : use_kimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg - : backend_ctx->kernel_gemm_noshuffle_q4_k_f32; - int padded_N = N + padding; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - if (use_kimg) { - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - } else { - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); - } - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_hi2)); + kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin; - size_t global_work_size[3]; - size_t local_work_size[3]; - if (use_cok) { - // (COK_SG lanes x COK_NSG subgroups): one row per lane, K split - // across the COK_NSG subgroups. ne01 is a multiple of 64. - global_work_size[0] = (size_t)ne01; // rows - global_work_size[1] = 8; // COK_NSG - global_work_size[2] = 1; - local_work_size[0] = 64; // COK_SG - local_work_size[1] = 8; // COK_NSG - local_work_size[2] = 1; - } else if (use_r1) { - // 1 row per WI (opt-in occupancy experiment). - global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); - global_work_size[1] = (size_t)ne01; - global_work_size[2] = 1; - local_work_size[0] = 1; - local_work_size[1] = 128; - local_work_size[2] = 1; - } else if (use_kimg) { - // kimg is a 2-row tile (opt-in weights-as-image experiment). - global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); - global_work_size[1] = (size_t)CEIL_DIV(ne01, 2); - global_work_size[2] = 1; - local_work_size[0] = 1; - local_work_size[1] = 128; - local_work_size[2] = 1; + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); + + size_t local_work_size_dp4a[3] = { 64, 1, 1 }; + size_t global_work_size_dp4a[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size_dp4a, local_work_size_dp4a, dst); + + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } + + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; } else { - // Default: x2-unified base kernel is the 4-row (gx<<2) tile. - global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); - global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); - global_work_size[2] = 1; - local_work_size[0] = 1; - local_work_size[1] = 128; - local_work_size[2] = 1; + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; } + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - if (q_img) CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); -#endif } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); @@ -22066,17 +29302,16 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const gg ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; - - const int ne1 = dst->ne[1]; + const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); @@ -22084,1867 +29319,3075 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const gg cl_kernel kernel; cl_int err; - cl_buffer_region region; cl_image_format img_fmt; cl_image_desc img_desc; + cl_buffer_region region; - const int M = ne01; - const int N = ne1; - const int K = ne00; + int M = ne01; + int N = ne1; + int K = ne00; - if (ne1 == 1) { - cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + cl_uchar mask_d6 = 0x3F; + cl_uchar mask_d4 = 0x0F; + cl_uchar mask_hi2 = 0xC0; + + // Multi-column (N=3) verify GEMV for q5_K: route the spec/MTP verify batch + // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- + // zone (gemm_noshuffle_q5_k, the #2 chunk of MTP decode on a Q4_0-mix model + // after q4_0 mc3). Reuses the ne1==1 GEMV image setup (q + qh + activations). + // Default on for X2E, GGML_OPENCL_Q5_K_MC3=0/1 forces it. Per-layer only (ne01 < 32768). + static const char * q5k_mc3_env = getenv("GGML_OPENCL_Q5_K_MC3"); + const bool q5k_mc3 = q5k_mc3_env ? atoi(q5k_mc3_env) != 0 : backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E; + const bool use_q5k_mc3 = q5k_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + + const bool use_bin = use_q5_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_q5k_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q5_K_MC3 is bypassed by Q5_K binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q5_K_f32_adreno_ila(backend, src0, src1, dst); + return; + } + + if (ne1 == 1 || use_q5k_mc3) { + cl_mem q_img = nullptr; + cl_mem qh_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + // image for q (CL_R, CL_UNSIGNED_INT32): width = M*K/2/4 + img_fmt = {CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q5_k->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // image for qh (CL_R, CL_HALF_FLOAT): width = M*K/16 + img_fmt = {CL_R, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 16; + img_desc.buffer = extra0_q5_k->qh; + CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // subbuffer for activations region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); + region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - img_fmt = { CL_RGBA, CL_FLOAT }; + // image for activations (CL_RGBA, CL_FLOAT): width = K*N/4 + img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)K * N / 4; + img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh_img)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + kernel = use_q5k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 + : backend_ctx->kernel_gemv_noshuffle_q5_k_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_hi2)); + if (use_q5k_mc3) { + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &ne1)); // n_cols + } + + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; - size_t local_work_size[3] = { 64, 8, 1 }; - size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(qh_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); } else { - const int gemm_tile_n = 64; - int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; + + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0) { + padding = 8 - extra_elements; + } + + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float) / 2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + + // transpose activations + int height_B = N / 4; + if (height_B == 0) height_B = 1; + int width_B = K / 4; + int padded_height_B = (N + padding) / 4; + + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + + size_t local_work_size_t[2] = {1, 16}; + size_t global_work_size_t[2] = {(size_t)width_B, (size_t)padded_height_B}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - static const char * q6_k_bin_dp4a_env = getenv("GGML_OPENCL_Q6_K_BIN_DP4A"); - bool q6_k_bin_dp4a_on = q6_k_bin_dp4a_env - ? (atoi(q6_k_bin_dp4a_env) != 0) - : true; + // dp4a (int8) dense q5_K prefill GEMM + static const char * q5k_dense_dp4a_env = getenv("GGML_OPENCL_Q5K_DENSE_DP4A"); + bool q5k_dense_dp4a_on = q5k_dense_dp4a_env + ? (atoi(q5k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); // dot prod has to be available - q6_k_bin_dp4a_on = backend_ctx->has_integer_dot && q6_k_bin_dp4a_on; - - if (q6_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin) { - const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; - const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + q5k_dense_dp4a_on = backend_ctx->has_integer_dot && q5k_dense_dp4a_on; - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { + const int Mm = ne01, Nn = ne1, Kk = ne00; + const size_t n_blocks = (size_t)Nn * (Kk / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)Nn * Kk * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); - cl_mem b_sub = nullptr; - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - cl_mem d_sub = nullptr; - cl_mem d_img = nullptr; - region.origin = offsetd; - region.size = (size_t)M * N * sizeof(float); - CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - img_fmt = { CL_R, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * N; - img_desc.buffer = d_sub; - CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - - kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; - - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->ql_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); - - size_t local_work_size[3] = { 64, 1, 1 }; - size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + static const char * q5_k_act_vload4_env = getenv("GGML_OPENCL_Q5_K_DP4A_ACT_VLOAD4"); + const bool q5_k_act_vload4_on = q5_k_act_vload4_env == nullptr || atoi(q5_k_act_vload4_env) != 0; + cl_kernel dk = q5_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Mm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Nn)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Kk)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(Mm / 64), (size_t)CEIL_DIV(Nn, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - CL_CHECK(clReleaseMemObject(b_sub)); - CL_CHECK(clReleaseMemObject(d_img)); - CL_CHECK(clReleaseMemObject(d_sub)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); return; } - cl_mem b_sub_buf = nullptr; - cl_mem b_padded = nullptr; - cl_mem b_buf = nullptr; - if (N_pad == N) { - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - b_buf = b_sub_buf; - } else { - CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); - const float zero = 0.0f; - CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); - b_buf = b_padded; - } - - img_fmt = { CL_R, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)K * N_pad; - img_desc.buffer = b_buf; - cl_mem b_img; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32; + int padded_N = N + padding; - region.origin = offsetd; - region.size = (size_t)M * N * sizeof(float); - cl_mem d_sub_buf; - CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - img_fmt = { CL_R, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * N; - img_desc.buffer = d_sub_buf; - cl_mem d_img; - CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_hi2)); - kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; - size_t local_work_size[3] = { 64, 2, 2 }; - size_t m_tiles = (size_t)CEIL_DIV(M, 64); - size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); - if (b_sub_buf) { - CL_CHECK(clReleaseMemObject(b_sub_buf)); - } - if (b_padded) { - CL_CHECK(clReleaseMemObject(b_padded)); - } - CL_CHECK(clReleaseMemObject(d_img)); - CL_CHECK(clReleaseMemObject(d_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img_trans)); } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - -static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - - ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; +// Dequant a possibly-strided q4_0/q8_0 tensor to tight-packed f16. Returns a +// temp cl_mem the caller must release. SoA inputs are reconstructed into a +// temp AoS buffer reported via *extra_reconstruct (also caller-released). +// this is for quantized K cache without FA. +static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + cl_mem * extra_reconstruct /* out, may be NULL */ +) { + GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; + if (extra_reconstruct) { + *extra_reconstruct = NULL; + } - const int ne1 = dst->ne[1]; + cl_mem src_buf; + cl_ulong src_offset; + cl_ulong src_nb1; + cl_ulong src_nb2; + cl_ulong src_nb3; - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + uintptr_t pool_key_buf = 0; + cl_ulong pool_key_off = (cl_ulong) tensor->view_offs; - cl_context context = backend_ctx->context; - cl_kernel kernel; + const bool is_soa = tensor->type == GGML_TYPE_Q4_0 + ? ggml_cl_is_q4_0_soa(tensor) + : ggml_cl_is_q8_0_soa(tensor); - cl_int err; - cl_buffer_region region; - cl_image_format img_fmt; - cl_image_desc img_desc; + cl_mem aos = nullptr; + if (is_soa) { + // Reconstruct full parent AoS; view's own nb[] then index it correctly. + const ggml_tensor * parent = tensor->view_src ? tensor->view_src : tensor; + const ggml_tensor * soa_src = parent; + const size_t block_bytes = (size_t) ggml_type_size(tensor->type); + const size_t blck_size = (size_t) ggml_blck_size(tensor->type); + const size_t parent_row_blocks = (size_t) parent->ne[0] / blck_size; + const size_t parent_row_bytes = parent_row_blocks * block_bytes; + const size_t parent_nbytes = (size_t) ggml_nelements(parent) / blck_size * block_bytes; - // subbuffer and image for activation - // Multi-column verify GEMV: route the spec/MTP verify q6_K matmuls (ne1==3) - // onto the efficient GEMV path instead of the transposed-GEMM dead-zone. - // Reuses the ne1==1 image setup (activation image sized by N=ne1). Byte- - // identical. Opt-in via GGML_OPENCL_Q6K_MC3=1 while validating. - static const bool q6k_mc3 = (getenv("GGML_OPENCL_Q6K_MC3") != nullptr); - // Per-layer only (ne01 < 32768): batched large-vocab lm_head stays on the - // existing path (x2-unified routes batched Q6_K lm_head to CPU; the Adreno - // GEMV corrupts it). Per-layer mc3 is byte-identical. - const bool use_q6k_mc3 = q6k_mc3 && (ne1 == 3) && (ne01 < 32768); - // Batched verify lm_head/embed (ne1==3, tiled layout): multi-column tiled - // GEMV — streams the large lm_head weight once across the 3 verify columns - // (the #1 MTP bottleneck; mc3 above can't, it reads the noshuffle layout). - const bool use_q6k_tiled_mc = q6k_mc3 && (ne1 == 3) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0); + cl_int err; + aos = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); + CL_CHECK(err); - const bool use_bin = use_q6_k_bin_kernels(backend_ctx, src0); + // large q4_0/q8_0 WEIGHTS are stored transposed and small weights + // (and the AoS KV-cache, handled in the else branch above) are not. + // choose a proper restore kernel based on this. + bool restored = false; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + const int p_ne00 = (int) parent->ne[0]; + const int p_ne01 = (int) parent->ne[1]; + if (tensor->type == GGML_TYPE_Q8_0 && enable_adreno_trans_weight(backend_ctx, parent)) { + auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + pool_key_buf = (uintptr_t) extra->q; + cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &p_ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &p_ne01)); + size_t gws[] = { (size_t)(((p_ne01 + 63) / 64) * 64), 1, 1 }; + size_t lws[] = { 64, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + restored = true; + } else if (tensor->type == GGML_TYPE_Q4_0 && + use_adreno_kernels(backend_ctx, parent) && + !use_adreno_moe_kernels(backend_ctx, parent)) { + auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + pool_key_buf = (uintptr_t) extra->q; + const size_t size_q = (size_t) ggml_nelements(parent) / blck_size * (blck_size / 2); + const size_t size_d = (size_t) ggml_nelements(parent) / blck_size * sizeof(ggml_fp16_t); + cl_int err2 = CL_SUCCESS; + cl_mem buf_tq = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_q, NULL, &err2); CL_CHECK(err2); + cl_mem buf_td = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_d, NULL, &err2); CL_CHECK(err2); + transpose_2d_as_16b(backend_ctx, extra->q, buf_tq, size_q, p_ne01, p_ne00 / 4); + transpose_2d_as_16b(backend_ctx, extra->d, buf_td, size_d, p_ne01, p_ne00 / 32); + cl_uchar mask_0F = 0x0F, mask_F0 = 0xF0; + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_tq)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_td)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + const size_t n_blk = parent_nbytes / block_bytes; + size_t gws[] = { n_blk, 1, 1 }; + size_t lws[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); - if (use_bin) { - if (use_q6k_mc3 || use_q6k_tiled_mc) { - static bool warned = false; - if (!warned) { - GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q6K_MC3 is bypassed by Q6_K binary kernels\n"); - warned = true; - } + CL_CHECK(clReleaseMemObject(buf_tq)); + CL_CHECK(clReleaseMemObject(buf_td)); + restored = true; } - ggml_cl_mul_mat_q6_K_f32_adreno_ila(backend, src0, src1, dst); - return; - } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - if (ne1 == 1 || use_q6k_mc3 || use_q6k_tiled_mc) { - cl_mem ql_img = nullptr; - cl_mem qh_img = nullptr; - cl_mem b_sub_buffer = nullptr; - cl_mem b_img = nullptr; + if (!restored) { + cl_kernel kernel; + if (tensor->type == GGML_TYPE_Q8_0) { + auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + kernel = backend_ctx->kernel_restore_block_q8_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + pool_key_buf = (uintptr_t) extra->q; + } else { + auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + kernel = backend_ctx->kernel_restore_block_q4_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + pool_key_buf = (uintptr_t) extra->q; + } - // o4 = 4-output-per-WI variant for long-vocab lm_head/embed; gated to - // ne01 >= 32768 so per-layer q6_K (ne01=hidden 2-8K) keeps the 2-output - // kernel (o4 regresses there). o4_global reads the weights from __global - // coalesced instead of image1d_buffer -- the texture cache caps the - // read-once-per-token lm_head bandwidth, while __global reaches the higher - // rate the rest of the model gets. Both default ON; opt out via - // GGML_OPENCL_Q6K_GEMV_O4 / GGML_OPENCL_Q6K_GEMV_O4_GLOBAL = 0. - static const bool gemv_o4_env = []{ - const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4"); - return !e || e[0] == '\0' || e[0] != '0'; - }(); - static const bool o4_global_env = []{ - const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4_GLOBAL"); - return !e || e[0] == '\0' || e[0] != '0'; - }(); - const bool use_tiled = !use_q6k_mc3 && use_q6k_tiled(backend_ctx, src0); - const bool use_o4 = !use_tiled && !use_q6k_mc3 && gemv_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); - const bool use_o4_global = use_o4 && o4_global_env; + const size_t n_blocks = parent_nbytes / block_bytes; + size_t gws_rec[] = { n_blocks, 1, 1 }; + size_t lws_rec[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws_rec, lws_rec, 0, NULL, NULL)); + } - // ql/qh image views are only needed when NOT reading weights from global. - if (!use_o4_global && !use_tiled) { - // image for ql - img_fmt.image_channel_order = CL_R; - img_fmt.image_channel_data_type = CL_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne01 * ne00 / 8; - img_desc.buffer = extra0_q6_K->ql; - CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + (void) parent_row_blocks; + (void) parent_row_bytes; + src_buf = aos; + src_offset = tensor->view_offs; + src_nb1 = tensor->nb[1]; + src_nb2 = tensor->nb[2]; + src_nb3 = tensor->nb[3]; - // image for qh - img_fmt.image_channel_order = CL_R; - img_fmt.image_channel_data_type = CL_HALF_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne01 * ne00 / 8; - img_desc.buffer = extra0_q6_K->qh; - CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + if (extra_reconstruct) { + *extra_reconstruct = aos; } + } else { + auto * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra && extra->data_device); + src_buf = extra->data_device; + src_offset = extra->offset + tensor->view_offs; + src_nb1 = tensor->nb[1]; + src_nb2 = tensor->nb[2]; + src_nb3 = tensor->nb[3]; + pool_key_buf = (uintptr_t) extra->data_device; + pool_key_off = (cl_ulong) src_offset; + } - region.origin = offset1; - region.size = ne00 * ne1 * sizeof(float); - CL_CHECK((b_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - - img_fmt.image_channel_order = CL_RGBA; - img_fmt.image_channel_data_type = CL_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne00 * ne1 / 4; - img_desc.buffer = b_sub_buffer; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + const cl_int nblk0 = (cl_int) (tensor->ne[0] / ggml_blck_size(tensor->type)); + const cl_int ne1_ = (cl_int) tensor->ne[1]; + const cl_int ne2_ = (cl_int) tensor->ne[2]; + const cl_int ne3_ = (cl_int) tensor->ne[3]; - kernel = use_q6k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 - : use_q6k_tiled_mc ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 - : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled - : use_o4_global ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global - : use_o4 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 - : backend_ctx->kernel_gemv_noshuffle_q6_K_f32; + const size_t out_bytes = (size_t) ggml_nelements(tensor) * sizeof(ggml_fp16_t); - if (use_o4_global || use_tiled) { - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + // reuse a pooled f16 buffer for this KV-cache view across decode steps instead of + // allocating new one per attention op + cl_mem out = nullptr; + { + auto & pool = backend_ctx->dequant_f16_pool; + ggml_backend_opencl_context::ImagePoolKey key{pool_key_buf, (uint64_t) pool_key_off}; + auto it = pool.find(key); + if (it != pool.end() && it->second.k_bytes >= out_bytes && it->second.image) { + out = it->second.image; } else { - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); + if (it != pool.end()) { + if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } + pool.erase(it); + } + cl_int err = CL_SUCCESS; + out = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); + CL_CHECK(err); + ggml_backend_opencl_context::ImagePoolEntry entry; + entry.image = out; + entry.k_bytes = out_bytes; + pool[key] = entry; } - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - - const size_t gws_x = use_tiled - ? (size_t) CEIL_DIV(ne01, 64) * 64 - : use_o4 - ? (size_t) CEIL_DIV(ne01/4, 64) * 64 - : (size_t) CEIL_DIV(ne01/2, 64) * 64; - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {gws_x, 4, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } - if (ql_img) CL_CHECK(clReleaseMemObject(ql_img)); - if (qh_img) CL_CHECK(clReleaseMemObject(qh_img)); - CL_CHECK(clReleaseMemObject(b_sub_buffer)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { - // Tiled-layout batched GEMM. When the weight was converted to the 64-row - // tiled canonical layout (use_q6k_tiled — the default for lm_head/embed), - // the plain noshuffle GEMM below reads it as plain-transposed and produces - // garbage. Use the batched GEMM that matches the decode tiled GEMV's - // layout; it reads the f32 activation directly (column-major, no transpose). - if (use_q6k_tiled(backend_ctx, src0)) { - cl_mem b_sub_buf_t = nullptr; - cl_mem b_img_t = nullptr; + cl_kernel dq_kernel = tensor->type == GGML_TYPE_Q8_0 + ? backend_ctx->kernel_dequant_q8_0_f16_view_aos + : backend_ctx->kernel_dequant_q4_0_f16_view_aos; - region.origin = offset1; - region.size = ne00 * ne1 * sizeof(float); - CL_CHECK((b_sub_buf_t = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK(clSetKernelArg(dq_kernel, 0, sizeof(cl_mem), &src_buf)); + CL_CHECK(clSetKernelArg(dq_kernel, 1, sizeof(cl_ulong), &src_offset)); + CL_CHECK(clSetKernelArg(dq_kernel, 2, sizeof(cl_ulong), &src_nb1)); + CL_CHECK(clSetKernelArg(dq_kernel, 3, sizeof(cl_ulong), &src_nb2)); + CL_CHECK(clSetKernelArg(dq_kernel, 4, sizeof(cl_ulong), &src_nb3)); + CL_CHECK(clSetKernelArg(dq_kernel, 5, sizeof(cl_int), &nblk0)); + CL_CHECK(clSetKernelArg(dq_kernel, 6, sizeof(cl_int), &ne1_)); + CL_CHECK(clSetKernelArg(dq_kernel, 7, sizeof(cl_int), &ne2_)); + CL_CHECK(clSetKernelArg(dq_kernel, 8, sizeof(cl_int), &ne3_)); + CL_CHECK(clSetKernelArg(dq_kernel, 9, sizeof(cl_mem), &out)); - img_fmt.image_channel_order = CL_RGBA; - img_fmt.image_channel_data_type = CL_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne00 * ne1 / 4; - img_desc.buffer = b_sub_buf_t; - CL_CHECK((b_img_t = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + size_t gws[3] = { (size_t) nblk0, (size_t) ne1_, (size_t) (ne2_ * ne3_) }; + size_t lws[3] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, dq_kernel, 3, NULL, gws, lws, 0, NULL, NULL)); - cl_kernel kt = backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled; - CL_CHECK(clSetKernelArg(kt, 0, sizeof(cl_mem), &extra0_q6_K->ql)); - CL_CHECK(clSetKernelArg(kt, 1, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(kt, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kt, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kt, 4, sizeof(cl_mem), &b_img_t)); - CL_CHECK(clSetKernelArg(kt, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kt, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kt, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kt, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kt, 9, sizeof(int), &ne1)); + // release the reconstructed aos if + // 1. it was actually reconstructed + // 2. the caller didn't request it to be returned + // src_buf may refer to aos, so we should release after this enqueue + if (aos && !extra_reconstruct) { + CL_CHECK(clReleaseMemObject(aos)); + } + return out; +} - // Must match the kernel: NTILES=4 64-row tiles per work-group (256 rows), - // BN=8 output columns per work-group. - const int BN_T = 16; - const int WROWS = 4 * 64; // NTILES * TILE_ROWS - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01, WROWS) * 64, 4, (size_t)CEIL_DIV(ne1, BN_T)}; - backend_ctx->enqueue_ndrange_kernel(kt, 3, global_work_size, local_work_size, dst); +// look up or create a pooled image1d_buffer over a KV-cache view. +static cl_mem ggml_cl_img_pool_get_or_create( + ggml_backend_opencl_context * backend_ctx, + std::map<ggml_backend_opencl_context::ImagePoolKey, + ggml_backend_opencl_context::ImagePoolEntry> & pool, + cl_mem data_device, + cl_ulong offset0, + size_t required_bytes, + cl_channel_type channel_data_type +) { + ggml_backend_opencl_context::ImagePoolKey key{(uintptr_t)data_device, (uint64_t)offset0}; + auto it = pool.find(key); + if (it != pool.end() + && it->second.k_bytes >= required_bytes + && it->second.channel_data_type == channel_data_type + && it->second.image != nullptr) { + return it->second.image; + } - CL_CHECK(clReleaseMemObject(b_img_t)); - CL_CHECK(clReleaseMemObject(b_sub_buf_t)); - return; - } + // need to create or recreate and release any stale entry first. + if (it != pool.end()) { + if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } + if (it->second.sub_buffer) {CL_CHECK(clReleaseMemObject(it->second.sub_buffer)); } + pool.erase(it); + } - cl_mem b_sub_buf; - cl_mem b_buf_trans; - cl_mem b_img; - cl_mem b_img_trans; + cl_int status = CL_SUCCESS; + cl_buffer_region region = {}; + region.origin = (size_t)offset0; + region.size = required_bytes; + cl_mem sub = clCreateSubBuffer(data_device, 0, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + if (status != CL_SUCCESS) { + return nullptr; + } - // subbuffer for activation - region.origin = offset1; - region.size = ne00 * ne1 * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + const size_t pixel_size = (channel_data_type == CL_HALF_FLOAT) ? 8 : 16; + cl_image_format fmt = {CL_RGBA, channel_data_type}; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = required_bytes / pixel_size; + desc.buffer = sub; + cl_mem img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, + &fmt, &desc, NULL, &status); + if (status != CL_SUCCESS) { + CL_CHECK(clReleaseMemObject(sub)); + return nullptr; + } - // dp4a (int8) dense q6_K prefill GEMM - static const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A"); - bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr) - ? (atoi(q6k_dense_dp4a_env) != 0) - : (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E); - // dot prod has to be available - q6k_dense_dp4a_on = backend_ctx->has_integer_dot && q6k_dense_dp4a_on; + ggml_backend_opencl_context::ImagePoolEntry entry; + entry.sub_buffer = sub; + entry.image = img; + entry.k_bytes = required_bytes; + entry.channel_data_type = channel_data_type; + pool[key] = entry; + return img; +} - const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 || - strncmp(src0->name, "token_embd", 10) == 0; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_mul_mat_q2_K_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - if (q6k_dense_dp4a_on && !is_output_w_dp4a && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { - const int M = ne01, N = ne1, K = ne00; - const size_t n_blocks = (size_t)N * (K / 32); - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); - backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); - backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 16 ? 8 : 32; + cl_kernel gemm = n <= 16 ? backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_Q2_K_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } - cl_int tb = (cl_int)n_blocks; - cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); - CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); - size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; - backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->dm)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} - cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a; - int ai = 0; - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); - size_t d_local[3] = { 64, 1, 1 }; - size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; - backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); +static bool ggml_cl_mul_mat_q3_K_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - CL_CHECK(clReleaseMemObject(b_sub_buf)); - return; - } + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 16 ? 8 : 32; + cl_kernel gemm = n <= 16 ? backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_Q3_K_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } - // image for activation - img_fmt.image_channel_order = CL_RGBA; - img_fmt.image_channel_data_type = CL_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne00 * ne1 / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} - // pad N to multiple of 8 - int extra_elements = ne1 % 8; - int padding = 0; - if (extra_elements > 0){ - padding = 8 - extra_elements; - } +static bool ggml_cl_mul_mat_iq3_s_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq3_s * extra0 = (ggml_tensor_extra_cl_iq3_s *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - // subbuffer for transposed activation - region.origin = 0; - region.size = ne00 * (ne1 + padding) * sizeof(float)/2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq3_s_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq3_s_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ3_S_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } - // image for transposed activation - img_fmt.image_channel_order = CL_RGBA; - img_fmt.image_channel_data_type = CL_HALF_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne00 * (ne1 + padding) / 4; - img_desc.buffer = b_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 11, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} - // transpose activation - int height_B = ne1/4; - if (height_B == 0) { - height_B = 1; - } - int width_B = ne00/4; - int padded_height_B = (ne1 + padding) / 4; +static bool ggml_cl_mul_mat_iq3_xxs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq3_xxs * extra0 = (ggml_tensor_extra_cl_iq3_xxs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ3_XXS_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } - size_t local_size_t[2] = { 1, 16 }; - size_t global_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst); + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} - // gemm - // Cooperative-K small-batch (n_q in [2..8]) path: intra-WG K-split, - // mirrors the q4_K _cok path (batched serving). OPT-IN - // (GGML_OPENCL_Q6K_GEMM_COK=1), DEFAULT OFF: q6_K is the tied lm_head/ - // output projection, so the K-reassociation perturbs final logits and - // greedy is NOT byte-identical (op-tests pass, output coherent, but not - // bit-exact). It is also NEUTRAL on end-to-end MTP (q4_K cok already - // captured that; the MTP bottleneck moved off the GEMMs). Keep opt-in - // for batched serving until PPL-validated on a non-GDN q6_K model. - static const char * q6k_cok_env = getenv("GGML_OPENCL_Q6K_GEMM_COK"); - static const bool q6k_gemm_cok = (q6k_cok_env != nullptr) && (atoi(q6k_cok_env) != 0); - const bool use_q6k_cok = q6k_gemm_cok && (ne1 <= 8); - kernel = use_q6k_cok ? backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok - : backend_ctx->kernel_gemm_noshuffle_q6_K_f32; - int padded_N = ne1 + padding; +static bool ggml_cl_mul_mat_iq1_m_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq1_m * extra0 = (ggml_tensor_extra_cl_iq1_m *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_ushort mask_f000 = 0xF000; - cl_uchar mask_c0 = 0xC0; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq1_m_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq1_m_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ1_M_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ushort),&mask_f000)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_c0)); + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} - size_t global_work_size[3]; - size_t local_work_size[3]; - if (use_q6k_cok) { - global_work_size[0] = (size_t)ne01; // rows (1 per lane) - global_work_size[1] = 8; // COK_NSG - global_work_size[2] = 1; - local_work_size[0] = 64; // COK_SG - local_work_size[1] = 8; // COK_NSG - local_work_size[2] = 1; - } else { - global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); - global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); - global_work_size[2] = 1; - local_work_size[0] = 2; - local_work_size[1] = 128; - local_work_size[2] = 1; - } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +static bool ggml_cl_mul_mat_iq1_s_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq1_s * extra0 = (ggml_tensor_extra_cl_iq1_s *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_buf_trans)); - CL_CHECK(clReleaseMemObject(b_img_trans)); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq1_s_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq1_s_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ1_S_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); -#endif + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q5_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); +static bool ggml_cl_mul_mat_iq2_s_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq2_s * extra0 = (ggml_tensor_extra_cl_iq2_s *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq2_s_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq2_s_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ2_S_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } - ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 11, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + +static bool ggml_cl_mul_mat_iq2_xs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq2_xs * extra0 = (ggml_tensor_extra_cl_iq2_xs *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ2_XS_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} - const int ne1 = dst->ne[1]; +static bool ggml_cl_mul_mat_iq2_xxs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq2_xxs * extra0 = (ggml_tensor_extra_cl_iq2_xxs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ2_XXS_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + +static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q2_k_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->dm)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_d)); + const size_t rows = 64*(size_t)backend_ctx->q2_k_mv_rows; + const size_t nsg = backend_ctx->q2_k_mv_nsg; + size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_q2_K_dp4a(backend, src0, src1, dst)) { + return; + } cl_context context = backend_ctx->context; - cl_kernel kernel; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - cl_int err; - cl_buffer_region region; - cl_image_format img_fmt; - cl_image_desc img_desc; + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - const int M = ne01; - const int N = ne1; - const int K = ne00; + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - if (ne1 == 1) { - cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_q2_k_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} - img_fmt = { CL_RGBA, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); +static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - kernel = backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q3_k_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t rows = 64*(size_t)backend_ctx->q3_k_mv_rows; + const size_t nsg = backend_ctx->q3_k_mv_nsg; + size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } - size_t local_work_size[3] = { 64, 8, 1 }; - size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + if (ggml_cl_mul_mat_q3_K_dp4a(backend, src0, src1, dst)) { + return; + } - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - } else { - static const char * q5_k_bin_dp4a_env = getenv("GGML_OPENCL_Q5_K_BIN_DP4A"); - bool q5_k_bin_dp4a_on = q5_k_bin_dp4a_env - ? (atoi(q5_k_bin_dp4a_env) != 0) - : true; - // dot prod has to be available - q5_k_bin_dp4a_on = backend_ctx->has_integer_dot && q5_k_bin_dp4a_on; + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - if (q5_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin) { - const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; - const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); - backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); - backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - cl_mem b_sub = nullptr; - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - cl_int tb = (cl_int)((size_t)N * (K / 32)); - cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); - CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); - size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; - backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_q3_k_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} - cl_mem d_sub = nullptr; - cl_mem d_img = nullptr; - region.origin = offsetd; - region.size = (size_t)M * N * sizeof(float); - CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); +static void ggml_cl_mul_mat_iq3_s_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq3_s * extra0 = (ggml_tensor_extra_cl_iq3_s *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - img_fmt = { CL_R, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * N; - img_desc.buffer = d_sub; - CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq3_s_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq3_s_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 12, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 13, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq3_s_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } - kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin; + if (ggml_cl_mul_mat_iq3_s_dp4a(backend, src0, src1, dst)) { + return; + } - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->q_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->qh)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->dm)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->s)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - size_t local_work_size_dp4a[3] = { 64, 1, 1 }; - size_t global_work_size_dp4a[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size_dp4a, local_work_size_dp4a, dst); + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - CL_CHECK(clReleaseMemObject(b_sub)); - CL_CHECK(clReleaseMemObject(d_img)); - CL_CHECK(clReleaseMemObject(d_sub)); - return; - } + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - const int gemm_tile_n = 64; - int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - cl_mem b_sub_buf = nullptr; - cl_mem b_padded = nullptr; - cl_mem b_buf = nullptr; - if (N_pad == N) { - region.origin = offset1; - region.size = (size_t)K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - b_buf = b_sub_buf; - } else { - CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); - const float zero = 0.0f; - CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); - b_buf = b_padded; - } + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq3_s_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} + +static void ggml_cl_mul_mat_iq3_xxs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq3_xxs * extra0 = (ggml_tensor_extra_cl_iq3_xxs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq3_xxs_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq3_xxs_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq3_xxs_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_iq3_xxs_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - img_fmt = { CL_R, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)K * N_pad; - img_desc.buffer = b_buf; - cl_mem b_img; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - region.origin = offsetd; - region.size = (size_t)M * N * sizeof(float); - cl_mem d_sub_buf; - CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - img_fmt = { CL_R, CL_FLOAT }; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = (size_t)M * N; - img_desc.buffer = d_sub_buf; - cl_mem d_img; - CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq3_xxs_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} - kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_img)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uint), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &N)); +static void ggml_cl_mul_mat_iq1_m_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq1_m * extra0 = (ggml_tensor_extra_cl_iq1_m *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - size_t local_work_size[3] = { 64, 2, 2 }; - size_t m_tiles = (size_t)CEIL_DIV(M, 64); - size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq1_m_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq1_s_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq1_m_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } - CL_CHECK(clReleaseMemObject(b_img)); - if (b_sub_buf) { - CL_CHECK(clReleaseMemObject(b_sub_buf)); - } - if (b_padded) { - CL_CHECK(clReleaseMemObject(b_padded)); - } - CL_CHECK(clReleaseMemObject(d_img)); - CL_CHECK(clReleaseMemObject(d_sub_buf)); + if (ggml_cl_mul_mat_iq1_m_dp4a(backend, src0, src1, dst)) { + return; } -} -#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq1_m_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} +static void ggml_cl_mul_mat_iq1_s_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq1_s * extra0 = (ggml_tensor_extra_cl_iq1_s *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq1_s_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq1_s_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq1_s_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne1 = dst->ne[1]; - - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + if (ggml_cl_mul_mat_iq1_s_dp4a(backend, src0, src1, dst)) { + return; + } cl_context context = backend_ctx->context; - cl_kernel kernel; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - int M = ne01; - int N = ne1; - int K = ne00; + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - cl_uchar mask_d6 = 0x3F; - cl_uchar mask_d4 = 0x0F; - cl_uchar mask_hi2 = 0xC0; + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - // Multi-column (N=3) verify GEMV for q5_K: route the spec/MTP verify batch - // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- - // zone (gemm_noshuffle_q5_k, the #2 chunk of MTP decode on a Q4_0-mix model - // after q4_0 mc3). Reuses the ne1==1 GEMV image setup (q + qh + activations). - // Opt-in via GGML_OPENCL_Q5K_MC3=1. Per-layer only (ne01 < 32768). - static const bool q5k_mc3 = (getenv("GGML_OPENCL_Q5K_MC3") != nullptr); - const bool use_q5k_mc3 = q5k_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq1_s_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} - const bool use_bin = use_q5_k_bin_kernels(backend_ctx, src0); +static void ggml_cl_mul_mat_iq2_s_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq2_s * extra0 = (ggml_tensor_extra_cl_iq2_s *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - if (use_bin) { - if (use_q5k_mc3) { - static bool warned = false; - if (!warned) { - GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q5K_MC3 is bypassed by Q5_K binary kernels\n"); - warned = true; - } - } - ggml_cl_mul_mat_q5_K_f32_adreno_ila(backend, src0, src1, dst); + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq2_s_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq2_s_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 12, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 13, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq2_s_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); return; } - if (ne1 == 1 || use_q5k_mc3) { - cl_mem q_img = nullptr; - cl_mem qh_img = nullptr; - cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + if (ggml_cl_mul_mat_iq2_s_dp4a(backend, src0, src1, dst)) { + return; + } - // image for q (CL_R, CL_UNSIGNED_INT32): width = M*K/2/4 - img_fmt = {CL_R, CL_UNSIGNED_INT32}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q5_k->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - // image for qh (CL_R, CL_HALF_FLOAT): width = M*K/16 - img_fmt = {CL_R, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 16; - img_desc.buffer = extra0_q5_k->qh; - CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - // image for activations (CL_RGBA, CL_FLOAT): width = K*N/4 - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - kernel = use_q5k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 - : backend_ctx->kernel_gemv_noshuffle_q5_k_f32; + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq2_s_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_hi2)); - if (use_q5k_mc3) { - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &ne1)); // n_cols - } +static void ggml_cl_mul_mat_iq2_xs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq2_xs * extra0 = (ggml_tensor_extra_cl_iq2_xs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq2_xs_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq2_xs_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq2_xs_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + if (ggml_cl_mul_mat_iq2_xs_dp4a(backend, src0, src1, dst)) { + return; + } - CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(qh_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { - cl_mem b_sub_buf = nullptr; - cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - // pad N to multiple of 8 - int extra_elements = N % 8; - int padding = 0; - if (extra_elements > 0) { - padding = 8 - extra_elements; - } + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - // subbuffer for transposed activations - region.origin = 0; - region.size = K * (N + padding) * sizeof(float) / 2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq2_xs_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} - // image for transposed activations - img_fmt = {CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); +static void ggml_cl_mul_mat_iq2_xxs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq2_xxs * extra0 = (ggml_tensor_extra_cl_iq2_xxs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq2_xxs_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq2_xxs_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq2_xxs_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } - // transpose activations - int height_B = N / 4; - if (height_B == 0) height_B = 1; - int width_B = K / 4; - int padded_height_B = (N + padding) / 4; + if (ggml_cl_mul_mat_iq2_xxs_dp4a(backend, src0, src1, dst)) { + return; + } - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - size_t local_work_size_t[2] = {1, 16}; - size_t global_work_size_t[2] = {(size_t)width_B, (size_t)padded_height_B}; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - // dp4a (int8) dense q5_K prefill GEMM - static const char * q5k_dense_dp4a_env = getenv("GGML_OPENCL_Q5K_DENSE_DP4A"); - bool q5k_dense_dp4a_on = q5k_dense_dp4a_env - ? (atoi(q5k_dense_dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); - // dot prod has to be available - q5k_dense_dp4a_on = backend_ctx->has_integer_dot && q5k_dense_dp4a_on; + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { - const int Mm = ne01, Nn = ne1, Kk = ne00; - const size_t n_blocks = (size_t)Nn * (Kk / 32); - backend_ctx->prealloc_moe_qa.allocate(context, (size_t)Nn * Kk * sizeof(cl_char)); - backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); - backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); - cl_int tb = (cl_int)n_blocks; - cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; - CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); - CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); - size_t q_local[1] = { 64 }; - size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; - backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq2_xxs_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} +#endif - cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a; - int ai = 0; - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->s)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->d)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->dm)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Mm)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Nn)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Kk)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); - size_t d_local[3] = { 64, 1, 1 }; - size_t d_global[3] = { 64, (size_t)(Mm / 64), (size_t)CEIL_DIV(Nn, 32) }; - backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); +static void ggml_cl_fwht(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + + auto * extra_src = (ggml_tensor_extra_cl *) src->extra; + auto * extra_dst = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset_src = extra_src->offset + src->view_offs; + const cl_ulong offset_dst = extra_dst->offset + dst->view_offs; + + const cl_ulong ne1 = src->ne[1]; + const cl_ulong ne2 = src->ne[2]; + const cl_ulong nb1 = src->nb[1]; + const cl_ulong nb2 = src->nb[2]; + const cl_ulong nb3 = src->nb[3]; + + const int n = src->ne[0]; + const float scale = 1.0f/sqrtf((float)n); + const int type_index = src->type == GGML_TYPE_F16 ? 1 : 0; + + cl_kernel kernel = backend_ctx->kernel_fwht[type_index]; + const size_t nth = std::min((size_t)n/2, backend_ctx->fwht_workgroup_size[type_index]); + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_src)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_dst)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, 11, n*sizeof(float), nullptr)); + + size_t global[] = { nth, (size_t)ggml_nrows(src) }; + size_t local[] = { nth, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global, local, dst); +} - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); - return; - } +static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32; - int padded_N = N + padding; + // bf16 is stored as f16 on device + const enum ggml_type src0t = (src0->type == GGML_TYPE_BF16) ? GGML_TYPE_F16 : src0->type; + const enum ggml_type src1t = src1->type; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_hi2)); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {1, 128, 1}; + if (ggml_cl_mul_mat_can_use_fwht(backend_ctx, dst)) { + ggml_cl_fwht(backend, src1, dst); + return; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + // quant kv without FA + // used for non-contiguous src0 (the usual head-major permuted K view when n_head_kv>1) + // AND for the contiguous case that occurs when n_head_kv==1 (e.g. Gemma-4 E2B) + // Q4_0 bin kernels use a special weight layout, which the restore below does not support. + bool q4_0_bin_layout = false; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + q4_0_bin_layout = src0t == GGML_TYPE_Q4_0 && src0->view_src == nullptr && + ggml_is_contiguous(src0) && + src0->ne[2] == 1 && src0->ne[3] == 1 && + use_adreno_kernels(backend_ctx, src0) && + !use_adreno_moe_kernels(backend_ctx, src0) && + use_q4_0_bin_kernels(backend_ctx, src0); +#endif + if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) && !q4_0_bin_layout && + (!ggml_is_contiguous(src0) || src1->ne[2] > src0->ne[2])) { + cl_mem f16_buf = ggml_cl_mul_mat_dequant_quant_to_f16(backend_ctx, src0, nullptr); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); + ggml_tensor fake_src0 = *src0; + ggml_tensor_extra_cl fake_extra = {}; + fake_extra.data_device = f16_buf; + fake_extra.offset = 0; + fake_src0.type = GGML_TYPE_F16; + fake_src0.extra = &fake_extra; + fake_src0.view_src = nullptr; + fake_src0.view_offs = 0; + fake_src0.nb[0] = sizeof(ggml_fp16_t); + fake_src0.nb[1] = fake_src0.nb[0] * src0->ne[0]; + fake_src0.nb[2] = fake_src0.nb[1] * src0->ne[1]; + fake_src0.nb[3] = fake_src0.nb[2] * src0->ne[2]; + + ggml_cl_mul_mat(backend, &fake_src0, src1, dst); + return; } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + +#ifdef GGML_OPENCL_SOA_Q + // view->extra stays pre-SoA; cast to the SoA struct would SIGSEGV. + // Follow view_src to reach the real SoA extra. + const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; + ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; + ggml_tensor_extra_cl_q2_0 * extra0_q2_0 = (ggml_tensor_extra_cl_q2_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)soa0_src->extra; + ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; + ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; + ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; + ggml_tensor_extra_cl_iq4_xs * extra0_iq4_xs = (ggml_tensor_extra_cl_iq4_xs *)soa0_src->extra; + ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra; + ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra; + ggml_tensor_extra_cl_iq2_s * extra0_iq2_s = (ggml_tensor_extra_cl_iq2_s *)soa0_src->extra; + ggml_tensor_extra_cl_iq2_xs * extra0_iq2_xs = (ggml_tensor_extra_cl_iq2_xs *)soa0_src->extra; + ggml_tensor_extra_cl_iq2_xxs * extra0_iq2_xxs = (ggml_tensor_extra_cl_iq2_xxs *)soa0_src->extra; + ggml_tensor_extra_cl_iq1_m * extra0_iq1_m = (ggml_tensor_extra_cl_iq1_m *)soa0_src->extra; + ggml_tensor_extra_cl_iq1_s * extra0_iq1_s = (ggml_tensor_extra_cl_iq1_s *)soa0_src->extra; + ggml_tensor_extra_cl_iq3_s * extra0_iq3_s = (ggml_tensor_extra_cl_iq3_s *)soa0_src->extra; + ggml_tensor_extra_cl_iq3_xxs * extra0_iq3_xxs = (ggml_tensor_extra_cl_iq3_xxs *)soa0_src->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; + ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; #endif -} -// Dequant a possibly-strided q4_0/q8_0 tensor to tight-packed f16. Returns a -// temp cl_mem the caller must release. SoA inputs are reconstructed into a -// temp AoS buffer reported via *extra_reconstruct (also caller-released). -// this is for quantized K cache without FA. -static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16( - ggml_backend_opencl_context * backend_ctx, - const ggml_tensor * tensor, - cl_mem * extra_reconstruct /* out, may be NULL */ -) { - GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0); + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(int, ne1, src1, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); - if (extra_reconstruct) { - *extra_reconstruct = NULL; + int r2 = ne12/ne02; + int r3 = ne13/ne03; + + GGML_ASSERT(ne00 == ne10); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // adreno GEMM/GEMV kernels do not support broadcast, assuming ne2 and ne3 are 1 for src1 + // so we handle broadcast here + if ((ne12 > 1 || ne13 > 1) && ne02 == 1 && ne03 == 1 && + src0t != GGML_TYPE_F16 && src0t != GGML_TYPE_F32) { + for (int i13 = 0; i13 < ne13; ++i13) { + for (int i12 = 0; i12 < ne12; ++i12) { + ggml_tensor s1 = *src1; + s1.ne[2] = 1; s1.ne[3] = 1; + s1.view_offs = src1->view_offs + (size_t)i12*nb12 + (size_t)i13*nb13; + ggml_tensor d = *dst; + d.ne[2] = 1; d.ne[3] = 1; + d.view_offs = dst->view_offs + (size_t)i12*nb2 + (size_t)i13*nb3; + d.src[1] = &s1; + ggml_cl_mul_mat(backend, src0, &s1, &d); + } + } + return; } +#endif - cl_mem src_buf; - cl_ulong src_offset; - cl_ulong src_nb1; - cl_ulong src_nb2; - cl_ulong src_nb3; + int nth0 = 32; + int nth1 = 1; + int nrows = 1; + // The number of values produced by each subgroup + int ndst = 4; - uintptr_t pool_key_buf = 0; - cl_ulong pool_key_off = (cl_ulong) tensor->view_offs; + cl_kernel kernel; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){ + // Two tiling assumptions these kernels make but nothing enforced: + // + // ne00 % TILESIZE_K(16): the K loop has no tail, so a K that does not + // divide folds 1-15 rows of whatever follows the operands into every + // output. + // + // ne01 % TILESIZE_M(64): mm_store_c_N guards the n direction with its + // `mask` argument but nothing guards m -- the store walks all 64 rows + // of the tile at a stride of M. When M does not divide, the last tile + // does not run off the end of the buffer, it writes 64 - (M % 64) + // values ON TOP OF the next column, so the result is silently wrong. + // Reachable on the KQV side for any head size >= 64 that is not a + // multiple of it (80, 96, 112). + // + // Attention shapes in the graph satisfy both -- head sizes are multiples + // of 64 and n_kv is padded -- which is why this has stayed latent. + // Declining leaves the odd shapes on the generic GEMM, which handles them. + if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && + (ne00 % 16) == 0 && (ne01 % 64) == 0 && (ne12 % ne02) == 0 && + // the KQ/KQV image kernels do not handle dim 3 (multi-stream batches) + ne03 == 1 && ne13 == 1 && + // dst is wrapped with image1d_buffer, the size limit applies, also src0 + (ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4 <= backend_ctx->image_max_buffer_size)) { + // For KQ. + // + // Layout admission, mirroring the KQV arm below. The KQ kernel takes + // no stride arguments for A or B: it derives them as K*D_A*2 and + // K*D_B*4, i.e. it assumes both operands pack exactly D heads of K + // elements per row. Every real KV-cache view and permuted-Q view + // does, but a view spanning part of a wider allocation does not, and + // the kernel then walks the wrong rows with nothing to range-check + // it. Gate on the packed layout itself rather than on the stride + // ORDERING, which a wider parent satisfies just as well. + const bool kq_packed_a = (nb01 == (cl_ulong)ne00 * ne02 * ggml_type_size(src0t)) && + (nb02 == (cl_ulong)ne00 * ggml_type_size(src0t)); + const bool kq_packed_b = (nb11 == (cl_ulong)ne10 * ne12 * ggml_type_size(src1t)) && + (nb12 == (cl_ulong)ne10 * ggml_type_size(src1t)); + // + // ggml_is_permuted(src0) stands in for "K is head-major", but it is + // only a proxy and it COLLAPSES at n_head_kv == 1: with a single + // head there is no head stride to be out of order, so nb01 == nb02 + // and the view reports itself unpermuted. Such a KQ was declined + // here and fell through to the generic GEMM (gemma-4 E2B, and any + // other multi-query model). The packed check above is the contract + // the kernel actually needs -- it pins both strides exactly -- so + // require permutedness only where there is more than one head for + // it to mean anything. + // + // Default on; GGML_OPENCL_KQ_NHEAD_KV1=0 restores the old proxy so + // the two routings can be compared in one binary. + static const char * kq_nhkv1_env = getenv("GGML_OPENCL_KQ_NHEAD_KV1"); + static const bool kq_nhkv1_on = + (kq_nhkv1_env == nullptr || kq_nhkv1_env[0] != '0'); + if ((ggml_is_permuted(src0) || (ne02 == 1 && kq_nhkv1_on)) && ggml_is_permuted(src1) && + kq_packed_a && kq_packed_b && + ((nb01 * ne01 / 4)/4 <= backend_ctx->image_max_buffer_size) && + nb00 <= nb02 && + nb02 <= nb01 && + nb01 <= nb03 && + nb10 <= nb12 && + nb12 <= nb11 && + nb11 <= nb13) { + ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst, /*is_kq =*/ true); + return; + } + // For KQV. Reaching this arm is what makes the op a KQV; the callee + // is told so explicitly rather than re-deriving it from the strides + // the arm above has already ruled on. + if (!ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && + ((nb02 * ne02 / 4)/4 <= backend_ctx->image_max_buffer_size)) { + ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst, /*is_kq =*/ false); + return; + } + } - const bool is_soa = tensor->type == GGML_TYPE_Q4_0 - ? ggml_cl_is_q4_0_soa(tensor) - : ggml_cl_is_q8_0_soa(tensor); + static const char * mm_kq_gqa_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_IMG"); + static const bool mm_kq_gqa_img_on = (mm_kq_gqa_img_env == nullptr || mm_kq_gqa_img_env[0] != '0'); + static const char * mm_kq_gqa_r4_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_R4_IMG"); + static const bool mm_kq_gqa_r4_img_on = (mm_kq_gqa_r4_img_env == nullptr || mm_kq_gqa_r4_img_env[0] != '0'); + const bool img_r4_gate = + mm_kq_gqa_r4_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 4 && (ne13 / ne03) == 1; + if (mm_kq_gqa_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; - cl_mem aos = nullptr; - if (is_soa) { - // Reconstruct full parent AoS; view's own nb[] then index it correctly. - const ggml_tensor * parent = tensor->view_src ? tensor->view_src : tensor; - const ggml_tensor * soa_src = parent; - const size_t block_bytes = (size_t) ggml_type_size(tensor->type); - const size_t blck_size = (size_t) ggml_blck_size(tensor->type); - const size_t parent_row_blocks = (size_t) parent->ne[0] / blck_size; - const size_t parent_row_bytes = parent_row_blocks * block_bytes; - const size_t parent_nbytes = (size_t) ggml_nelements(parent) / blck_size * block_bytes; + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); - cl_int err; - aos = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); - CL_CHECK(err); + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } - // large q4_0/q8_0 WEIGHTS are stored transposed and small weights - // (and the AoS KV-cache, handled in the else branch above) are not. - // choose a proper restore kernel based on this. - bool restored = false; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - const int p_ne00 = (int) parent->ne[0]; - const int p_ne01 = (int) parent->ne[1]; - if (tensor->type == GGML_TYPE_Q8_0 && enable_adreno_trans_weight(backend_ctx, parent)) { - auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; - pool_key_buf = (uintptr_t) extra->q; - cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &p_ne00)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &p_ne01)); - size_t gws[] = { (size_t)(((p_ne01 + 63) / 64) * 64), 1, 1 }; - size_t lws[] = { 64, 1, 1 }; - CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); - restored = true; - } else if (tensor->type == GGML_TYPE_Q4_0 && - use_adreno_kernels(backend_ctx, parent) && - !use_adreno_moe_kernels(backend_ctx, parent)) { - auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; - pool_key_buf = (uintptr_t) extra->q; - const size_t size_q = (size_t) ggml_nelements(parent) / blck_size * (blck_size / 2); - const size_t size_d = (size_t) ggml_nelements(parent) / blck_size * sizeof(ggml_fp16_t); - cl_int err2 = CL_SUCCESS; - cl_mem buf_tq = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_q, NULL, &err2); CL_CHECK(err2); - cl_mem buf_td = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_d, NULL, &err2); CL_CHECK(err2); - transpose_2d_as_16b(backend_ctx, extra->q, buf_tq, size_q, p_ne01, p_ne00 / 4); - transpose_2d_as_16b(backend_ctx, extra->d, buf_td, size_d, p_ne01, p_ne00 / 32); - cl_uchar mask_0F = 0x0F, mask_F0 = 0xF0; - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_tq)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_td)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); - const size_t n_blk = parent_nbytes / block_bytes; - size_t gws[] = { n_blk, 1, 1 }; - size_t lws[] = { 1, 1, 1 }; - CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + // r2=4 specialization + if (img_r4_gate) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); - CL_CHECK(clReleaseMemObject(buf_tq)); - CL_CHECK(clReleaseMemObject(buf_td)); - restored = true; + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - if (!restored) { - cl_kernel kernel; - if (tensor->type == GGML_TYPE_Q8_0) { - auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; - kernel = backend_ctx->kernel_restore_block_q8_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); - pool_key_buf = (uintptr_t) extra->q; - } else { - auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; - kernel = backend_ctx->kernel_restore_block_q4_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); - pool_key_buf = (uintptr_t) extra->q; + // DK=256, r2=2 specialization + static const char * mm_kq_r2_dk256_env = getenv("GGML_OPENCL_MM_KQ_GQA_R2_DK256_IMG"); + static const bool mm_kq_r2_dk256_on = (mm_kq_r2_dk256_env != nullptr && mm_kq_r2_dk256_env[0] != '0'); + if (mm_kq_r2_dk256_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 256 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 2 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } } - - const size_t n_blocks = parent_nbytes / block_bytes; - size_t gws_rec[] = { n_blocks, 1, 1 }; - size_t lws_rec[] = { 1, 1, 1 }; - CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws_rec, lws_rec, 0, NULL, NULL)); } - (void) parent_row_blocks; - (void) parent_row_bytes; - src_buf = aos; - src_offset = tensor->view_offs; - src_nb1 = tensor->nb[1]; - src_nb2 = tensor->nb[2]; - src_nb3 = tensor->nb[3]; + // GQA-coalesced KQV for decode using image1d_buffer_t + static const char * mm_kqv_gqa_img_env = getenv("GGML_OPENCL_MM_KQV_GQA_IMG"); + static const bool mm_kqv_gqa_img_on = (mm_kqv_gqa_img_env != nullptr && mm_kqv_gqa_img_env[0] != '0'); + if (mm_kqv_gqa_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img != nullptr && + ne11 == 1 && ne01 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t v_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t v_bytes = (v_bytes_span + 7) & ~(size_t)7; + const size_t v_pixels = v_bytes >> 3; + if (v_pixels > 0 && v_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img; + cl_mem V_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kqv_img_pool, + extra0->data_device, offset0, v_bytes, CL_HALF_FLOAT); + if (V_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &V_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); - if (extra_reconstruct) { - *extra_reconstruct = aos; + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 8; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } } - } else { - auto * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra && extra->data_device); - src_buf = extra->data_device; - src_offset = extra->offset + tensor->view_offs; - src_nb1 = tensor->nb[1]; - src_nb2 = tensor->nb[2]; - src_nb3 = tensor->nb[3]; - pool_key_buf = (uintptr_t) extra->data_device; - pool_key_off = (cl_ulong) src_offset; } - const cl_int nblk0 = (cl_int) (tensor->ne[0] / ggml_blck_size(tensor->type)); - const cl_int ne1_ = (cl_int) tensor->ne[1]; - const cl_int ne2_ = (cl_int) tensor->ne[2]; - const cl_int ne3_ = (cl_int) tensor->ne[3]; + if (ne01 && ne1 && use_adreno_kernels(backend_ctx, src0)) { + // NOTE: Kernels using image1d_buffer_t (e.g., src0_q) would normally require + // a limit check, but q4_0 / q4_1 tensors are very unlikely to exceed that + // limit, so the check is omitted. - const size_t out_bytes = (size_t) ggml_nelements(tensor) * sizeof(ggml_fp16_t); + // q1_0 x fp32 + if (src0t == GGML_TYPE_Q1_0 && src1t == GGML_TYPE_F32 && + enable_adreno_trans_weight(backend_ctx, src0)) { + ggml_cl_mul_mat_q1_0_f32_adreno(backend, src0, src1, dst); + return; + } - // reuse a pooled f16 buffer for this KV-cache view across decode steps instead of - // allocating new one per attention op - cl_mem out = nullptr; - { - auto & pool = backend_ctx->dequant_f16_pool; - ggml_backend_opencl_context::ImagePoolKey key{pool_key_buf, (uint64_t) pool_key_off}; - auto it = pool.find(key); - if (it != pool.end() && it->second.k_bytes >= out_bytes && it->second.image) { - out = it->second.image; - } else { - if (it != pool.end()) { - if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } - pool.erase(it); - } - cl_int err = CL_SUCCESS; - out = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); - CL_CHECK(err); - ggml_backend_opencl_context::ImagePoolEntry entry; - entry.image = out; - entry.k_bytes = out_bytes; - pool[key] = entry; + // q2_0 x fp32 + if (src0t == GGML_TYPE_Q2_0 && src1t == GGML_TYPE_F32 && extra0_q2_0->noshuffle) { + ggml_cl_mul_mat_q2_0_f32_adreno(backend, src0, src1, dst); + return; } - } - cl_kernel dq_kernel = tensor->type == GGML_TYPE_Q8_0 - ? backend_ctx->kernel_dequant_q8_0_f16_view_aos - : backend_ctx->kernel_dequant_q4_0_f16_view_aos; + // q4_0 x fp32 + if(src0t == GGML_TYPE_Q4_0 && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_q4_0_f32_adreno(backend, src0, src1, dst); + return; + } - CL_CHECK(clSetKernelArg(dq_kernel, 0, sizeof(cl_mem), &src_buf)); - CL_CHECK(clSetKernelArg(dq_kernel, 1, sizeof(cl_ulong), &src_offset)); - CL_CHECK(clSetKernelArg(dq_kernel, 2, sizeof(cl_ulong), &src_nb1)); - CL_CHECK(clSetKernelArg(dq_kernel, 3, sizeof(cl_ulong), &src_nb2)); - CL_CHECK(clSetKernelArg(dq_kernel, 4, sizeof(cl_ulong), &src_nb3)); - CL_CHECK(clSetKernelArg(dq_kernel, 5, sizeof(cl_int), &nblk0)); - CL_CHECK(clSetKernelArg(dq_kernel, 6, sizeof(cl_int), &ne1_)); - CL_CHECK(clSetKernelArg(dq_kernel, 7, sizeof(cl_int), &ne2_)); - CL_CHECK(clSetKernelArg(dq_kernel, 8, sizeof(cl_int), &ne3_)); - CL_CHECK(clSetKernelArg(dq_kernel, 9, sizeof(cl_mem), &out)); + // q4_1 x fp32 + if (src0t == GGML_TYPE_Q4_1 && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_q4_1_f32_adreno(backend, src0, src1, dst); + return; + } - size_t gws[3] = { (size_t) nblk0, (size_t) ne1_, (size_t) (ne2_ * ne3_) }; - size_t lws[3] = { 1, 1, 1 }; - CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, dq_kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + // q5_0 x fp32 + if (src0t == GGML_TYPE_Q5_0 && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_q5_0_f32_adreno(backend, src0, src1, dst); + return; + } - // release the reconstructed aos if - // 1. it was actually reconstructed - // 2. the caller didn't request it to be returned - // src_buf may refer to aos, so we should release after this enqueue - if (aos && !extra_reconstruct) { - CL_CHECK(clReleaseMemObject(aos)); - } - return out; -} + // q5_1 x fp32 + if (src0t == GGML_TYPE_Q5_1 && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_q5_1_f32_adreno(backend, src0, src1, dst); + return; + } -// look up or create a pooled image1d_buffer over a KV-cache view. -static cl_mem ggml_cl_img_pool_get_or_create( - ggml_backend_opencl_context * backend_ctx, - std::map<ggml_backend_opencl_context::ImagePoolKey, - ggml_backend_opencl_context::ImagePoolEntry> & pool, - cl_mem data_device, - cl_ulong offset0, - size_t required_bytes, - cl_channel_type channel_data_type -) { - ggml_backend_opencl_context::ImagePoolKey key{(uintptr_t)data_device, (uint64_t)offset0}; - auto it = pool.find(key); - if (it != pool.end() - && it->second.k_bytes >= required_bytes - && it->second.channel_data_type == channel_data_type - && it->second.image != nullptr) { - return it->second.image; - } + // iq4_nl x fp32 + if (src0t == GGML_TYPE_IQ4_NL && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_iq4_nl_f32_adreno(backend, src0, src1, dst); + return; + } - // need to create or recreate and release any stale entry first. - if (it != pool.end()) { - if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } - if (it->second.sub_buffer) {CL_CHECK(clReleaseMemObject(it->second.sub_buffer)); } - pool.erase(it); - } + // iq4_xs x fp32 + if (src0t == GGML_TYPE_IQ4_XS && src1t == GGML_TYPE_F32 && extra0_iq4_xs->noshuffle) { + ggml_cl_mul_mat_iq4_xs_f32_adreno(backend, src0, src1, dst); + return; + } - cl_int status = CL_SUCCESS; - cl_buffer_region region = {}; - region.origin = (size_t)offset0; - region.size = required_bytes; - cl_mem sub = clCreateSubBuffer(data_device, 0, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - if (status != CL_SUCCESS) { - return nullptr; - } + // q8_0 x fp32 + if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 && + enable_adreno_trans_weight(backend_ctx, src0)) { + ggml_cl_mul_mat_q8_0_f32_adreno(backend, src0, src1, dst); + return; + } - const size_t pixel_size = (channel_data_type == CL_HALF_FLOAT) ? 8 : 16; - cl_image_format fmt = {CL_RGBA, channel_data_type}; - cl_image_desc desc = {}; - desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - desc.image_width = required_bytes / pixel_size; - desc.buffer = sub; - cl_mem img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, - &fmt, &desc, NULL, &status); - if (status != CL_SUCCESS) { - CL_CHECK(clReleaseMemObject(sub)); - return nullptr; - } + // q2_k x fp32 + if (src0t == GGML_TYPE_Q2_K && src1t == GGML_TYPE_F32 && extra0_q2_K->noshuffle) { + ggml_cl_mul_mat_q2_k_f32_adreno(backend, src0, src1, dst); + return; + } - ggml_backend_opencl_context::ImagePoolEntry entry; - entry.sub_buffer = sub; - entry.image = img; - entry.k_bytes = required_bytes; - entry.channel_data_type = channel_data_type; - pool[key] = entry; - return img; -} + // q3_k x fp32 + if (src0t == GGML_TYPE_Q3_K && src1t == GGML_TYPE_F32 && extra0_q3_K->noshuffle) { + ggml_cl_mul_mat_q3_k_f32_adreno(backend, src0, src1, dst); + return; + } + + // iq3_s x fp32 + if (src0t == GGML_TYPE_IQ3_S && src1t == GGML_TYPE_F32 && extra0_iq3_s->noshuffle) { + ggml_cl_mul_mat_iq3_s_f32_adreno(backend, src0, src1, dst); + return; + } -static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + // iq3_xxs x fp32 + if (src0t == GGML_TYPE_IQ3_XXS && src1t == GGML_TYPE_F32 && extra0_iq3_xxs->noshuffle) { + ggml_cl_mul_mat_iq3_xxs_f32_adreno(backend, src0, src1, dst); + return; + } - // bf16 is stored as f16 on device - const enum ggml_type src0t = (src0->type == GGML_TYPE_BF16) ? GGML_TYPE_F16 : src0->type; - const enum ggml_type src1t = src1->type; + // iq1_m x fp32 + if (src0t == GGML_TYPE_IQ1_M && src1t == GGML_TYPE_F32 && extra0_iq1_m->noshuffle) { + ggml_cl_mul_mat_iq1_m_f32_adreno(backend, src0, src1, dst); + return; + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // iq1_s x fp32 + if (src0t == GGML_TYPE_IQ1_S && src1t == GGML_TYPE_F32 && extra0_iq1_s->noshuffle) { + ggml_cl_mul_mat_iq1_s_f32_adreno(backend, src0, src1, dst); + return; + } - // quant kv without FA - // used for non-contiguous src0 (the usual head-major permuted K view when n_head_kv>1) - // AND for the contiguous case that occurs when n_head_kv==1 (e.g. Gemma-4 E2B) - if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) && - (!ggml_is_contiguous(src0) || src1->ne[2] > src0->ne[2])) { - cl_mem f16_buf = ggml_cl_mul_mat_dequant_quant_to_f16(backend_ctx, src0, nullptr); + // iq2_s x fp32 + if (src0t == GGML_TYPE_IQ2_S && src1t == GGML_TYPE_F32 && extra0_iq2_s->noshuffle) { + ggml_cl_mul_mat_iq2_s_f32_adreno(backend, src0, src1, dst); + return; + } - ggml_tensor fake_src0 = *src0; - ggml_tensor_extra_cl fake_extra = {}; - fake_extra.data_device = f16_buf; - fake_extra.offset = 0; - fake_src0.type = GGML_TYPE_F16; - fake_src0.extra = &fake_extra; - fake_src0.view_src = nullptr; - fake_src0.view_offs = 0; - fake_src0.nb[0] = sizeof(ggml_fp16_t); - fake_src0.nb[1] = fake_src0.nb[0] * src0->ne[0]; - fake_src0.nb[2] = fake_src0.nb[1] * src0->ne[1]; - fake_src0.nb[3] = fake_src0.nb[2] * src0->ne[2]; + // iq2_xs x fp32 + if (src0t == GGML_TYPE_IQ2_XS && src1t == GGML_TYPE_F32 && extra0_iq2_xs->noshuffle) { + ggml_cl_mul_mat_iq2_xs_f32_adreno(backend, src0, src1, dst); + return; + } - ggml_cl_mul_mat(backend, &fake_src0, src1, dst); - return; - } + // iq2_xxs x fp32 + if (src0t == GGML_TYPE_IQ2_XXS && src1t == GGML_TYPE_F32 && extra0_iq2_xxs->noshuffle) { + ggml_cl_mul_mat_iq2_xxs_f32_adreno(backend, src0, src1, dst); + return; + } - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // q4_k x fp32 + if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) { + ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst); + return; + } - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + // q6_K x fp32 + if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(backend_ctx, src0)) { + ggml_cl_mul_mat_q6_K_f32_adreno(backend, src0, src1, dst); + return; + } -#ifdef GGML_OPENCL_SOA_Q - // view->extra stays pre-SoA; cast to the SoA struct would SIGSEGV. - // Follow view_src to reach the real SoA extra. - const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; - ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; - ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra; - ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra; - ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra; - ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)soa0_src->extra; - ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; - ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; - ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; - ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; - ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; - ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; -#endif + // q5_K x fp32 + if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32 && + enable_adreno_trans_weight_q5_K(backend_ctx, src0)) { + ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst); + return; + } + } // if (ne01 && ne1) +#endif // GGML_OPENCL_USE_ADRENO_KERNELS - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - GGML_TENSOR_LOCALS(int, ne1, src1, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); - GGML_TENSOR_LOCALS(int, ne, dst, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + // GEMM using local memory + // Current BK = 16, so ne00 % 16 == 0 + // + // Certain A7X compiler (E031.41) executes kernel_mul_mm_f32_f32_l4_lm poorly; + // matrices with ne11 <= 8 appears OK. + // Fallback to the MV style kernels for A7x and ne11 > 8. + // Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0. + static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS"); + static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0'); + // The 64-column tile wastes work on small F32 matrices, including block rotations. + const bool f32_use_mv = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + (backend_ctx->kernel_mul_mv_f32_f32_r2 != nullptr && ne00 <= 512)) && + src0t == GGML_TYPE_F32 && ne00 >= 256 && ne00 <= 1024 && ne00 % 16 == 0 && ne01 <= 512 && ne11 <= 16 && + ne02 == 1 && ne12 == 1 && ne13 == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && (offset0 | offset1) % 16 == 0; + if (src1t == GGML_TYPE_F32 && + ne00 % 16 == 0 && + ne11 > 1 && !f32_use_mv && + !(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 && + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) { + switch(src0t) { + case GGML_TYPE_F32: { + // Small-N f32 GEMV for the spec/MTP verify batch: the tiled GEMM + // below always computes a full 64x64 tile, so at ne11=3 with a + // skinny f32 weight (GDN ssm_alpha/ssm_beta, M=32) it launches one + // under-occupied WG at ~2.3% tile utilization. Route to a per-output + // (m,n) GEMV (64-thread WG, K-split + __local reduce) instead. + // 2D contiguous, small N + skinny M only. Default on for X2E, GGML_OPENCL_F32_MC=0/1 forces it. + static const char * f32_mc_env = getenv("GGML_OPENCL_F32_MC"); + const bool f32_mc = f32_mc_env ? atoi(f32_mc_env) != 0 : backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E; + if (f32_mc && ne11 >= 2 && ne11 <= 8 && ne01 <= 512 && (ne00 % 4 == 0) && + ne02 == 1 && ne12 == 1 && ne13 == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { + cl_kernel kmc = backend_ctx->kernel_gemv_f32_f32_mc; + int stride_a = ne00, stride_b = ne00, stride_d = ne01; + CL_CHECK(clSetKernelArg(kmc, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kmc, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kmc, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kmc, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kmc, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kmc, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kmc, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kmc, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kmc, 8, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kmc, 9, sizeof(int), &stride_a)); + CL_CHECK(clSetKernelArg(kmc, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(kmc, 11, sizeof(int), &stride_d)); + size_t gws[3] = {64, (size_t)ne01 * (size_t)ne11, 1}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kmc, 3, gws, lws, dst); + return; + } + kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - int r2 = ne12/ne02; - int r3 = ne13/ne03; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - GGML_ASSERT(ne00 == ne10); + cl_mem mem_src0 = extra0->data_device; + cl_mem mem_src1 = extra1->data_device; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // adreno GEMM/GEMV kernels do not support broadcast, assuming ne2 and ne3 are 1 for src1 - // so we handle broadcast here - if ((ne12 > 1 || ne13 > 1) && ne02 == 1 && ne03 == 1 && - src0t != GGML_TYPE_F16 && src0t != GGML_TYPE_F32) { - for (int i13 = 0; i13 < ne13; ++i13) { - for (int i12 = 0; i12 < ne12; ++i12) { - ggml_tensor s1 = *src1; - s1.ne[2] = 1; s1.ne[3] = 1; - s1.view_offs = src1->view_offs + (size_t)i12*nb12 + (size_t)i13*nb13; - ggml_tensor d = *dst; - d.ne[2] = 1; d.ne[3] = 1; - d.view_offs = dst->view_offs + (size_t)i12*nb2 + (size_t)i13*nb3; - ggml_cl_mul_mat(backend, src0, &s1, &d); - } - } - return; - } -#endif + cl_ulong nb00_cont = nb00; + cl_ulong nb01_cont = nb01; + cl_ulong nb02_cont = nb02; + cl_ulong nb03_cont = nb03; - int nth0 = 32; - int nth1 = 1; - int nrows = 1; - // The number of values produced by each subgroup - int ndst = 4; + cl_ulong nb10_cont = nb10; + cl_ulong nb11_cont = nb11; + cl_ulong nb12_cont = nb12; + cl_ulong nb13_cont = nb13; - cl_kernel kernel; + cl_ulong offset0_cont = offset0; + cl_ulong offset1_cont = offset1; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){ - // Two tiling assumptions these kernels make but nothing enforced: - // - // ne00 % TILESIZE_K(16): the K loop has no tail, so a K that does not - // divide folds 1-15 rows of whatever follows the operands into every - // output. - // - // ne01 % TILESIZE_M(64): mm_store_c_N guards the n direction with its - // `mask` argument but nothing guards m -- the store walks all 64 rows - // of the tile at a stride of M. When M does not divide, the last tile - // does not run off the end of the buffer, it writes 64 - (M % 64) - // values ON TOP OF the next column, so the result is silently wrong. - // Reachable on the KQV side for any head size >= 64 that is not a - // multiple of it (80, 96, 112). - // - // Attention shapes in the graph satisfy both -- head sizes are multiples - // of 64 and n_kv is padded -- which is why this has stayed latent. - // Declining leaves the odd shapes on the generic GEMM, which handles them. - if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && - (ne00 % 16) == 0 && (ne01 % 64) == 0 && (ne12 % ne02) == 0 && - // the KQ/KQV image kernels do not handle dim 3 (multi-stream batches) - ne03 == 1 && ne13 == 1 && - // dst is wrapped with image1d_buffer, the size limit applies, also src0 - (ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4 <= backend_ctx->image_max_buffer_size)) { - // For KQ. - // - // Layout admission, mirroring the KQV arm below. The KQ kernel takes - // no stride arguments for A or B: it derives them as K*D_A*2 and - // K*D_B*4, i.e. it assumes both operands pack exactly D heads of K - // elements per row. Every real KV-cache view and permuted-Q view - // does, but a view spanning part of a wider allocation does not, and - // the kernel then walks the wrong rows with nothing to range-check - // it. Gate on the packed layout itself rather than on the stride - // ORDERING, which a wider parent satisfies just as well. - const bool kq_packed_a = (nb01 == (cl_ulong)ne00 * ne02 * ggml_type_size(src0t)) && - (nb02 == (cl_ulong)ne00 * ggml_type_size(src0t)); - const bool kq_packed_b = (nb11 == (cl_ulong)ne10 * ne12 * ggml_type_size(src1t)) && - (nb12 == (cl_ulong)ne10 * ggml_type_size(src1t)); - // - // ggml_is_permuted(src0) stands in for "K is head-major", but it is - // only a proxy and it COLLAPSES at n_head_kv == 1: with a single - // head there is no head stride to be out of order, so nb01 == nb02 - // and the view reports itself unpermuted. Such a KQ was declined - // here and fell through to the generic GEMM (gemma-4 E2B, and any - // other multi-query model). The packed check above is the contract - // the kernel actually needs -- it pins both strides exactly -- so - // require permutedness only where there is more than one head for - // it to mean anything. - // - // Default on; GGML_OPENCL_KQ_NHEAD_KV1=0 restores the old proxy so - // the two routings can be compared in one binary. - static const char * kq_nhkv1_env = getenv("GGML_OPENCL_KQ_NHEAD_KV1"); - static const bool kq_nhkv1_on = - (kq_nhkv1_env == nullptr || kq_nhkv1_env[0] != '0'); - if ((ggml_is_permuted(src0) || (ne02 == 1 && kq_nhkv1_on)) && ggml_is_permuted(src1) && - kq_packed_a && kq_packed_b && - ((nb01 * ne01 / 4)/4 <= backend_ctx->image_max_buffer_size) && - nb00 <= nb02 && - nb02 <= nb01 && - nb01 <= nb03 && - nb10 <= nb12 && - nb12 <= nb11 && - nb11 <= nb13) { - ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst, /*is_kq =*/ true); - return; - } - // For KQV. Reaching this arm is what makes the op a KQV; the callee - // is told so explicitly rather than re-deriving it from the strides - // the arm above has already ruled on. - if (!ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && - ((nb02 * ne02 / 4)/4 <= backend_ctx->image_max_buffer_size)) { - ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst, /*is_kq =*/ false); - return; - } - } + if (!ggml_is_contiguous(src0)) { + backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); + ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, + nb00_cont, nb01_cont, nb02_cont, nb03_cont); + mem_src0 = backend_ctx->prealloc_src0.buffer; + offset0_cont = 0; + } - static const char * mm_kq_gqa_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_IMG"); - static const bool mm_kq_gqa_img_on = (mm_kq_gqa_img_env == nullptr || mm_kq_gqa_img_env[0] != '0'); - static const char * mm_kq_gqa_r4_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_R4_IMG"); - static const bool mm_kq_gqa_r4_img_on = (mm_kq_gqa_r4_img_env == nullptr || mm_kq_gqa_r4_img_env[0] != '0'); - const bool img_r4_gate = - mm_kq_gqa_r4_img_on && - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img != nullptr && - ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && - (ne12 % ne02) == 0 && (ne12 / ne02) == 4 && (ne13 / ne03) == 1; - if (mm_kq_gqa_img_on && - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img != nullptr && - ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && - (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { - const size_t nb00_bytes = sizeof(uint16_t); - const size_t k_bytes_span = - (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + - (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + - (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + - (size_t)ne00 * nb00_bytes; + if (!ggml_is_contiguous(src1)) { + backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); + ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, + nb10_cont, nb11_cont, nb12_cont, nb13_cont); + mem_src1 = backend_ctx->prealloc_src1.buffer; + offset1_cont = 0; + } - const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; - const size_t k_pixels = k_bytes >> 4; - if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { - cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img; - cl_mem K_img = ggml_cl_img_pool_get_or_create( - backend_ctx, backend_ctx->kq_img_pool, - extra0->data_device, offset0, k_bytes, CL_FLOAT); - if (K_img != nullptr) { - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - const int nth0_d = 64; - const int64_t n_wg_x = ne01 / 16; - size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; - size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_F16: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (ggml_cl_can_use_adreno_xmem_gemm_f16_f32(backend_ctx, src0, src1, dst)) { + ggml_cl_mul_mat_f16_f32_adreno_xmem(backend, src0, src1, dst); return; } - } - } +#endif + static const char * mm_narrow_n_env = getenv("GGML_OPENCL_MM_NARROW_N"); + const bool use_narrow_n = (mm_narrow_n_env == nullptr || mm_narrow_n_env[0] != '0') && ne11 >= 2 && ne11 <= 8; + kernel = use_narrow_n ? backend_ctx->kernel_mul_mm_f16_f32_l4_lm_n8 : backend_ctx->kernel_mul_mm_f16_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - // r2=4 specialization - if (img_r4_gate) { - const size_t nb00_bytes = sizeof(uint16_t); - const size_t k_bytes_span = - (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + - (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + - (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + - (size_t)ne00 * nb00_bytes; - const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; - const size_t k_pixels = k_bytes >> 4; - if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { - cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img; - cl_mem K_img = ggml_cl_img_pool_get_or_create( - backend_ctx, backend_ctx->kq_img_pool, - extra0->data_device, offset0, k_bytes, CL_FLOAT); - if (K_img != nullptr) { - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - const int nth0_d = 64; - const int64_t n_wg_x = ne01 / 16; - size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; - size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; + cl_mem mem_src0 = extra0->data_device; + cl_mem mem_src1 = extra1->data_device; + + cl_ulong nb00_cont = nb00; + cl_ulong nb01_cont = nb01; + cl_ulong nb02_cont = nb02; + cl_ulong nb03_cont = nb03; + + cl_ulong nb10_cont = nb10; + cl_ulong nb11_cont = nb11; + cl_ulong nb12_cont = nb12; + cl_ulong nb13_cont = nb13; + + cl_ulong offset0_cont = offset0; + cl_ulong offset1_cont = offset1; + + if (!ggml_is_contiguous(src0)) { + backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); + ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, + nb00_cont, nb01_cont, nb02_cont, nb03_cont); + mem_src0 = backend_ctx->prealloc_src0.buffer; + offset0_cont = 0; } - } - } - // DK=256, r2=2 specialization - static const char * mm_kq_r2_dk256_env = getenv("GGML_OPENCL_MM_KQ_GQA_R2_DK256_IMG"); - static const bool mm_kq_r2_dk256_on = (mm_kq_r2_dk256_env != nullptr && mm_kq_r2_dk256_env[0] != '0'); - if (mm_kq_r2_dk256_on && - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img != nullptr && - ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 256 && - (ne12 % ne02) == 0 && (ne12 / ne02) == 2 && (ne13 / ne03) == 1) { - const size_t nb00_bytes = sizeof(uint16_t); - const size_t k_bytes_span = - (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + - (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + - (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + - (size_t)ne00 * nb00_bytes; - const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; - const size_t k_pixels = k_bytes >> 4; - if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { - cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img; - cl_mem K_img = ggml_cl_img_pool_get_or_create( - backend_ctx, backend_ctx->kq_img_pool, - extra0->data_device, offset0, k_bytes, CL_FLOAT); - if (K_img != nullptr) { - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + if (!ggml_is_contiguous(src1)) { + backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); + ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, + nb10_cont, nb11_cont, nb12_cont, nb13_cont); + mem_src1 = backend_ctx->prealloc_src1.buffer; + offset1_cont = 0; + } - const int nth0_d = 64; - const int64_t n_wg_x = ne01 / 16; - size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; - size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + + // BN=8/TN=1 keeps 128 threads for small batches, including attention KQ/KQV. + const int bn = use_narrow_n ? 8 : 64; + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, bn)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q1_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; } - } - } - // GQA-coalesced KQV for decode using image1d_buffer_t - static const char * mm_kqv_gqa_img_env = getenv("GGML_OPENCL_MM_KQV_GQA_IMG"); - static const bool mm_kqv_gqa_img_on = (mm_kqv_gqa_img_env != nullptr && mm_kqv_gqa_img_env[0] != '0'); - if (mm_kqv_gqa_img_on && - backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img != nullptr && - ne11 == 1 && ne01 == 128 && - (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { - const size_t nb00_bytes = sizeof(uint16_t); - const size_t v_bytes_span = - (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + - (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + - (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + - (size_t)ne00 * nb00_bytes; - const size_t v_bytes = (v_bytes_span + 7) & ~(size_t)7; - const size_t v_pixels = v_bytes >> 3; - if (v_pixels > 0 && v_pixels <= backend_ctx->image_max_buffer_size) { - cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img; - cl_mem V_img = ggml_cl_img_pool_get_or_create( - backend_ctx, backend_ctx->kqv_img_pool, - extra0->data_device, offset0, v_bytes, CL_HALF_FLOAT); - if (V_img != nullptr) { - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &V_img)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + kernel = backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - const int nth0_d = 64; - const int64_t n_wg_x = ne01 / 8; - size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; - size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q2_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; } + + kernel = backend_ctx->kernel_mul_mm_q2_0_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; } - } - } + case GGML_TYPE_Q4_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - if (ne01 && ne1 && use_adreno_kernels(backend_ctx, src0)) { - // NOTE: Kernels using image1d_buffer_t (e.g., src0_q) would normally require - // a limit check, but q4_0 / q4_1 tensors are very unlikely to exceed that - // limit, so the check is omitted. + kernel = backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - // q1_0 x fp32 - if (src0t == GGML_TYPE_Q1_0 && src1t == GGML_TYPE_F32 && - enable_adreno_trans_weight(backend_ctx, src0)) { - ggml_cl_mul_mat_q1_0_f32_adreno(backend, src0, src1, dst); + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; - } + } + case GGML_TYPE_Q4_1: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - // q4_0 x fp32 - if(src0t == GGML_TYPE_Q4_0 && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_q4_0_f32_adreno(backend, src0, src1, dst); - return; - } + kernel = backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - // q4_1 x fp32 - if (src0t == GGML_TYPE_Q4_1 && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_q4_1_f32_adreno(backend, src0, src1, dst); - return; - } + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - // q5_0 x fp32 - if (src0t == GGML_TYPE_Q5_0 && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_q5_0_f32_adreno(backend, src0, src1, dst); - return; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); - // q5_1 x fp32 - if (src0t == GGML_TYPE_Q5_1 && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_q5_1_f32_adreno(backend, src0, src1, dst); - return; - } + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - // iq4_nl x fp32 - if (src0t == GGML_TYPE_IQ4_NL && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_iq4_nl_f32_adreno(backend, src0, src1, dst); - return; - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q5_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - // q8_0 x fp32 - if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 && - enable_adreno_trans_weight(backend_ctx, src0)) { - ggml_cl_mul_mat_q8_0_f32_adreno(backend, src0, src1, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; - } + } + case GGML_TYPE_Q5_1: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - // q4_k x fp32 - if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) { - ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst); - return; - } + kernel = backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - // q6_K x fp32 - if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(backend_ctx, src0)) { - ggml_cl_mul_mat_q6_K_f32_adreno(backend, src0, src1, dst); - return; - } + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - // q5_K x fp32 - if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32 && - enable_adreno_trans_weight_q5_K(backend_ctx, src0)) { - ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst); - return; - } - } // if (ne01 && ne1) -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); - // GEMM using local memory - // Current BK = 16, so ne00 % 16 == 0 - // - // Certain A7X compiler (E031.41) executes kernel_mul_mm_f32_f32_l4_lm poorly; - // matrices with ne11 <= 8 appears OK. - // Fallback to the MV style kernels for A7x and ne11 > 8. - // Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0. - static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS"); - static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0'); - if (src1t == GGML_TYPE_F32 && - ne00 % 16 == 0 && - ne11 > 1 && - !(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 && - backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) { - switch(src0t) { - case GGML_TYPE_F32: { - // Small-N f32 GEMV for the spec/MTP verify batch: the tiled GEMM - // below always computes a full 64x64 tile, so at ne11=3 with a - // skinny f32 weight (GDN ssm_alpha/ssm_beta, M=32) it launches one - // under-occupied WG at ~2.3% tile utilization. Route to a per-output - // (m,n) GEMV (64-thread WG, K-split + __local reduce) instead. - // Opt-in GGML_OPENCL_F32_MC=1; 2D contiguous, small N + skinny M only. - static const bool f32_mc = (getenv("GGML_OPENCL_F32_MC") != nullptr); - if (f32_mc && ne11 >= 2 && ne11 <= 8 && ne01 <= 512 && (ne00 % 4 == 0) && - ne02 == 1 && ne12 == 1 && ne13 == 1 && - ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - cl_kernel kmc = backend_ctx->kernel_gemv_f32_f32_mc; - int stride_a = ne00, stride_b = ne00, stride_d = ne01; - CL_CHECK(clSetKernelArg(kmc, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kmc, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kmc, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kmc, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kmc, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kmc, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kmc, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kmc, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kmc, 8, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kmc, 9, sizeof(int), &stride_a)); - CL_CHECK(clSetKernelArg(kmc, 10, sizeof(int), &stride_b)); - CL_CHECK(clSetKernelArg(kmc, 11, sizeof(int), &stride_d)); - size_t gws[3] = {64, (size_t)ne01 * (size_t)ne11, 1}; - size_t lws[3] = {64, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kmc, 3, gws, lws, dst); - return; + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q8_0: { + if (ne11 < 32) { + break; } - kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm; + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - cl_mem mem_src0 = extra0->data_device; - cl_mem mem_src1 = extra1->data_device; - - cl_ulong nb00_cont = nb00; - cl_ulong nb01_cont = nb01; - cl_ulong nb02_cont = nb02; - cl_ulong nb03_cont = nb03; - - cl_ulong nb10_cont = nb10; - cl_ulong nb11_cont = nb11; - cl_ulong nb12_cont = nb12; - cl_ulong nb13_cont = nb13; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - cl_ulong offset0_cont = offset0; - cl_ulong offset1_cont = offset1; + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - if (!ggml_is_contiguous(src0)) { - backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); - ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, - nb00_cont, nb01_cont, nb02_cont, nb03_cont); - mem_src0 = backend_ctx->prealloc_src0.buffer; - offset0_cont = 0; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_IQ4_NL: { + if (ne11 < 32) { + break; } - - if (!ggml_is_contiguous(src1)) { - backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); - ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, - nb10_cont, nb11_cont, nb12_cont, nb13_cont); - mem_src1 = backend_ctx->prealloc_src1.buffer; - offset1_cont = 0; + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); + kernel = backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); @@ -23961,87 +32404,282 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_IQ4_XS: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_iq4_xs_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_xs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_xs->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq4_xs->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq4_xs->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q2_K: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q2_k_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q2_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q3_K: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q3_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q3_K->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q3_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q3_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_F16: { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (ggml_cl_can_use_adreno_xmem_gemm_f16_f32(backend_ctx, src0, src1, dst)) { - ggml_cl_mul_mat_f16_f32_adreno_xmem(backend, src0, src1, dst); - return; + case GGML_TYPE_IQ3_S: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; } -#endif - kernel = backend_ctx->kernel_mul_mm_f16_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + kernel = backend_ctx->kernel_mul_mm_iq3_s_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - cl_mem mem_src0 = extra0->data_device; - cl_mem mem_src1 = extra1->data_device; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq3_s->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq3_s->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq3_s->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq3_s->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_iq3_s->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - cl_ulong nb00_cont = nb00; - cl_ulong nb01_cont = nb01; - cl_ulong nb02_cont = nb02; - cl_ulong nb03_cont = nb03; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_IQ3_XXS: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - cl_ulong nb10_cont = nb10; - cl_ulong nb11_cont = nb11; - cl_ulong nb12_cont = nb12; - cl_ulong nb13_cont = nb13; + kernel = backend_ctx->kernel_mul_mm_iq3_xxs_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; - cl_ulong offset0_cont = offset0; - cl_ulong offset1_cont = offset1; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - if (!ggml_is_contiguous(src0)) { - backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); - ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, - nb00_cont, nb01_cont, nb02_cont, nb03_cont); - mem_src0 = backend_ctx->prealloc_src0.buffer; - offset0_cont = 0; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq3_xxs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq3_xxs->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq3_xxs->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - if (!ggml_is_contiguous(src1)) { - backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); - ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, - nb10_cont, nb11_cont, nb12_cont, nb13_cont); - mem_src1 = backend_ctx->prealloc_src1.buffer; - offset1_cont = 0; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_IQ1_M: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + kernel = backend_ctx->kernel_mul_mm_iq1_m_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq1_m->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq1_m->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq1_m->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_Q1_0: { + case GGML_TYPE_IQ1_S: { if (ne11 < 32) { break; } @@ -24049,41 +32687,43 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co break; } - kernel = backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + kernel = backend_ctx->kernel_mul_mm_iq1_s_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq1_s->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq1_s->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq1_s->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_Q4_0: { + case GGML_TYPE_IQ2_S: { if (ne11 < 32) { break; } @@ -24091,41 +32731,45 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co break; } - kernel = backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + kernel = backend_ctx->kernel_mul_mm_iq2_s_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_s->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_s->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_s->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq2_s->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_iq2_s->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_Q4_1: { + case GGML_TYPE_IQ2_XS: { if (ne11 < 32) { break; } @@ -24133,42 +32777,43 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co break; } - kernel = backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + kernel = backend_ctx->kernel_mul_mm_iq2_xs_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_xs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_xs->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_xs->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_Q5_0: { + case GGML_TYPE_IQ2_XXS: { if (ne11 < 32) { break; } @@ -24176,42 +32821,43 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co break; } - kernel = backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + kernel = backend_ctx->kernel_mul_mm_iq2_xxs_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_xxs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_xxs->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_xxs->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_Q5_1: { + case GGML_TYPE_Q4_K: { if (ne11 < 32) { break; } @@ -24219,17 +32865,18 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co break; } - kernel = backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + kernel = backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm; + // (BM*BN)/(TM*TN): Intel uses an 8x8 microtile (WG=64), others 4x8 (WG=128) + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_K->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_K->s)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_K->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_K->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); @@ -24255,7 +32902,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_Q8_0: { + case GGML_TYPE_Q5_K: { if (ne11 < 32) { break; } @@ -24263,32 +32910,35 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co break; } - kernel = backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + kernel = backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; // Intel 8x8 microtile int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_K->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; @@ -24297,7 +32947,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_IQ4_NL: { + case GGML_TYPE_Q6_K: { if (ne11 < 32) { break; } @@ -24305,15 +32955,88 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co break; } - kernel = backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm; + kernel = backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + default: + break; + } + } + + if (src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32 && + src0->ne[1] > 32 && // M > 32 + src1->ne[1] > 32 && // N > 32 + src0->ne[0] > 32 && // K > 32 + src0->ne[2] == 1 && src0->ne[3] == 1 && + src1->ne[2] == 1 && src1->ne[3] == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && + backend_ctx->kernel_mul_mat_f16_f32_tiled != NULL) { + ggml_cl_mul_mat_f16_f32_tiled(backend, src0, src1, dst); + return; + } + + if (!ggml_is_transposed(src0) && + !ggml_is_transposed(src1) && + src1t == GGML_TYPE_F32 && + ne00%32 == 0 && + ne11 > 2) { +#ifdef GGML_OPENCL_SOA_Q + // Set up kernel. + switch(src0t) { + case GGML_TYPE_Q4_0: + // This should have been satisfied. + GGML_ASSERT(ne11 == ne1); + GGML_ASSERT(ne01 == ne0); + + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); @@ -24321,261 +33044,574 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + break; + default: + break; + } - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; + // Launch kernel. + if (src0t == GGML_TYPE_Q4_0) { + size_t global_work_size[] = {(size_t)(ne01 + 7)/8*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; + if (backend_ctx->gpu_family == INTEL) { + // Set global size for Intel. It uses 16x output values. + global_work_size[0] = (size_t)(ne01 + 15)/16*nth0; + global_work_size[1] = (size_t)ne11*nth1; + global_work_size[2] = (size_t)ne12*ne13; } - case GGML_TYPE_Q4_K: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } +#else // GGML_OPENCL_SOA_Q + // TODO: add block_q4_0 variant. +#endif // GGML_OPENCL_SOA_Q + } + + // use custom matrix x vector kernel + bool use_f16_mrow = false; + switch (src0t) { + case GGML_TYPE_F32: + //GGML_ASSERT(ne02 == ne12); + GGML_ASSERT(src1t == GGML_TYPE_F32); + kernel = backend_ctx->kernel_mul_mat_f32_f32; + nrows = 4; + if (f32_use_mv && ne11 > 1 && backend_ctx->kernel_mul_mv_f32_f32_r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mv_f32_f32_r2; + } + + if (backend_ctx->gpu_family == INTEL) { + nth0 = 32; + nth1 = 1; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); + break; + case GGML_TYPE_F16: + //GGML_ASSERT(ne02 == ne12); + if (backend_ctx->gpu_family == INTEL) { + nth0 = 32; + nth1 = 1; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + + if (src1t == GGML_TYPE_F32) { + // heuristic for packing more work for Adreno + const bool adreno_use_lane_split = + backend_ctx->gpu_family == ADRENO && + ne11 == 1 && + ne01 >= 8 && + ne00 % 4 == 0 && + r3 == 1 && r2 >= 1 && r2 <= 8 && + (ne12 % r2) == 0; + + if (ne11 * ne12 < 4) { + // Decode (single token): the legacy _1row runs one 64-lane + // subgroup per WG (one output row), under-utilizing BW. Route the + // wide f16 weight matmuls (attn proj + lm_head) to the multi-row + // variant: MROW rows per WG -> more loads in flight + activation + // staged once in __local. ne00<=8192 bounds the LDS. The mrow WG + // is 64 x MROW = 1024 work-items (> Intel's 512 max) and reduces + // within a 64-wide subgroup, so skip on Intel. + if (backend_ctx->f16_mrow && backend_ctx->gpu_family != INTEL && + backend_ctx->kernel_mul_mat_f16_f32_mrow != nullptr && + ne00 >= 128 && ne01 >= 8 && ne00 % 4 == 0 && ne00 <= 8192) { + // The register-blocked / half8 variants cast the src0 row pointer to + // half4 / half8 (8- and 16-byte loads) with no scalar fallback inside + // the kernel. ne00 % 4 == 0 constrains the element count per row, NOT + // the byte stride between rows: a permuted or strided src0 (or a view + // at an odd offset) can leave nb01/nb02/nb03 unaligned. Only take them + // when every row this dispatch touches is aligned; the base mrow kernel + // re-checks per row and falls back to its scalar loop. + const cl_ulong row_addr_bits = offset0 | nb01 | nb02 | nb03; + const bool aligned8 = (row_addr_bits & 7) == 0; + const bool aligned16 = (row_addr_bits & 15) == 0; + + // Register-blocked variants: each subgroup does RPT rows (more + // weight loads in flight per lane). 8/16 use half8 (128-bit) + // loads, gated on ne00 % 8 == 0. + const int rpt = backend_ctx->f16_mrow_rpt; + if (rpt == 16 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2; + } else if (rpt == 8 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8; + } else if (rpt == 4 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r4; + } else if (rpt == 2 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r2; + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow; + } + use_f16_mrow = true; + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_1row; + } + } else if (adreno_use_lane_split && ne00 >= 64 && ne00 <= 128) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq; + nrows = 1; + } else if (adreno_use_lane_split && r2 >= 2 && ne00 > 128 && ne00 <= 256) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls; + nrows = 1; + } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { + // multi-output decode variants when Q is a single row + static const char * mm_force_l4_env = getenv("GGML_OPENCL_MM_F16_FORCE_L4"); + static const bool mm_force_l4_on = (mm_force_l4_env != nullptr && mm_force_l4_env[0] != '0'); + const bool can_multi_out = !mm_force_l4_on && ne11 == 1 && ne01 >= 64 && ne01 % 8 == 0; + // paired-K-row variant that doubles per-wave-cycle + static const char * mm_kq_pair_env = getenv("GGML_OPENCL_MM_KQ_PAIR"); + static const bool mm_kq_pair_on = (mm_kq_pair_env != nullptr && mm_kq_pair_env[0] != '0'); + // GQA-coalesced variant that reads each K-row once and + // emits gqa_ratio outputs + static const char * mm_kq_gqa_env = getenv("GGML_OPENCL_MM_KQ_GQA"); + static const bool mm_kq_gqa_on = (mm_kq_gqa_env != nullptr && mm_kq_gqa_env[0] != '0'); + // GQA-coalesced KQV variant (DK=128/r2=8/r3=1) that reads + // each V slab once per K-head and emits all r2 Q-heads + static const char * mm_kqv_gqa_env = getenv("GGML_OPENCL_MM_KQV_GQA"); + static const bool mm_kqv_gqa_on = (mm_kqv_gqa_env != nullptr && mm_kqv_gqa_env[0] != '0'); + if (can_multi_out && (ne01 % 16) == 0 && ne00 == 128 && r2 == 8 && r3 == 1 && mm_kq_gqa_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4; + nrows = 1; + } else if (can_multi_out && ne00 <= 256 && mm_kq_pair_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair; + nrows = 1; + } else if (can_multi_out && ne00 <= 256 && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8; + nrows = 1; + } else if (can_multi_out && ne01 == 128 && r2 == 8 && r3 == 1 && mm_kqv_gqa_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa; + nrows = 1; + } else if (can_multi_out && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8; + nrows = 1; + } else if (ne11 == 1) { + // Decode shapes that don't satisfy the x8/y8 row + // constraints (ne01 < 64 or ne01 % 8 != 0) fall back to + // upstream's 4-output _dr kernel. + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr; + nrows = 1; // not used by this kernel + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4; + nrows = ne11; + } + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32; + nrows = 4; } + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f16; + nrows = 4; + } - kernel = backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm; - // (BM*BN)/(TM*TN): Intel uses an 8x8 microtile (WG=64), others 4x8 (WG=128) - nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_K->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_K->s)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_K->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_K->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); + if (use_f16_mrow) { + const int MROW = 16; // must match MROW in mul_mv_f16_f32_mrow.cl + // rows-per-subgroup multiplier for the selected variant: + // 1/2/4 -> half4 register blocking; 8 -> half8(1 row); 16 -> half8(2 rows) + const int rpt = backend_ctx->f16_mrow_rpt; + int rmul; + if (rpt == 16) rmul = (ne00 % 8 == 0) ? 2 : 1; + else if (rpt == 8) rmul = 1; + else rmul = rpt; // 1,2,4 + const int rows_per_wg = MROW * rmul; + // __local activation buffer: ne00 floats, rounded up for float4 access + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float) * ((ne00 + 3) / 4 * 4), nullptr)); + size_t mrow_global[] = { (size_t)((ne01 + rows_per_wg - 1) / rows_per_wg) * 64, (size_t)ne11 * MROW, (size_t)ne12 * ne13 }; + size_t mrow_local[] = { 64, (size_t)MROW, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, mrow_global, mrow_local, dst); return; } - case GGML_TYPE_Q5_K: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } - - kernel = backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm; - nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; // Intel 8x8 microtile - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_K->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_K->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; + break; + case GGML_TYPE_Q1_0: { +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q1_0_f32_flat; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; + // nth0 - subgroup size + // nth1 - number of subgroups per workgroup + // ndst - number of output values per workgroup = output per subgroup * number of subgroups + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); } - case GGML_TYPE_Q6_K: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } - kernel = backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q1_0_f32; - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_Q2_0: { + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q2_0_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q2_0_f32; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - default: - break; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; } - } - - if (src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32 && - src0->ne[1] > 32 && // M > 32 - src1->ne[1] > 32 && // N > 32 - src0->ne[0] > 32 && // K > 32 - src0->ne[2] == 1 && src0->ne[3] == 1 && - src1->ne[2] == 1 && src1->ne[3] == 1 && - ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && - backend_ctx->kernel_mul_mat_f16_f32_tiled != NULL) { - ggml_cl_mul_mat_f16_f32_tiled(backend, src0, src1, dst); - return; - } + case GGML_TYPE_Q4_0: + // This should have been satisfied. + GGML_ASSERT(ne11 == ne1); + GGML_ASSERT(ne01 == ne0); - if (!ggml_is_transposed(src0) && - !ggml_is_transposed(src1) && - src1t == GGML_TYPE_F32 && - ne00%32 == 0 && - ne11 > 2) { #ifdef GGML_OPENCL_SOA_Q - // Set up kernel. - switch(src0t) { - case GGML_TYPE_Q4_0: - // This should have been satisfied. - GGML_ASSERT(ne11 == ne1); - GGML_ASSERT(ne01 == ne0); + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; + ndst = 8; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; + ndst =8; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#else // GGML_OPENCL_SOA_Q + if (backend_ctx->gpu_family == INTEL) { + // Use 1D local size. Each workgroup is a SIMD group. Each SIMD + // group produces N_DST (4 for Q4_0 kernel) values in the result. + // The number of workgroups on dim 0 (the leading dimension) is + // the nearest multiple of 4 that covers ne0 (equals ne01). + nth0 = 16; + nth1 = 1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); - break; - default: - break; - } + kernel = backend_ctx->kernel_mul_mat_q4_0_f32; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; - // Launch kernel. - if (src0t == GGML_TYPE_Q4_0) { - size_t global_work_size[] = {(size_t)(ne01 + 7)/8*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_v; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + case GGML_TYPE_Q4_1: { +#ifdef GGML_OPENCL_SOA_Q if (backend_ctx->gpu_family == INTEL) { - // Set global size for Intel. It uses 16x output values. - global_work_size[0] = (size_t)(ne01 + 15)/16*nth0; - global_work_size[1] = (size_t)ne11*nth1; - global_work_size[2] = (size_t)ne12*ne13; + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } -#else // GGML_OPENCL_SOA_Q - // TODO: add block_q4_0 variant. + kernel = backend_ctx->kernel_mul_mv_q4_1_f32_flat; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); +#else + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + + kernel = backend_ctx->kernel_mul_mv_q4_1_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q - } + break; + } + case GGML_TYPE_Q5_0: { +#ifdef GGML_OPENCL_SOA_Q + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // use custom matrix x vector kernel - bool use_f16_mrow = false; - switch (src0t) { - case GGML_TYPE_F32: - //GGML_ASSERT(ne02 == ne12); - GGML_ASSERT(src1t == GGML_TYPE_F32); - kernel = backend_ctx->kernel_mul_mat_f32_f32; - nrows = 4; + kernel = backend_ctx->kernel_mul_mv_q5_0_f32_flat; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); +#else if (backend_ctx->gpu_family == INTEL) { - nth0 = 32; + nth0 = 16; nth1 = 1; + ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; + ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } + kernel = backend_ctx->kernel_mul_mv_q5_0_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); @@ -24585,146 +33621,63 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q break; - case GGML_TYPE_F16: - //GGML_ASSERT(ne02 == ne12); + } + case GGML_TYPE_Q5_1: { +#ifdef GGML_OPENCL_SOA_Q if (backend_ctx->gpu_family == INTEL) { - nth0 = 32; + nth0 = 16; nth1 = 1; + ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; + ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } - if (src1t == GGML_TYPE_F32) { - // heuristic for packing more work for Adreno - const bool adreno_use_lane_split = - backend_ctx->gpu_family == ADRENO && - ne11 == 1 && - ne01 >= 8 && - ne00 % 4 == 0 && - r3 == 1 && r2 >= 1 && r2 <= 8 && - (ne12 % r2) == 0; - - if (ne11 * ne12 < 4) { - // Decode (single token): the legacy _1row runs one 64-lane - // subgroup per WG (one output row), under-utilizing BW. Route the - // wide f16 weight matmuls (attn proj + lm_head) to the multi-row - // variant: MROW rows per WG -> more loads in flight + activation - // staged once in __local. ne00<=8192 bounds the LDS. The mrow WG - // is 64 x MROW = 1024 work-items (> Intel's 512 max) and reduces - // within a 64-wide subgroup, so skip on Intel. - if (backend_ctx->f16_mrow && backend_ctx->gpu_family != INTEL && - backend_ctx->kernel_mul_mat_f16_f32_mrow != nullptr && - ne00 >= 128 && ne01 >= 8 && ne00 % 4 == 0 && ne00 <= 8192) { - // The register-blocked / half8 variants cast the src0 row pointer to - // half4 / half8 (8- and 16-byte loads) with no scalar fallback inside - // the kernel. ne00 % 4 == 0 constrains the element count per row, NOT - // the byte stride between rows: a permuted or strided src0 (or a view - // at an odd offset) can leave nb01/nb02/nb03 unaligned. Only take them - // when every row this dispatch touches is aligned; the base mrow kernel - // re-checks per row and falls back to its scalar loop. - const cl_ulong row_addr_bits = offset0 | nb01 | nb02 | nb03; - const bool aligned8 = (row_addr_bits & 7) == 0; - const bool aligned16 = (row_addr_bits & 15) == 0; + kernel = backend_ctx->kernel_mul_mv_q5_1_f32_flat; - // Register-blocked variants: each subgroup does RPT rows (more - // weight loads in flight per lane). 8/16 use half8 (128-bit) - // loads, gated on ne00 % 8 == 0. - const int rpt = backend_ctx->f16_mrow_rpt; - if (rpt == 16 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2; - } else if (rpt == 8 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8; - } else if (rpt == 4 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r4; - } else if (rpt == 2 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r2; - } else { - kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow; - } - use_f16_mrow = true; - } else { - kernel = backend_ctx->kernel_mul_mat_f16_f32_1row; - } - } else if (adreno_use_lane_split && ne00 >= 64 && ne00 <= 128) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq; - nrows = 1; - } else if (adreno_use_lane_split && r2 >= 2 && ne00 > 128 && ne00 <= 256) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls; - nrows = 1; - } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { - // multi-output decode variants when Q is a single row - static const char * mm_force_l4_env = getenv("GGML_OPENCL_MM_F16_FORCE_L4"); - static const bool mm_force_l4_on = (mm_force_l4_env != nullptr && mm_force_l4_env[0] != '0'); - const bool can_multi_out = !mm_force_l4_on && ne11 == 1 && ne01 >= 64 && ne01 % 8 == 0; - // paired-K-row variant that doubles per-wave-cycle - static const char * mm_kq_pair_env = getenv("GGML_OPENCL_MM_KQ_PAIR"); - static const bool mm_kq_pair_on = (mm_kq_pair_env != nullptr && mm_kq_pair_env[0] != '0'); - // GQA-coalesced variant that reads each K-row once and - // emits gqa_ratio outputs - static const char * mm_kq_gqa_env = getenv("GGML_OPENCL_MM_KQ_GQA"); - static const bool mm_kq_gqa_on = (mm_kq_gqa_env != nullptr && mm_kq_gqa_env[0] != '0'); - // GQA-coalesced KQV variant (DK=128/r2=8/r3=1) that reads - // each V slab once per K-head and emits all r2 Q-heads - static const char * mm_kqv_gqa_env = getenv("GGML_OPENCL_MM_KQV_GQA"); - static const bool mm_kqv_gqa_on = (mm_kqv_gqa_env != nullptr && mm_kqv_gqa_env[0] != '0'); - if (can_multi_out && (ne01 % 16) == 0 && ne00 == 128 && r2 == 8 && r3 == 1 && mm_kq_gqa_on && - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4; - nrows = 1; - } else if (can_multi_out && ne00 <= 256 && mm_kq_pair_on && - backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair; - nrows = 1; - } else if (can_multi_out && ne00 <= 256 && - backend_ctx->kernel_mul_mat_f16_f32_l4_x8 != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8; - nrows = 1; - } else if (can_multi_out && ne01 == 128 && r2 == 8 && r3 == 1 && mm_kqv_gqa_on && - backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa; - nrows = 1; - } else if (can_multi_out && - backend_ctx->kernel_mul_mat_f16_f32_l4_y8 != nullptr) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8; - nrows = 1; - } else if (ne11 == 1) { - // Decode shapes that don't satisfy the x8/y8 row - // constraints (ne01 < 64 or ne01 % 8 != 0) fall back to - // upstream's 4-output _dr kernel. - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr; - nrows = 1; // not used by this kernel - } else { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4; - nrows = ne11; - } - } else { - kernel = backend_ctx->kernel_mul_mat_f16_f32; - nrows = 4; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); +#else + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; } else { - kernel = backend_ctx->kernel_mul_mat_f16_f16; - nrows = 4; + GGML_ASSERT(false && "TODO: Unknown GPU"); } + kernel = backend_ctx->kernel_mul_mv_q5_1_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); @@ -24734,42 +33687,18 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); - if (use_f16_mrow) { - const int MROW = 16; // must match MROW in mul_mv_f16_f32_mrow.cl - // rows-per-subgroup multiplier for the selected variant: - // 1/2/4 -> half4 register blocking; 8 -> half8(1 row); 16 -> half8(2 rows) - const int rpt = backend_ctx->f16_mrow_rpt; - int rmul; - if (rpt == 16) rmul = (ne00 % 8 == 0) ? 2 : 1; - else if (rpt == 8) rmul = 1; - else rmul = rpt; // 1,2,4 - const int rows_per_wg = MROW * rmul; - // __local activation buffer: ne00 floats, rounded up for float4 access - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float) * ((ne00 + 3) / 4 * 4), nullptr)); - size_t mrow_global[] = { (size_t)((ne01 + rows_per_wg - 1) / rows_per_wg) * 64, (size_t)ne11 * MROW, (size_t)ne12 * ne13 }; - size_t mrow_local[] = { 64, (size_t)MROW, 1 }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, mrow_global, mrow_local, dst); - return; - } + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q break; - case GGML_TYPE_Q1_0: { + } + case GGML_TYPE_Q8_0: { #ifdef GGML_OPENCL_SOA_Q - kernel = backend_ctx->kernel_mul_mv_q1_0_f32_flat; + kernel = backend_ctx->kernel_mul_mv_q8_0_f32_flat; // nth0 - subgroup size // nth1 - number of subgroups per workgroup @@ -24786,8 +33715,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(false && "TODO: Unknown GPU"); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); @@ -24806,8 +33735,11 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #else - kernel = backend_ctx->kernel_mul_mv_q1_0_f32; + kernel = backend_ctx->kernel_mul_mv_q8_0_f32; + // nth0 - subgroup size + // nth1 - number of subgroups per workgroup + // ndst - number of output values per workgroup = output per subgroup * number of subgroups if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; @@ -24842,30 +33774,24 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co #endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_Q4_0: - // This should have been satisfied. - GGML_ASSERT(ne11 == ne1); - GGML_ASSERT(ne01 == ne0); - + case GGML_TYPE_IQ4_NL: { #ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq4_nl_f32_flat; + if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; - - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; ndst = 8; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; - - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; - ndst =8; + ndst = 8; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); @@ -24879,22 +33805,16 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); -#else // GGML_OPENCL_SOA_Q +#else + kernel = backend_ctx->kernel_mul_mv_iq4_nl_f32; + if (backend_ctx->gpu_family == INTEL) { - // Use 1D local size. Each workgroup is a SIMD group. Each SIMD - // group produces N_DST (4 for Q4_0 kernel) values in the result. - // The number of workgroups on dim 0 (the leading dimension) is - // the nearest multiple of 4 that covers ne0 (equals ne01). nth0 = 16; nth1 = 1; - - kernel = backend_ctx->kernel_mul_mat_q4_0_f32; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; - - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_v; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); @@ -24917,8 +33837,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; - case GGML_TYPE_Q4_1: { -#ifdef GGML_OPENCL_SOA_Q + } + case GGML_TYPE_IQ4_XS: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; @@ -24931,25 +33851,58 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(false && "TODO: Unknown GPU"); } - kernel = backend_ctx->kernel_mul_mv_q4_1_f32_flat; +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq4_xs_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_xs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_xs->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq4_xs->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq4_xs->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq4_xs_f32; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); -#else + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_Q2_K: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; @@ -24962,7 +33915,33 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(false && "TODO: Unknown GPU"); } - kernel = backend_ctx->kernel_mul_mv_q4_1_f32; +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q2_K_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q2_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q2_K_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); @@ -24972,49 +33951,85 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_Q5_0: { -#ifdef GGML_OPENCL_SOA_Q + case GGML_TYPE_Q3_K: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; - ndst = 4; + ndst = 2; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; - ndst = 4; + ndst = 2; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } - kernel = backend_ctx->kernel_mul_mv_q5_0_f32_flat; +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q3_K_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q3_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q3_K->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q3_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q3_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q3_K_f32; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); -#else + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_IQ3_XXS: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; @@ -25027,7 +34042,33 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(false && "TODO: Unknown GPU"); } - kernel = backend_ctx->kernel_mul_mv_q5_0_f32; +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq3_xxs_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq3_xxs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq3_xxs->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq3_xxs->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq3_xxs_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); @@ -25037,18 +34078,21 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_Q5_1: { -#ifdef GGML_OPENCL_SOA_Q + case GGML_TYPE_IQ3_S: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; @@ -25061,26 +34105,59 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(false && "TODO: Unknown GPU"); } - kernel = backend_ctx->kernel_mul_mv_q5_1_f32_flat; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq3_s_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq3_s->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq3_s->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq3_s->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq3_s->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_iq3_s->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r3)); #else + kernel = backend_ctx->kernel_mul_mv_iq3_s_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_IQ1_M: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; @@ -25093,7 +34170,33 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(false && "TODO: Unknown GPU"); } - kernel = backend_ctx->kernel_mul_mv_q5_1_f32; +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq1_m_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq1_m->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq1_m->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq1_m->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq1_m_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); @@ -25103,37 +34206,63 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_Q8_0: { -#ifdef GGML_OPENCL_SOA_Q - kernel = backend_ctx->kernel_mul_mv_q8_0_f32_flat; - - // nth0 - subgroup size - // nth1 - number of subgroups per workgroup - // ndst - number of output values per workgroup = output per subgroup * number of subgroups + case GGML_TYPE_IQ1_S: { if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 2; - ndst = nth1*4; + nth0 = 16; + nth1 = 1; + ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; - nth1 = 2; - ndst = nth1*4; + nth1 = 1; + ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq1_s_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq1_s->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq1_s->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq1_s->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq1_s_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); @@ -25151,24 +34280,52 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); -#else - kernel = backend_ctx->kernel_mul_mv_q8_0_f32; - - // nth0 - subgroup size - // nth1 - number of subgroups per workgroup - // ndst - number of output values per workgroup = output per subgroup * number of subgroups +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_IQ2_S: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; - nth1 = 2; - ndst = nth1*4; + nth1 = 1; + ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; - nth1 = 2; - ndst = nth1*4; + nth1 = 1; + ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq2_s_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_s->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_s->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_s->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq2_s->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_iq2_s->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq2_s_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); @@ -25191,40 +34348,70 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co #endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_IQ4_NL: { -#ifdef GGML_OPENCL_SOA_Q - kernel = backend_ctx->kernel_mul_mv_iq4_nl_f32_flat; - + case GGML_TYPE_IQ2_XS: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; - ndst = 8; + ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; - ndst = 8; + ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq2_xs_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_xs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_xs->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_xs->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq2_xs_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); -#else - kernel = backend_ctx->kernel_mul_mv_iq4_nl_f32; - + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_IQ2_XXS: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; @@ -25237,6 +34424,34 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(false && "TODO: Unknown GPU"); } +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq2_xxs_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_xxs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_xxs->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_xxs->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq2_xxs_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); @@ -25245,18 +34460,20 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat; @@ -25563,8 +34780,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || src0t == GGML_TYPE_Q1_0 || + src0t == GGML_TYPE_Q2_0 || src0t == GGML_TYPE_IQ4_NL || - src0t == GGML_TYPE_Q2_K) { + src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ3_S || src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ4_XS) { // Each SIMD group produces N_DST values in the result. Assuming each // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size @@ -25580,8 +34798,6 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else if (src0t == GGML_TYPE_Q3_K) { - GGML_ASSERT(false && "not implemented"); } else if (src0t == GGML_TYPE_Q5_K) { size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; @@ -25631,8 +34847,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { int64_t ny = (ne11 + nrows - 1)/nrows; + int64_t nx = kernel == backend_ctx->kernel_mul_mv_f32_f32_r2 ? CEIL_DIV(ne01, 2) : ne01; - size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; + size_t global_work_size[] = {(size_t)nx*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); @@ -25654,10 +34871,6 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, const int max_post_router_tile = (ne20 * ne21 / n_tile_size) + ne02; cl_buffer_region region; - region.origin = offset; - region.size = nb21 * ne21; - cl_mem original_router_buf = clCreateSubBuffer(extra->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); backend_ctx->prealloc_post_router.allocate(backend_ctx->context, sizeof(int) * max_post_router_tile * n_tile_size); region.origin = 0; @@ -25671,7 +34884,11 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, cl_mem emap_buf = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); - backend_ctx->prealloc_hist.allocate(backend_ctx->context, sizeof(int) * ne02); + if (sizeof(int) * ne02 > backend_ctx->prealloc_hist.size) { + backend_ctx->prealloc_hist.allocate(backend_ctx->context, sizeof(int) * ne02); + const int zero = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_hist.buffer, &zero, sizeof(zero), 0, sizeof(int) * ne02, 0, nullptr, nullptr)); + } region.origin = 0; region.size = sizeof(int) * ne02; cl_mem hist_buf = clCreateSubBuffer(backend_ctx->prealloc_hist.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); @@ -25697,11 +34914,12 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, // Histogram cl_kernel kernel = backend_ctx->kernel_moe_histogram; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &hist_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset)); size_t histogram_global_size[] = {(size_t)(((ne21 + 63) / 64) * 64), static_cast<size_t>(ne20), 1}; size_t histogram_local_size[] = {64, 1, 1}; @@ -25741,13 +34959,14 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, if (stable_scatter) { kernel = backend_ctx->kernel_moe_scatter_stable; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset)); // one workgroup (one wave) per expert; each ranks its own tokens size_t scatter_global_size[] = {64, (size_t)ne02}; @@ -25755,7 +34974,7 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, backend_ctx->enqueue_ndrange_kernel(kernel, 2, scatter_global_size, scatter_local_size, src); } else { kernel = backend_ctx->kernel_moe_scatter; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); @@ -25763,6 +34982,7 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offset)); backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src); } @@ -25784,7 +35004,6 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, fflush(stderr); } - CL_CHECK(clReleaseMemObject(original_router_buf)); CL_CHECK(clReleaseMemObject(hist_buf)); CL_CHECK(clReleaseMemObject(tile_offset_buf)); CL_CHECK(clReleaseMemObject(total_tiles_buf)); @@ -25793,7 +35012,125 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, CL_CHECK(clReleaseMemObject(emap_buf)); } +// Use the per-token MoE GEMV instead of the tiled expert GEMM for small batches. +// The GEMV costs one weight pass per token, the GEMM pads every expert to a +// 32-token tile, so the GEMV wins while tokens rarely share an expert. +// Default on for X2E up to 4 tokens while 2 * tokens * used < experts. +// GGML_OPENCL_MOE_GEMV_MAX_TOK overrides with a plain token limit (1 = decode only). +static bool use_adreno_moe_gemv_small_batch(const ggml_backend_opencl_context * backend_ctx, int n_tokens, int n_used, int n_expert) { + if (n_tokens <= 1) { + return true; + } + static const char * env = getenv("GGML_OPENCL_MOE_GEMV_MAX_TOK"); + if (env) { + return n_tokens <= std::max(1, atoi(env)); + } + return backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_tokens <= 4 && + 2 * (int64_t) n_tokens * n_used < (int64_t) n_expert; +} + +static void ggml_cl_mul_mat_id_float(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + cl_kernel kernel = backend_ctx->kernel_mul_mv_id_f32_f32; + const ggml_tensor * src2 = dst->src[2]; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offset1 = extra1->offset + src1->view_offs; + const cl_ulong offset2 = extra2->offset + src2->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ncols = src0->ne[0]; + const int ne11 = src1->ne[1]; + const int nrows = dst->ne[0]; + // BF16 tensors are stored as F16 on the device. Activations remain F32. + const int src0_f16 = src0->type == GGML_TYPE_F32 ? 0 : 1; + + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb20 = src2->nb[0]; + const cl_ulong nb21 = src2->nb[1]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + + // The router uses the IDs row stride as its expert count. + const size_t router_experts = src2->nb[1]/sizeof(int32_t); + if (src1->ne[2] >= 16 && ncols >= 64 && nrows >= 32 && ggml_is_contiguous(dst) && + src2->nb[0] == sizeof(int32_t) && src2->nb[1] % sizeof(int32_t) == 0 && + router_experts >= (size_t) src0->ne[2] && router_experts <= 1024 && + dst->ne[1]*dst->ne[2] >= (int64_t) router_experts*4 && + dst->ne[1]*dst->ne[2] >= 16384/nrows && + dst->ne[1]*dst->ne[2] <= INT_MAX - 32*(int64_t) router_experts) { + const int nused = dst->ne[1]; + const size_t max_tiles = (size_t) nused*dst->ne[2]/32 + router_experts; + moe_router_reoerder(backend, src2, nused); + backend_ctx->toggle_reorder = true; + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, extrad->data_device, &zero, sizeof(zero), offsetd, ggml_nbytes(dst), 0, nullptr, nullptr)); + kernel = backend_ctx->kernel_mul_mm_id_f32_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &backend_ctx->prealloc_post_router.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &backend_ctx->prealloc_emap.buffer)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &backend_ctx->prealloc_total_tiles.buffer)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ncols)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &nrows)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &nused)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &src0_f16)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb2)); + size_t global_work_size[] = {(size_t) ((nrows + 63)/64)*128, max_tiles}; + size_t local_work_size[] = {128, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + return; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ncols)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &src0_f16)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb20)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &nrows)); + + size_t global_work_size[] = {(size_t) ((dst->ne[0] + 3)/4)*64, (size_t) dst->ne[1], (size_t) dst->ne[2]}; + size_t local_work_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) { + ggml_cl_mul_mat_id_float(backend, src0, src1, dst); + return; + } + GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); @@ -25881,6 +35218,25 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, GGML_ASSERT(ne00 == ne10); + // the matmul kernels only write the rows an expert owns, so zero the skipped rows here + { + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + cl_kernel kernel_zero = backend_ctx->kernel_moe_zero_dst; + CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel_zero, 6, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel_zero, 7, sizeof(int), &ne0)); + + size_t zero_local[3] = {1, 1, 64}; + size_t zero_global[3] = {(size_t) ne20, (size_t) ne21, 64}; + backend_ctx->enqueue_ndrange_kernel(kernel_zero, 3, zero_global, zero_local, dst); + } + int sgs = 32; // subgroup size int nsg = 1; // number of subgroups int nrows = 1; // number of row in src1 @@ -25903,7 +35259,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_0_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -25911,14 +35267,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast<size_t>(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast<size_t>(ne20); + global_size[2] = static_cast<size_t>(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -25935,6 +35291,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); @@ -25944,6 +35301,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -26212,7 +35571,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_1_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -26220,14 +35579,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast<size_t>(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast<size_t>(ne20); + global_size[2] = static_cast<size_t>(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -26244,6 +35603,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->m)); @@ -26254,6 +35614,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -26398,7 +35760,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_0_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -26406,14 +35768,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast<size_t>(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast<size_t>(ne20); + global_size[2] = static_cast<size_t>(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -26430,6 +35792,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qs)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->d)); @@ -26440,6 +35803,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -26651,7 +36016,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_1_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -26659,14 +36024,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast<size_t>(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast<size_t>(ne20); + global_size[2] = static_cast<size_t>(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -26683,6 +36048,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qs)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->d)); @@ -26694,6 +36060,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -27083,7 +36451,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns; // Weight-as-texture MoE decode GEMV @@ -27103,14 +36471,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast<size_t>(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast<size_t>(ne20); + global_size[2] = static_cast<size_t>(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -27127,6 +36495,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_q4_K->q_img : &extra0_q4_K->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm)); @@ -27138,6 +36507,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -27171,8 +36542,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; - // bin kernel takes precedence - use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr; + // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel + if (backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin == nullptr) { + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr; + } cl_buffer_region region; region.origin = 0; @@ -27274,6 +36647,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // dp4a GEMM cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a; + if (backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin) { + dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin; + } + int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->q_img)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->d)); @@ -27347,7 +36724,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_k_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -27355,14 +36732,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast<size_t>(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast<size_t>(ne20); + global_size[2] = static_cast<size_t>(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -27379,6 +36756,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->d)); @@ -27391,6 +36769,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -27604,7 +36984,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q6_k_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -27612,14 +36992,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast<size_t>(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast<size_t>(ne20); + global_size[2] = static_cast<size_t>(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -27636,6 +37016,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->s)); @@ -27647,6 +37028,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -27681,8 +37064,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; - // bin kernel takes precedence - use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr; + // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel + if (backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin == nullptr) { + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr; + } cl_buffer_region region; region.origin = 0; @@ -27784,6 +37169,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); cl_kernel dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a; + if (backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin) { + dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin; + } + int qi = 0; CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->ql_img)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->qh)); @@ -27936,11 +37325,19 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A"); bool use_moe_dp4a = mxfp4_moe_dp4a_env ? (atoi(mxfp4_moe_dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel - if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) { + // ldck dp4a bin kernel (X1) has the same priority as normal bin kernel (X2/A8x), they are not expected to coexist. + const bool use_moe_dp4a_ldck = backend_ctx->has_integer_dot && + (!mxfp4_moe_dp4a_env || atoi(mxfp4_moe_dp4a_env) != 0) && + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr && + backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr && + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin != nullptr; + if (use_moe_dp4a_ldck) { + use_moe_dp4a = true; + } else if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) { use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr; } @@ -28029,6 +37426,12 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + if (use_moe_dp4a_ldck) { + backend_ctx->prealloc_moe_qa_trans.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da_trans.allocate(backend_ctx->context, n_blocks * sizeof(cl_float)); + // matches max_constant_size of src1_a_cm in the ldck kernel + backend_ctx->prealloc_moe_qda_on_chip.allocate(backend_ctx->context, 2304); + } // fused reorder + q8_1 quant straight from the original // activations (no intermediate f32 reorder buffer). mxfp4 has no @@ -28053,13 +37456,33 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a; if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) { dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin; + } else if (use_moe_dp4a_ldck) { + dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin; + + // per-tile transpose of qa/da for the ldck kernel + cl_kernel tk = backend_ctx->kernel_transpose_activation_q8; + CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer)); + CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer)); + CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(tk, 5, sizeof(cl_uint), &ne00)); + size_t tk_local[3] = { 32, 8, 1 }; + size_t tk_global[3] = { 32, (size_t)(ne00 / 4), (size_t)max_post_router_tile }; + backend_ctx->enqueue_ndrange_kernel(tk, 3, tk_global, tk_local, dst); } int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e)); - CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + if (use_moe_dp4a_ldck) { + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qda_on_chip.buffer)); + } else { + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + } CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); @@ -28329,11 +37752,15 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const switch (src0t) { case GGML_TYPE_F32: switch (src1t) { + case GGML_TYPE_I32: + kernel = backend_ctx->kernel_cpy_f32_i32; + break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_cpy_f32_f16; break; case GGML_TYPE_F32: - kernel = ne00 < 32 ? backend_ctx->kernel_cpy_f32_f32_pack + kernel = (ne00 < 32 && backend_ctx->kernel_cpy_f32_f32_pack) + ? backend_ctx->kernel_cpy_f32_f32_pack : backend_ctx->kernel_cpy_f32_f32; break; default: @@ -28354,6 +37781,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const break; case GGML_TYPE_I32: switch (src1t) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_cpy_i32_f32; + break; case GGML_TYPE_I32: kernel = backend_ctx->kernel_cpy_i32_i32; break; @@ -28470,93 +37900,31 @@ static void ggml_cl_set(ggml_backend_t backend, const ggml_tensor * src0, const CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &pnb1)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &pnb2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &pnb3)); - - int max_local_size = backend_ctx->get_kernel_workgroup_size(kernel); - - const int nth = MIN(max_local_size, ne00); - - size_t global_work_size[] = {(size_t)ne11*nth, (size_t)ne12, (size_t)ne13}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} - -static void ggml_cl_diag_mask_inf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - - UNUSED(src1); - - int n_past = ((int32_t *)(dst->op_params))[0]; - - const int ne00 = src0 ? src0->ne[0] : 0; - const int ne01 = src0 ? src0->ne[1] : 0; - const int ne02 = src0 ? src0->ne[2] : 0; - - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; - - cl_kernel kernel; - - if (ne00%8 == 0) { - kernel = backend_ctx->kernel_diag_mask_inf_8; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_past)); - - size_t global_work_size[] = {(size_t)ne00*ne01*ne02/8, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - kernel = backend_ctx->kernel_diag_mask_inf; + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &pnb1)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &pnb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &pnb3)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_past)); + int max_local_size = backend_ctx->get_kernel_workgroup_size(kernel); - size_t global_work_size[] = {(size_t)ne00, (size_t)ne01, (size_t)ne02}; - size_t local_work_size[] = {64, 1, 1}; + const int nth = MIN(max_local_size, ne00); - size_t * local_work_size_ptr = local_work_size; - if (ne00 % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } + size_t global_work_size[] = {(size_t)ne11*nth, (size_t)ne12, (size_t)ne13}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_diag(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -28730,6 +38098,102 @@ static void ggml_cl_soft_max(ggml_backend_t backend, const ggml_tensor * src0, c backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +// Softmax for the decomposed flash attention: like ggml_cl_soft_max, but dst keeps +// exp(x - max) and the row sums go to `sums`, one float per (row, head). src2 is the +// optional sinks tensor. +static void ggml_cl_fa_decompose_soft_max(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, cl_mem sums) { + GGML_ASSERT(src0 && src0->extra); + GGML_ASSERT(src1 && src1->extra && src1->type == GGML_TYPE_F16); + GGML_ASSERT(dst && dst->extra); + GGML_ASSERT(sums != nullptr); + GGML_ASSERT(src0->ne[0] % 4 == 0); + + if (src2) { + GGML_ASSERT(src2->extra); + } + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = src2 ? (ggml_tensor_extra_cl *)src2->extra : nullptr; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2 ? extra2->offset + src2->view_offs : offset0; + cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_ulong offsets = 0; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; + + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; + + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + float scale, max_bias; + memcpy(&scale, dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, dst->op_params + 1, sizeof(float)); + + const int n_head = src0->ne[2]; + const int n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + // one wave per row, same as ggml_cl_soft_max on Adreno + const int nth = 64; + + cl_kernel kernel = backend_ctx->kernel_fa_decompose_soft_max_4_f16; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), extra2 ? &extra2->data_device : &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &sums)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsets)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &n_head_log2)); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -29070,6 +38534,400 @@ static void ggml_cl_im2col(ggml_backend_t backend, const ggml_tensor * src0, con backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +static void ggml_cl_im2col_3d(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int32_t s0 = ((const int32_t*)(dst->op_params))[0]; + const int32_t s1 = ((const int32_t*)(dst->op_params))[1]; + const int32_t s2 = ((const int32_t*)(dst->op_params))[2]; + const int32_t p0 = ((const int32_t*)(dst->op_params))[3]; + const int32_t p1 = ((const int32_t*)(dst->op_params))[4]; + const int32_t p2 = ((const int32_t*)(dst->op_params))[5]; + const int32_t d0 = ((const int32_t*)(dst->op_params))[6]; + const int32_t d1 = ((const int32_t*)(dst->op_params))[7]; + const int32_t d2 = ((const int32_t*)(dst->op_params))[8]; + const cl_long IC = ((const int32_t*)(dst->op_params))[9]; + + const cl_long N = src1->ne[3] / IC; + const cl_long ID = src1->ne[2]; + const cl_long IH = src1->ne[1]; + const cl_long IW = src1->ne[0]; + + const cl_long KD = src0->ne[2]; + const cl_long KH = src0->ne[1]; + const cl_long KW = src0->ne[0]; + + const cl_long OD = dst->ne[3] / N; + const cl_long OH = dst->ne[2]; + const cl_long OW = dst->ne[1]; + + const size_t es = ggml_element_size(src1); + const cl_long stride_x = src1->nb[0] / es; + const cl_long stride_y = src1->nb[1] / es; + const cl_long stride_z = src1->nb[2] / es; + const cl_long stride_q = src1->nb[3] / es; + + const cl_long IC_KD_KH_KW = IC * KD * KH * KW; + + cl_kernel kernel; + + if (dst->type == GGML_TYPE_F16) { + kernel = backend_ctx->kernel_im2col_3d_f16; + } else { + kernel = backend_ctx->kernel_im2col_3d_f32; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_long), &IC)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_long), &ID)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_long), &IH)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_long), &IW)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_long), &KD)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_long), &KH)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_long), &KW)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_long), &OD)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_long), &OH)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_long), &OW)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_long), &N)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_long), &stride_x)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_long), &stride_y)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_long), &stride_z)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_long), &stride_q)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &s0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &s1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &s2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &p0)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &p1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &p2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &d0)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &d1)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(int), &d2)); + + const size_t local_col = 64; + const size_t num_blocks = (IC_KD_KH_KW + local_col - 1) / local_col; + size_t global_work_size[] = { num_blocks * local_col, (size_t)OW, (size_t)(N * OD * OH) }; + size_t local_work_size[] = { local_col, 1, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_argmax(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_UNUSED(src1); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_I32); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + const int ne00 = src0->ne[0]; + const int nrows = ggml_nrows(src0); + constexpr int nth = 64; + + cl_kernel kernel = backend_ctx->kernel_argmax_f32_i32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &nrows)); + + size_t global_work_size[] = { (size_t) nth, (size_t) CEIL_DIV(nrows, 2), 1 }; + size_t local_work_size[] = { (size_t) nth, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_lightning_indexer(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + cl_kernel kernel = backend_ctx->kernel_lightning_indexer; + const ggml_tensor * src2 = dst->src[2]; + const ggml_tensor * src3 = dst->src[3]; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) src2->extra; + ggml_tensor_extra_cl * extra3 = (ggml_tensor_extra_cl *) src3->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offset1 = extra1->offset + src1->view_offs; + const cl_ulong offset2 = extra2->offset + src2->view_offs; + const cl_ulong offset3 = extra3->offset + src3->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int head_size = src0->ne[0]; + const int n_heads = src0->ne[1]; + const int n_tokens = src0->ne[2]; + const int n_keys = src1->ne[2]; + const int n_masks = src3->ne[3]; + // Keep the local-memory reduction for small grids or few heads. + const int use_shuffle = head_size == 128 && backend_ctx->has_khr_subgroup_shuffle && n_heads >= 32 && ggml_nelements(dst) >= 1024; + // BF16 tensors are stored as F16 on the device. + const int type_k = src1->type == GGML_TYPE_F32 ? 0 : 1; + + const cl_ulong nbq1 = src0->nb[1]; + const cl_ulong nbq2 = src0->nb[2]; + const cl_ulong nbq3 = src0->nb[3]; + const cl_ulong nbk2 = src1->nb[2]; + const cl_ulong nbk3 = src1->nb[3]; + const cl_ulong nbw1 = src2->nb[1]; + const cl_ulong nbw3 = src2->nb[3]; + const cl_ulong nbm1 = src3->nb[1]; + const cl_ulong nbm3 = src3->nb[3]; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra3->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &head_size)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &n_heads)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_tokens)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &n_keys)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &n_masks)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &type_k)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nbq1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nbq2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nbq3)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nbk2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nbk3)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nbw1)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nbw3)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &nbm1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &nbm3)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &use_shuffle)); + + const int keys_per_group = use_shuffle ? 16 : 4; + size_t global_work_size[] = {(size_t) ((src1->ne[2] + keys_per_group - 1)/keys_per_group)*64, (size_t) src0->ne[2], (size_t) src0->ne[3]}; + size_t local_work_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_top_k(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_UNUSED(src1); + + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + + const int k = dst->ne[0]; + cl_kernel kernel = backend_ctx->kernel_top_k_f32; + const int max_wg = backend_ctx->get_kernel_workgroup_size(kernel); + + int nth = 1; + const int target = MIN(512, MAX(32, ne00/4)); + while (2*nth <= MIN(target, max_wg)) { + nth *= 2; + } + + const size_t n_rows = ggml_nrows(src0); + const bool tiled = ne00 > 32768 && k > 0 && k <= 256 && n_rows <= 4; + const int tile_width = tiled ? 4096 : ne00; + const int n_tiles = tiled ? 1 + (ne00 - 1)/tile_width : 1; + const int n_candidates = (n_tiles - 1)*k + MIN(k, ne00 - (n_tiles - 1)*tile_width); + cl_mem candidates = nullptr; + if (tiled) { + GGML_ASSERT(n_rows <= SIZE_MAX/sizeof(cl_uint2)/(size_t) n_candidates); + backend_ctx->prealloc_top_k.allocate(backend_ctx->context, n_rows*(size_t) n_candidates*sizeof(cl_uint2)); + candidates = backend_ctx->prealloc_top_k.buffer; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 12, MAX(259, 2*nth)*sizeof(cl_uint), nullptr)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_mem), &candidates)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &tile_width)); + + size_t global_work_size[] = {(size_t) nth*n_tiles, n_rows, 1}; + size_t local_work_size[] = {(size_t) nth, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + + if (tiled) { + // The global top-k is contained in the top-k of each tile. + kernel = backend_ctx->kernel_top_k_merge_f32; + const int max_wg_merge = backend_ctx->get_kernel_workgroup_size(kernel); + + int nth_merge = 1; + const int target_merge = MIN(512, MAX(32, n_candidates/4)); + while (2*nth_merge <= MIN(target_merge, max_wg_merge)) { + nth_merge *= 2; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &candidates)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &n_candidates)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 5, MAX(259, 2*nth_merge)*sizeof(cl_uint), nullptr)); + + global_work_size[0] = nth_merge; + local_work_size[0] = nth_merge; + + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + } +} + +static void ggml_cl_top_k_mask(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const ggml_tensor * score = dst->src[2]; + const ggml_tensor * blocks = dst->src[3]; + const ggml_tensor * extra = dst->src[4]; + const int type = dst->type == GGML_TYPE_F16 ? 0 : 1; + cl_kernel kernel = backend_ctx->kernel_top_k_mask[type]; + + const ggml_tensor_extra_cl * extram = (const ggml_tensor_extra_cl *) src0->extra; + const ggml_tensor_extra_cl * extrak = (const ggml_tensor_extra_cl *) src1->extra; + const ggml_tensor_extra_cl * extras = score ? (const ggml_tensor_extra_cl *) score->extra : nullptr; + const ggml_tensor_extra_cl * extrab = blocks ? (const ggml_tensor_extra_cl *) blocks->extra : nullptr; + const ggml_tensor_extra_cl * extrae = extra ? (const ggml_tensor_extra_cl *) extra->extra : nullptr; + const ggml_tensor_extra_cl * extrad = (const ggml_tensor_extra_cl *) dst->extra; + + const cl_mem buffer_s = extras ? extras->data_device : nullptr; + const cl_mem buffer_b = extrab ? extrab->data_device : nullptr; + const cl_mem buffer_e = extrae ? extrae->data_device : nullptr; + + const cl_ulong offsetm = extram->offset + src0->view_offs; + const cl_ulong offsetk = extrak->offset + src1->view_offs; + const cl_ulong offsets = extras ? extras->offset + score->view_offs : 0; + const cl_ulong offsetb = extrab ? extrab->offset + blocks->view_offs : 0; + const cl_ulong offsete = extrae ? extrae->offset + extra->view_offs : 0; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + // single launch: fill == true, two launches: fill = false + // X2 benefits from a parallel fill for long single rows; mobile Adreno favors one launch. + const cl_int fill = !(backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && + ggml_nrows(dst) == 1 && dst->ne[0] >= 65536 && ggml_is_contiguous(dst)); + + if (!fill) { + GGML_ASSERT(ggml_is_contiguous(dst)); + + cl_kernel kernel_fill = backend_ctx->kernel_top_k_mask_fill[type]; + + const cl_ulong n = ggml_nelements(dst); + const size_t nv = 16/ggml_type_size(dst->type); + const size_t nth = MIN((size_t) 256, backend_ctx->get_kernel_workgroup_size(kernel_fill)); + + CL_CHECK(clSetKernelArg(kernel_fill, 0, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_fill, 1, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_fill, 2, sizeof(cl_ulong), &n)); + + size_t global[] = {((n + nv - 1)/nv + nth - 1)/nth*nth}; + size_t local[] = {nth}; + backend_ctx->enqueue_ndrange_kernel(kernel_fill, 1, global, local, dst); + } + + GGML_TENSOR_LOCALS(cl_ulong, nem, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nbm, src0, nb); + GGML_TENSOR_LOCALS(cl_ulong, nek, src1, ne); + GGML_TENSOR_LOCALS(cl_ulong, nbk, src1, nb); + GGML_TENSOR_LOCALS(cl_ulong, nes, score, ne); + GGML_TENSOR_LOCALS(cl_ulong, nbs, score, nb); + GGML_TENSOR_LOCALS(cl_ulong, neb, blocks, ne); + GGML_TENSOR_LOCALS(cl_ulong, nbb, blocks, nb); + GGML_TENSOR_LOCALS(cl_ulong, nee, extra, ne); + GGML_TENSOR_LOCALS(cl_ulong, nbe, extra, nb); + GGML_TENSOR_LOCALS(cl_ulong, ned, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nbd, dst, nb); + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extram->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrak->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetk)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buffer_s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsets)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &buffer_b)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetb)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &buffer_e)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsete)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nem0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nem1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nem2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nem3)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nbm1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nbm2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nbm3)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nek0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nbk0)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nbk1)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nbk2)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &nbk3)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &nes0)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(cl_ulong), &nbs0)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nbs1)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nbs2)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nbs3)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &neb0)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &neb1)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nbb0)); + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &nbb1)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &nee0)); + CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &nbe0)); + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &nbe1)); + CL_CHECK(clSetKernelArg(kernel, 36, sizeof(cl_ulong), &nbe2)); + CL_CHECK(clSetKernelArg(kernel, 37, sizeof(cl_ulong), &nbe3)); + CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_ulong), &ned1)); + CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &ned2)); + CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_ulong), &nbd1)); + CL_CHECK(clSetKernelArg(kernel, 41, sizeof(cl_ulong), &nbd2)); + CL_CHECK(clSetKernelArg(kernel, 42, sizeof(cl_ulong), &nbd3)); + CL_CHECK(clSetKernelArg(kernel, 43, sizeof(cl_int), &fill)); + + const size_t nth = MIN((size_t) 512, backend_ctx->get_kernel_workgroup_size(kernel)); + size_t global[] = {nth, (size_t) ggml_nrows(dst)}; + size_t local[] = {nth, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global, local, dst); +} + static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -29123,6 +38981,47 @@ static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, co #endif // GGML_OPENCL_USE_ADRENO_KERNELS } +static void ggml_cl_dsv4_hc(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const bool pre = dst->op == GGML_OP_DSV4_HC_PRE; + const bool vec = !pre && src0->ne[0] % 4 == 0 && src0->nb[0] == sizeof(float) && src1->nb[0] == sizeof(float); + cl_kernel kernel = pre ? backend_ctx->kernel_dsv4_hc_pre : backend_ctx->kernel_dsv4_hc_post[vec]; + + cl_uint arg = 0; + for (int i = 0; i < (pre ? 2 : 4); ++i) { + const ggml_tensor * src = dst->src[i]; + auto * extra = src ? (ggml_tensor_extra_cl *) src->extra : nullptr; + const cl_mem buffer = extra ? extra->data_device : nullptr; + const cl_ulong offset = src ? extra->offset + src->view_offs : 0; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &buffer)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset)); + for (int j = 0; j < 3; ++j) { + const cl_ulong stride = src ? src->nb[j] : 0; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &stride)); + } + } + + auto * extra = (ggml_tensor_extra_cl *) dst->extra; + const cl_ulong offset = extra->offset + dst->view_offs; + const cl_long n_embd = src0->ne[0]; + const cl_long n_hc = pre ? src0->ne[1] : src1->ne[1]; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_embd)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_hc)); + if (pre) { + const float scale = ggml_get_op_params_f32(dst, 0); + const cl_int gated = ggml_get_op_params_i32(dst, 1); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_int), &gated)); + } + + const size_t n_work = (size_t) n_embd / (vec ? 4 : 1); + size_t local_work_size[] = { pre ? backend_ctx->local_dsv4_hc_pre : backend_ctx->local_dsv4_hc_post[vec], 1, 1 }; + size_t global_work_size[] = { CEIL_DIV(n_work, local_work_size[0]) * local_work_size[0], (size_t) dst->ne[1], (size_t) dst->ne[2] }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -29820,6 +39719,12 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_fill; break; + case GGML_OP_ARANGE: + if (!any_on_device) { + return false; + } + func = ggml_cl_arange; + break; case GGML_OP_CLAMP: if (!any_on_device) { return false; @@ -29874,6 +39779,12 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_conv_2d; break; + case GGML_OP_CONV_2D_DW: + if (!any_on_device) { + return false; + } + func = ggml_cl_conv2_dw; + break; case GGML_OP_SSM_CONV: if (!any_on_device) { return false; @@ -29941,12 +39852,6 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_diag; break; - case GGML_OP_DIAG_MASK_INF: - if (!any_on_device) { - return false; - } - func = ggml_cl_diag_mask_inf; - break; case GGML_OP_SOFT_MAX: if (!any_on_device) { return false; @@ -29971,6 +39876,27 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_im2col; break; + case GGML_OP_IM2COL_3D: + if (!any_on_device) { + return false; + } + func = ggml_cl_im2col_3d; + break; + case GGML_OP_ARGMAX: + if (!any_on_device) { + return false; + } + func = ggml_cl_argmax; + break; + case GGML_OP_LIGHTNING_INDEXER: + func = ggml_cl_lightning_indexer; + break; + case GGML_OP_TOP_K: + func = ggml_cl_top_k; + break; + case GGML_OP_TOP_K_MASK: + func = ggml_cl_top_k_mask; + break; case GGML_OP_ARGSORT: if (!any_on_device) { return false; @@ -29983,6 +39909,10 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_sum_rows; break; + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + func = ggml_cl_dsv4_hc; + break; case GGML_OP_CUMSUM: if (!any_on_device) { return false; diff --git a/ggml/src/ggml-opencl/kernels/add_id.cl b/ggml/src/ggml-opencl/kernels/add_id.cl index e9c6d55e6..e9aa5a150 100644 --- a/ggml/src/ggml-opencl/kernels/add_id.cl +++ b/ggml/src/ggml-opencl/kernels/add_id.cl @@ -34,6 +34,14 @@ kernel void kernel_add_id( global float * dst_row = (global float *)((global char *)dst + i1*nb1 + i2*nb2); global float * src0_row = (global float *)((global char *)src0 + i1*nb01 + i2*nb02); + + if (i11 == -1) { // skipped slot, add nothing + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { + dst_row[i0] = src0_row[i0]; + } + return; + } + global float * src1_row = (global float *)((global char *)src1 + i11*nb11); for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { diff --git a/ggml/src/ggml-opencl/kernels/arange.cl b/ggml/src/ggml-opencl/kernels/arange.cl new file mode 100644 index 000000000..53dc383e6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/arange.cl @@ -0,0 +1,14 @@ +__kernel void kernel_arange_f32( + __global float * dst, + ulong offsetd, + int n, + float start, + float step +) { + dst = (global float*)((global char*)dst + offsetd); + + const int i = get_global_id(0); + if (i < n) { + dst[i] = start + step * i; + } +} diff --git a/ggml/src/ggml-opencl/kernels/argmax.cl b/ggml/src/ggml-opencl/kernels/argmax.cl new file mode 100644 index 000000000..24ed0e68b --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/argmax.cl @@ -0,0 +1,55 @@ +#define ARGMAX_LANES_PER_ROW 32 +#define ARGMAX_ROWS_PER_GROUP 2 +#define ARGMAX_BLOCK_SIZE (ARGMAX_LANES_PER_ROW * ARGMAX_ROWS_PER_GROUP) + +kernel void kernel_argmax_f32_i32( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + int ne00, + int nrows) { + const int local_id = get_local_id(0); + const int col = local_id % ARGMAX_LANES_PER_ROW; + const int row = get_group_id(1) * ARGMAX_ROWS_PER_GROUP + local_id / ARGMAX_LANES_PER_ROW; + + float max_value = -INFINITY; + int max_index = 0; + + if (row < nrows) { + global const float * src_row = (global const float *) (src0 + offset0) + row * ne00; + for (int i = col; i < ne00; i += ARGMAX_LANES_PER_ROW) { + const float value = src_row[i]; + if (value > max_value || (value == max_value && i > max_index)) { + max_value = value; + max_index = i; + } + } + } + + local float max_values[ARGMAX_BLOCK_SIZE]; + local int max_indices[ARGMAX_BLOCK_SIZE]; + + max_values[local_id] = max_value; + max_indices[local_id] = max_index; + + barrier(CLK_LOCAL_MEM_FENCE); + + const int row_offset = local_id / ARGMAX_LANES_PER_ROW * ARGMAX_LANES_PER_ROW; + for (int stride = ARGMAX_LANES_PER_ROW / 2; stride > 0; stride /= 2) { + if (col < stride) { + const float value = max_values[row_offset + col + stride]; + const int index = max_indices[row_offset + col + stride]; + if (value > max_values[row_offset + col] || (value == max_values[row_offset + col] && index > max_indices[row_offset + col])) { + max_values[row_offset + col] = value; + max_indices[row_offset + col] = index; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (col == 0 && row < nrows) { + global int * dst_row = (global int *) (dst + offsetd) + row; + dst_row[0] = max_indices[row_offset]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/clamp.cl b/ggml/src/ggml-opencl/kernels/clamp.cl index ae6032444..366a06836 100644 --- a/ggml/src/ggml-opencl/kernels/clamp.cl +++ b/ggml/src/ggml-opencl/kernels/clamp.cl @@ -18,3 +18,33 @@ kernel void kernel_clamp( min : (src0[get_global_id(0)] > max ? max : src0[get_global_id(0)]); } + +kernel void kernel_clamp_nc( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + float min, + float max, + int ne00, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + dst = dst + offsetd; + + const int i3 = get_group_id(2); + const int i2 = get_group_id(1); + const int i1 = get_group_id(0); + + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { + const float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = x < min ? min : (x > max ? max : x); + } +} diff --git a/ggml/src/ggml-opencl/kernels/conv2_dw.cl b/ggml/src/ggml-opencl/kernels/conv2_dw.cl new file mode 100644 index 000000000..2a35b9c96 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/conv2_dw.cl @@ -0,0 +1,139 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +kernel void kernel_conv2d_dw_f32_f32( + global const float * weights, + long weights_offset, + global const float * input, + long input_offset, + global float * output, + long output_offset, + long C, + long N, + long KW, + long KH, + long W, + long H, + long OW, + long OH, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, + long nb01, + long nb03, + long nb11, + long nb12, + long nb13, + long nb1, + long nb2, + long nb3 +) { + weights = (global const float *) ((global const char *) weights + weights_offset); + input = (global const float *) ((global const char *) input + input_offset); + output = (global float *) ((global char *) output + output_offset); + + const long gid = get_global_id(0); + const long OHW = OH * OW; + const long COHW = C * OHW; + const long total = N * COHW; + if (gid >= total) { + return; + } + + const long n = gid / COHW; + const long c_rem = gid - n * COHW; + const long c = c_rem / OHW; + const long oh = (c_rem - c * OHW) / OW; + const long ow = c_rem - c * OHW - oh * OW; + + float sum = 0.0f; + for (long kh = 0; kh < KH; ++kh) { + const long ih = oh * s1 + kh * d1 - p1; + if (ih < 0 || ih >= H) { + continue; + } + for (long kw = 0; kw < KW; ++kw) { + const long iw = ow * s0 + kw * d0 - p0; + if (iw < 0 || iw >= W) { + continue; + } + const long weight_idx = kw + kh * nb01 + c * nb03; + const long input_idx = iw + ih * nb11 + c * nb12 + n * nb13; + sum += weights[weight_idx] * input[input_idx]; + } + } + + const long output_idx = ow + oh * nb1 + c * nb2 + n * nb3; + output[output_idx] = sum; +} + +kernel void kernel_conv2d_dwf16_f32( + global const half * weights, + long weights_offset, + global const float * input, + long input_offset, + global float * output, + long output_offset, + long C, + long N, + long KW, + long KH, + long W, + long H, + long OW, + long OH, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, + long nb01, + long nb03, + long nb11, + long nb12, + long nb13, + long nb1, + long nb2, + long nb3 +) { + weights = (global const half *) ((global const char *) weights + weights_offset); + input = (global const float *) ((global const char *) input + input_offset); + output = (global float *) ((global char *) output + output_offset); + + const long gid = get_global_id(0); + const long OHW = OH * OW; + const long COHW = C * OHW; + const long total = N * COHW; + if (gid >= total) { + return; + } + + const long n = gid / COHW; + const long c_rem = gid - n * COHW; + const long c = c_rem / OHW; + const long oh = (c_rem - c * OHW) / OW; + const long ow = c_rem - c * OHW - oh * OW; + + float sum = 0.0f; + for (long kh = 0; kh < KH; ++kh) { + const long ih = oh * s1 + kh * d1 - p1; + if (ih < 0 || ih >= H) { + continue; + } + for (long kw = 0; kw < KW; ++kw) { + const long iw = ow * s0 + kw * d0 - p0; + if (iw < 0 || iw >= W) { + continue; + } + const long weight_idx = kw + kh * nb01 + c * nb03; + const long input_idx = iw + ih * nb11 + c * nb12 + n * nb13; + sum += (float) weights[weight_idx] * input[input_idx]; + } + } + + const long output_idx = ow + oh * nb1 + c * nb2 + n * nb3; + output[output_idx] = sum; +} diff --git a/ggml/src/ggml-opencl/kernels/cpy.cl b/ggml/src/ggml-opencl/kernels/cpy.cl index e875bfaf7..18400d37f 100644 --- a/ggml/src/ggml-opencl/kernels/cpy.cl +++ b/ggml/src/ggml-opencl/kernels/cpy.cl @@ -183,6 +183,7 @@ kernel void kernel_cpy_f32_f32( } } +#ifndef GGML_CL_NO_CPY_PACK kernel void kernel_cpy_f32_f32_pack( global float * src0, ulong offset0, @@ -241,6 +242,7 @@ kernel void kernel_cpy_f32_f32_pack( dst_data[i00] = src[0]; } } +#endif // GGML_CL_NO_CPY_PACK kernel void kernel_cpy_i32_i32( global int * src0, @@ -311,3 +313,88 @@ kernel void kernel_cpy_f32_f32_flat( } } } + +kernel void kernel_cpy_i32_f32( + global int * src0, + ulong offset0, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne0, + int ne1, + int ne2, + int ne3, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global int *) ((global char *) src0 + offset0); + dst = (global float *) ((global char *) dst + offsetd); + + const ulong i01 = get_group_id(0); + const ulong i02 = get_group_id(1); + const ulong i03 = get_group_id(2); + + for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { + const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00; + const ulong i0 = n % ne0; + const ulong i1 = n / ne0 % ne1; + const ulong i2 = n / ne0 / ne1 % ne2; + const ulong i3 = n / ne0 / ne1 / ne2; + + global const int * src = (global const int *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); + global float * out = (global float *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + out[0] = convert_float(src[0]); + } +} + + +kernel void kernel_cpy_f32_i32( + global float * src0, + ulong offset0, + global int * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne0, + int ne1, + int ne2, + int ne3, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global float *) ((global char *) src0 + offset0); + dst = (global int *) ((global char *) dst + offsetd); + + const ulong i01 = get_group_id(0); + const ulong i02 = get_group_id(1); + const ulong i03 = get_group_id(2); + + for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { + const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00; + const ulong i0 = n % ne0; + const ulong i1 = n / ne0 % ne1; + const ulong i2 = n / ne0 / ne1 % ne2; + const ulong i3 = n / ne0 / ne1 / ne2; + + global const float * src = (global const float *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); + global int * out = (global int *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + out[0] = convert_int_rtz(src[0]); + } +} diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index acc8f9807..1a50ec4da 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -2661,3 +2661,720 @@ kernel void kernel_moe_expand_scale_q5_K( dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn); } } + +struct block_q2_K { + uint8_t scales[QK_K/16]; // low nibble scale, high nibble min, one per 16 weights + uint8_t qs[QK_K/4]; // quants, 2 bits each + half d; // super-block scale for the quantized scales + half dmin; // super-block scale for the quantized mins +}; + +kernel void kernel_convert_block_q2_K( + global struct block_q2_K * src0, + global uchar * dst_qs, + global uchar * dst_sc, + global half * dst_dm, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q2_K * b = src0 + i; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = b->qs[j]; + } + for (int j = 0; j < 16; ++j) { + dst_sc[16*i + j] = b->scales[j]; + } + dst_dm[2*i + 0] = b->d; + dst_dm[2*i + 1] = b->dmin; +} + +kernel void kernel_restore_block_q2_K( + global uchar * src_qs, + global uchar * src_sc, + global half * src_dm, + global struct block_q2_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q2_K * b = dst + i; + for (int j = 0; j < 64; ++j) { + b->qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 16; ++j) { + b->scales[j] = src_sc[16*i + j]; + } + b->d = src_dm[2*i + 0]; + b->dmin = src_dm[2*i + 1]; +} + +kernel void kernel_convert_block_q2_k_noshuffle( + global struct block_q2_K * src0, + global uchar * dst_qs, // QK_K/4 per block + global uchar * dst_sc, // QK_K/16 per block + global half * dst_dm, // 2 per block (d, dmin) + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q2_K * b = (global struct block_q2_K *) src0 + i; + + dst_dm[2*i + 0] = b->d; + dst_dm[2*i + 1] = b->dmin; + + for (int j = 0; j < QK_K/16; ++j) { dst_sc[(QK_K/16) * i + j] = b->scales[j]; } + + for (int g = 0; g < QK_K/4; ++g) { + uint pk = 0; + for (int t = 0; t < 4; ++t) { + const int e = 4*g + t; + const int sb = e >> 5; + const int ee = e & 31; + pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t)); + } + dst_qs[(QK_K/4) * i + g] = (uchar)pk; + } +} + +kernel void kernel_restore_block_q2_k_noshuffle( + global uchar * src_qs, + global uchar * src_sc, + global half * src_dm, + global struct block_q2_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q2_K * b = (global struct block_q2_K *) dst + i; + + b->d = src_dm[2*i + 0]; + b->dmin = src_dm[2*i + 1]; + + for (int j = 0; j < QK_K/16; ++j) { b->scales[j] = src_sc[(QK_K/16) * i + j]; } + + for (int n2 = 0; n2 < 2; ++n2) { + for (int ee = 0; ee < 32; ++ee) { + uint v = 0; + for (int j = 0; j < 4; ++j) { + const int sb = 4*n2 + j; + const int e = 32*sb + ee; + v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j); + } + b->qs[32*n2 + ee] = (uchar)v; + } + } +} + +struct block_q3_K { + uchar hmask[QK_K/8]; + uchar qs[QK_K/4]; + uchar scales[12]; + half d; +}; + +kernel void kernel_convert_block_q3_K( + global struct block_q3_K * src0, + global uchar * dst_qs, + global uchar * dst_hm, + global uchar * dst_sc, + global half * dst_d, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = src0 + i; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = b->qs[j]; + } + for (int j = 0; j < 32; ++j) { + dst_hm[32*i + j] = b->hmask[j]; + } + for (int j = 0; j < 12; ++j) { + dst_sc[12*i + j] = b->scales[j]; + } + dst_d[i] = b->d; +} + +kernel void kernel_restore_block_q3_K( + global uchar * src_qs, + global uchar * src_hm, + global uchar * src_sc, + global half * src_d, + global struct block_q3_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = dst + i; + for (int j = 0; j < 64; ++j) { + b->qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 32; ++j) { + b->hmask[j] = src_hm[32*i + j]; + } + for (int j = 0; j < 12; ++j) { + b->scales[j] = src_sc[12*i + j]; + } + b->d = src_d[i]; +} + +kernel void kernel_convert_block_q3_k_noshuffle( + global struct block_q3_K * src0, + global uchar * dst_qs, // QK_K/4 per block + global uchar * dst_hm, // QK_K/8 per block + global uint * dst_sc, // 3 per block + global half * dst_d, // 1 + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = (global struct block_q3_K *) src0 + i; + + dst_d[i] = b->d; + + for (int w = 0; w < 3; ++w) { + global const uchar * p = b->scales + 4*w; + dst_sc[3*i + w] = (uint)p[0] | ((uint)p[1] << 8) | ((uint)p[2] << 16) | ((uint)p[3] << 24); + } + + for (int g = 0; g < QK_K/4; ++g) { + uint pk = 0; + for (int t = 0; t < 4; ++t) { + const int e = 4*g + t; + const int sb = e >> 5; + const int ee = e & 31; + pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t)); + } + dst_qs[(QK_K/4) * i + g] = (uchar)pk; + } + + // two groups per hmask byte, so this loop owns the byte and cannot race + for (int t = 0; t < QK_K/8; ++t) { + uint v = 0; + for (int s = 0; s < 2; ++s) { + const int g = 2*t + s; + for (int u = 0; u < 4; ++u) { + const int e = 4*g + u; + const int sb = e >> 5; + const int ee = e & 31; + v |= (uint)(((b->hmask[ee] >> sb) & 1) << (4*s + u)); + } + } + dst_hm[(QK_K/8) * i + t] = (uchar)v; + } +} + +kernel void kernel_restore_block_q3_k_noshuffle( + global uchar * src_qs, + global uchar * src_hm, + global uint * src_sc, + global half * src_d, + global struct block_q3_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = (global struct block_q3_K *) dst + i; + + b->d = src_d[i]; + + for (int w = 0; w < 3; ++w) { + const uint x = src_sc[3*i + w]; + global uchar * p = b->scales + 4*w; + p[0] = (uchar)( x & 0xFF); + p[1] = (uchar)((x >> 8) & 0xFF); + p[2] = (uchar)((x >> 16) & 0xFF); + p[3] = (uchar)((x >> 24) & 0xFF); + } + + for (int n2 = 0; n2 < 2; ++n2) { + for (int ee = 0; ee < 32; ++ee) { + uint v = 0; + for (int j = 0; j < 4; ++j) { + const int sb = 4*n2 + j; + const int e = 32*sb + ee; + v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j); + } + b->qs[32*n2 + ee] = (uchar)v; + } + } + + for (int ee = 0; ee < 32; ++ee) { + uint v = 0; + for (int sb = 0; sb < 8; ++sb) { + const int e = 32*sb + ee; + const int g = e >> 2; + v |= ((uint)(src_hm[(QK_K/8) * i + (g >> 1)] >> (4*(g & 1) + (e & 3))) & 1u) << sb; + } + b->hmask[ee] = (uchar)v; + } +} + +struct block_iq2_s { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar scales[QK_K/32]; +}; + +kernel void kernel_convert_block_iq2_s( + global struct block_iq2_s * src0, + global uchar * dst_qs, // QK_K/8 per block + global uchar * dst_sg, // QK_K/8 + global uchar * dst_qh, // QK_K/32 + global uchar * dst_sc, // QK_K/32 + global half * dst_d, // 1 + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_iq2_s * b = src0 + i; + + dst_d[i] = b->d; + for (int j = 0; j < QK_K/8; ++j) { dst_qs[(QK_K/8) * i + j] = b->qs[j]; } + for (int j = 0; j < QK_K/8; ++j) { dst_sg[(QK_K/8) * i + j] = b->qs[QK_K/8 + j]; } + for (int j = 0; j < QK_K/32; ++j) { dst_qh[(QK_K/32) * i + j] = b->qh[j]; } + for (int j = 0; j < QK_K/32; ++j) { dst_sc[(QK_K/32) * i + j] = b->scales[j]; } +} + +kernel void kernel_restore_block_iq2_s( + global uchar * src_qs, + global uchar * src_sg, + global uchar * src_qh, + global uchar * src_sc, + global half * src_d, + global struct block_iq2_s * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_iq2_s * b = dst + i; + + b->d = src_d[i]; + for (int j = 0; j < QK_K/8; ++j) { b->qs[j] = src_qs[(QK_K/8) * i + j]; } + for (int j = 0; j < QK_K/8; ++j) { b->qs[QK_K/8 + j] = src_sg[(QK_K/8) * i + j]; } + for (int j = 0; j < QK_K/32; ++j) { b->qh[j] = src_qh[(QK_K/32) * i + j]; } + for (int j = 0; j < QK_K/32; ++j) { b->scales[j] = src_sc[(QK_K/32) * i + j]; } +} + +struct block_iq2_xs { + half d; + ushort qs[QK_K/8]; + uchar scales[QK_K/32]; +}; + +kernel void kernel_convert_block_iq2_xs( + global struct block_iq2_xs * src0, + global ushort * dst_qs, + global uchar * dst_sc, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + for (int j = 0; j < 32; ++j) { dst_qs[32*i + j] = src0[i].qs[j]; } + for (int j = 0; j < 8; ++j) { dst_sc[8*i + j] = src0[i].scales[j]; } +} + +kernel void kernel_restore_block_iq2_xs( + global ushort * src_qs, + global uchar * src_sc, + global half * src_d, + global struct block_iq2_xs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + for (int j = 0; j < 32; ++j) { dst[i].qs[j] = src_qs[32*i + j]; } + for (int j = 0; j < 8; ++j) { dst[i].scales[j] = src_sc[8*i + j]; } +} + +struct block_iq2_xxs { + half d; + ushort qs[QK_K/8]; +}; + +kernel void kernel_convert_block_iq2_xxs( + global struct block_iq2_xxs * src0, + global uchar * dst_qs, + global uint * dst_sas, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + for (int j = 0; j < 8; ++j) { + global ushort * q = src0[i].qs + 4*j; + uint indices = (uint)q[0] | ((uint)q[1] << 16); + vstore4(as_uchar4(indices), 8*i + j, dst_qs); + dst_sas[8*i + j] = (uint)q[2] | ((uint)q[3] << 16); + } +} + +kernel void kernel_restore_block_iq2_xxs( + global uchar * src_qs, + global uint * src_sas, + global half * src_d, + global struct block_iq2_xxs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + for (int j = 0; j < 8; ++j) { + global ushort * q = dst[i].qs + 4*j; + uint indices = as_uint(vload4(8*i + j, src_qs)); + uint sas = src_sas[8*i + j]; + q[0] = (ushort)indices; + q[1] = (ushort)(indices >> 16); + q[2] = (ushort)sas; + q[3] = (ushort)(sas >> 16); + } +} + +struct block_q2_0 { + half d; + uchar qs[16]; +}; + +kernel void kernel_convert_block_q2_0( + global struct block_q2_0 * src0, + global uchar * dst_qs, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + for (int j = 0; j < 16; ++j) { + dst_qs[16*i + j] = src0[i].qs[j]; + } +} + +kernel void kernel_restore_block_q2_0( + global uchar * src_qs, + global half * src_d, + global struct block_q2_0 * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + for (int j = 0; j < 16; ++j) { + dst[i].qs[j] = src_qs[16*i + j]; + } +} + +struct block_iq1_s { + half d; + uchar qs[QK_K/8]; + ushort qh[QK_K/32]; +}; + +kernel void kernel_convert_block_iq1_s( + global struct block_iq1_s * src0, + global uchar * dst_qs, + global ushort * dst_qh, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + for (int j = 0; j < 32; ++j) { + dst_qs[32*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 8; ++j) { + dst_qh[8*i + j] = src0[i].qh[j]; + } +} + +kernel void kernel_restore_block_iq1_s( + global uchar * src_qs, + global ushort * src_qh, + global half * src_d, + global struct block_iq1_s * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + for (int j = 0; j < 32; ++j) { + dst[i].qs[j] = src_qs[32*i + j]; + } + for (int j = 0; j < 8; ++j) { + dst[i].qh[j] = src_qh[8*i + j]; + } +} + +struct block_iq1_m { + uchar qs[QK_K/8]; + uchar qh[QK_K/16]; + uchar scales[QK_K/32]; +}; + +kernel void kernel_convert_block_iq1_m( + global struct block_iq1_m * src0, + global uchar * dst_qs, + global uchar * dst_qh, + global ushort * dst_sc, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + for (int j = 0; j < 32; ++j) { + dst_qs[32*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 16; ++j) { + dst_qh[16*i + j] = src0[i].qh[j]; + } + for (int j = 0; j < 4; ++j) { + dst_sc[4*i + j] = (ushort)src0[i].scales[2*j] | ((ushort)src0[i].scales[2*j + 1] << 8); + } +} + +kernel void kernel_restore_block_iq1_m( + global uchar * src_qs, + global uchar * src_qh, + global ushort * src_sc, + global struct block_iq1_m * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + for (int j = 0; j < 32; ++j) { + dst[i].qs[j] = src_qs[32*i + j]; + } + for (int j = 0; j < 16; ++j) { + dst[i].qh[j] = src_qh[16*i + j]; + } + for (int j = 0; j < 4; ++j) { + ushort sc = src_sc[4*i + j]; + dst[i].scales[2*j] = (uchar)sc; + dst[i].scales[2*j + 1] = (uchar)(sc >> 8); + } +} + +struct block_iq3_s { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar signs[QK_K/8]; + uchar scales[QK_K/64]; +}; + +kernel void kernel_convert_block_iq3_s( + global struct block_iq3_s * src0, + global uchar * dst_qs, + global uchar * dst_sg, + global uchar * dst_qh, + global uchar * dst_sc, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 32; ++j) { + dst_sg[32*i + j] = src0[i].signs[j]; + } + for (int j = 0; j < 8; ++j) { + dst_qh[8*i + j] = src0[i].qh[j]; + } + for (int j = 0; j < 4; ++j) { + dst_sc[4*i + j] = src0[i].scales[j]; + } +} + +kernel void kernel_restore_block_iq3_s( + global uchar * src_qs, + global uchar * src_sg, + global uchar * src_qh, + global uchar * src_sc, + global half * src_d, + global struct block_iq3_s * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + for (int j = 0; j < 64; ++j) { + dst[i].qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 32; ++j) { + dst[i].signs[j] = src_sg[32*i + j]; + } + for (int j = 0; j < 8; ++j) { + dst[i].qh[j] = src_qh[8*i + j]; + } + for (int j = 0; j < 4; ++j) { + dst[i].scales[j] = src_sc[4*i + j]; + } +} + +struct block_iq3_xxs { + half d; + uchar qs[3*QK_K/8]; +}; + +kernel void kernel_convert_block_iq3_xxs( + global struct block_iq3_xxs * src0, + global uchar * dst_qs, + global uint * dst_sas, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 8; ++j) { + global ushort * q = (global ushort *)(src0[i].qs + 64 + 4*j); + dst_sas[8*i + j] = (uint)q[0] | ((uint)q[1] << 16); + } +} + +kernel void kernel_restore_block_iq3_xxs( + global uchar * src_qs, + global uint * src_sas, + global half * src_d, + global struct block_iq3_xxs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + for (int j = 0; j < 64; ++j) { + dst[i].qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 8; ++j) { + global ushort * q = (global ushort *)(dst[i].qs + 64 + 4*j); + uint sas = src_sas[8*i + j]; + q[0] = (ushort)sas; + q[1] = (ushort)(sas >> 16); + } +} + +struct block_iq4_xs { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +}; + +kernel void kernel_convert_block_iq4_xs( + global struct block_iq4_xs * src0, + global uchar * dst_qs, + global uchar * dst_sl, + global ushort * dst_sh, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + dst_sh[i] = src0[i].scales_h; + for (int j = 0; j < 128; ++j) { + dst_qs[128*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 4; ++j) { + dst_sl[4*i + j] = src0[i].scales_l[j]; + } +} + +kernel void kernel_restore_block_iq4_xs( + global uchar * src_qs, + global uchar * src_sl, + global ushort * src_sh, + global half * src_d, + global struct block_iq4_xs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + dst[i].scales_h = src_sh[i]; + for (int j = 0; j < 128; ++j) { + dst[i].qs[j] = src_qs[128*i + j]; + } + for (int j = 0; j < 4; ++j) { + dst[i].scales_l[j] = src_sl[4*i + j]; + } +} + +kernel void kernel_convert_block_iq4_xs_noshuffle( + global struct block_iq4_xs * src0, + global ushort * dst_qs, + global uint * dst_sl, + global ushort * dst_sh, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + dst_sh[i] = src0[i].scales_h; + dst_sl[i] = as_uint(vload4(0, src0[i].scales_l)); + for (int sb = 0; sb < 8; ++sb) { + for (int u = 0; u < 8; ++u) { + ushort q = 0; + for (int j = 0; j < 4; ++j) { + uchar v = src0[i].qs[16*sb + 4*(u & 3) + j]; + q |= (ushort)(((v >> (4*(u >> 2))) & 15) << (4*j)); + } + dst_qs[64*i + 8*sb + u] = q; + } + } +} + +kernel void kernel_restore_block_iq4_xs_noshuffle( + global ushort * src_qs, + global uint * src_sl, + global ushort * src_sh, + global half * src_d, + global struct block_iq4_xs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + dst[i].scales_h = src_sh[i]; + vstore4(as_uchar4(src_sl[i]), 0, dst[i].scales_l); + for (int sb = 0; sb < 8; ++sb) { + for (int j = 0; j < 16; ++j) { + const int shift = 4*(j & 3); + const ushort lo = src_qs[64*i + 8*sb + j/4]; + const ushort hi = src_qs[64*i + 8*sb + j/4 + 4]; + dst[i].qs[16*sb + j] = (uchar)(((lo >> shift) & 15) | (((hi >> shift) & 15) << 4)); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl b/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl deleted file mode 100644 index 36eff0439..000000000 --- a/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +++ /dev/null @@ -1,58 +0,0 @@ -#pragma OPENCL EXTENSION cl_khr_fp16 : enable - -//------------------------------------------------------------------------------ -// diag_mask_inf kernels -//------------------------------------------------------------------------------ -kernel void kernel_diag_mask_inf( - global float * src0, - ulong offset0, - global float * dst, - ulong offsetd, - int ne00, - int ne01, - int n_past -) { - src0 = (global float*)((global char*)src0 + offset0); - dst = (global float*)((global char*)dst + offsetd); - - int i02 = get_global_id(2); - int i01 = get_global_id(1); - int i00 = get_global_id(0); - - if (i00 > n_past + i01) { - dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY; - } else { - dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00]; - } -} - -kernel void kernel_diag_mask_inf_8( - global float4 * src0, - ulong offset0, - global float4 * dst, - ulong offsetd, - int ne00, - int ne01, - int n_past -) { - src0 = (global float4*)((global char*)src0 + offset0); - dst = (global float4*)((global char*)dst + offsetd); - - int i = 2*get_global_id(0); - - dst[i+0] = src0[i+0]; - dst[i+1] = src0[i+1]; - int i4 = 4*i; - int i02 = i4/(ne00*ne01); i4 -= i02*ne00*ne01; - int i01 = i4/(ne00); i4 -= i01*ne00; - int i00 = i4; - for (int k = 3; k >= 0; --k) { - if (i00 + 4 + k <= n_past + i01) { - break; - } - (&dst[i+1])[k] = -INFINITY; - if (i00 + k > n_past + i01) { - (&dst[i])[k] = -INFINITY; - } - } -} diff --git a/ggml/src/ggml-opencl/kernels/dsv4_hc.cl b/ggml/src/ggml-opencl/kernels/dsv4_hc.cl new file mode 100644 index 000000000..164ab1dae --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/dsv4_hc.cl @@ -0,0 +1,101 @@ +#if HC_VEC == 4 +typedef float4 hc_float; +#define HC_LOAD(p) vload4(0, (global const float *) (p)) +#define HC_STORE(v, p) vstore4(v, 0, (global float *) (p)) +#else +typedef float hc_float; +#define HC_LOAD(p) (*(global const float *) (p)) +#define HC_STORE(v, p) (*(global float *) (p) = (v)) +#endif + +#if HC_VEC == 1 +kernel void kernel_dsv4_hc_pre( + global const char * x, + ulong offset_x, + ulong nbx0, + ulong nbx1, + ulong nbx2, + global const char * weights, + ulong offset_w, + ulong nbw0, + ulong nbw1, + ulong nbw2, + global char * dst, + ulong offset_dst, + long n_embd, + long n_hc, + float scale, + int gated +) { + const size_t i = get_global_id(0); + const size_t t = get_global_id(1); + if (i >= n_embd) { + return; + } + + x += offset_x + i*nbx0 + t*nbx2; + weights += offset_w + (gated ? i*nbw0 + t*nbw2 : t*nbw1); + + float sum = 0.0f; + for (long h = 0; h < n_hc; ++h) { + const float xv = *(global const float *) (x + h*nbx1); + float wv = *(global const float *) (weights + h*(gated ? nbw1 : nbw0)); + if (gated) { + wv = 1.0f / (1.0f + exp(-wv)); + } + sum += xv * wv; + } + ((global float *) (dst + offset_dst))[i + t*n_embd] = scale * sum; +} +#endif + +kernel void kernel_dsv4_hc_post( + global const char * x, + ulong offset_x, + ulong nbx0, + ulong nbx1, + ulong nbx2, + global const char * residual, + ulong offset_r, + ulong nbr0, + ulong nbr1, + ulong nbr2, + global const char * post, + ulong offset_p, + ulong nbp0, + ulong nbp1, + ulong nbp2, + global const char * comb, + ulong offset_c, + ulong nbc0, + ulong nbc1, + ulong nbc2, + global char * dst, + ulong offset_dst, + long n_embd, + long n_hc +) { + const size_t i = get_global_id(0) * HC_VEC; + const size_t h = get_global_id(1); + const size_t t = get_global_id(2); + if (i >= n_embd) { + return; + } + + const hc_float xv = HC_LOAD(x + offset_x + i*nbx0 + t*nbx1); + const float pv = *(global const float *) (post + offset_p + h*nbp0 + t*nbp1); + residual += offset_r + i*nbr0 + t*nbr2; + + hc_float sum = xv * pv; + if (comb) { + comb += offset_c + h*nbc0 + t*nbc2; + for (long s = 0; s < n_hc; ++s) { + const hc_float rv = HC_LOAD(residual + s*nbr1); + const float cv = *(global const float *) (comb + s*nbc1); + sum += rv * cv; + } + } else { + sum += HC_LOAD(residual + h*nbr1); + } + HC_STORE(sum, dst + offset_dst + (i + (h + t*n_hc)*n_embd)*sizeof(float)); +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl index f9797d346..cb6fd8930 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl @@ -175,6 +175,12 @@ __kernel void flash_attn_f16( ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } if (is_causal) { const int causal_limit = n_kv - n_q + my_query_row; @@ -196,13 +202,6 @@ __kernel void flash_attn_f16( if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3]; } - if (logit_softcap > 0.0f) { - s0 = logit_softcap * tanh(s0 / logit_softcap); - s1 = logit_softcap * tanh(s1 / logit_softcap); - s2 = logit_softcap * tanh(s2 / logit_softcap); - s3 = logit_softcap * tanh(s3 / logit_softcap); - } - const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); const ACC_TYPE scale_prev = native_exp(m_i - m_new); const ACC_TYPE p0 = native_exp(s0 - m_new); @@ -331,13 +330,13 @@ __kernel void flash_attn_f16_q1( dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } m_i = max(m_i, score); } @@ -367,13 +366,13 @@ __kernel void flash_attn_f16_q1( dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE p = exp(score - m_final); l_i += p; FA_UNROLL diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl index 5911524e1..15b9cf30a 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl @@ -176,6 +176,12 @@ __kernel void flash_attn_f32( ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } if (is_causal) { const int causal_limit = n_kv - n_q + my_query_row; @@ -197,13 +203,6 @@ __kernel void flash_attn_f32( if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3]; } - if (logit_softcap > 0.0f) { - s0 = logit_softcap * tanh(s0 / logit_softcap); - s1 = logit_softcap * tanh(s1 / logit_softcap); - s2 = logit_softcap * tanh(s2 / logit_softcap); - s3 = logit_softcap * tanh(s3 / logit_softcap); - } - const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); const ACC_TYPE scale_prev = native_exp(m_i - m_new); const ACC_TYPE p0 = native_exp(s0 - m_new); @@ -332,13 +331,13 @@ __kernel void flash_attn_f32_q1( dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } m_i = max(m_i, score); } @@ -368,13 +367,13 @@ __kernel void flash_attn_f32_q1( dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE p = exp(score - m_final); l_i += p; FA_UNROLL diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl index bf7695a2c..d1cc14801 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl @@ -336,6 +336,10 @@ __kernel void FA_TILE_NAME( ACC_TYPE score0 = partial0 * scale; ACC_TYPE score1 = partial1 * scale; + if (logit_softcap > 0.0f) { + score0 = logit_softcap * tanh(score0 / logit_softcap); + score1 = logit_softcap * tanh(score1 / logit_softcap); + } if (!query_valid) { score0 = FA_M_INIT; score1 = FA_M_INIT; } if (is_causal) { @@ -359,11 +363,6 @@ __kernel void FA_TILE_NAME( } } - if (logit_softcap > 0.0f) { - score0 = logit_softcap * tanh(score0 / logit_softcap); - score1 = logit_softcap * tanh(score1 / logit_softcap); - } - const ACC_TYPE m_new = max(m_i, max(score0, score1)); // Whole tile masked (m_new == FA_M_INIT): force the exp() args // far negative so the tile contributes 0, not exp(0)=1. @@ -408,6 +407,9 @@ __kernel void FA_TILE_NAME( score += local_partial[j][q_lane * N_SPLIT + s]; } score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (is_causal && k_row > (n_kv - n_q + my_query_row)) score = FA_M_INIT; if (k_row >= n_kv) score = FA_M_INIT; @@ -424,10 +426,6 @@ __kernel void FA_TILE_NAME( } } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } - m_new = max(m_new, score); local_p[q_lane][j] = score; } @@ -502,6 +500,12 @@ __kernel void FA_TILE_NAME( ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } if (is_causal) { const int causal_limit = n_kv - n_q + my_query_row; @@ -531,13 +535,6 @@ __kernel void FA_TILE_NAME( } } - if (logit_softcap > 0.0f) { - s0 = logit_softcap * tanh(s0 / logit_softcap); - s1 = logit_softcap * tanh(s1 / logit_softcap); - s2 = logit_softcap * tanh(s2 / logit_softcap); - s3 = logit_softcap * tanh(s3 / logit_softcap); - } - const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); // Whole tile masked (m_new == FA_M_INIT): force the exp() args // far negative so the tile contributes 0, not exp(0)=1. @@ -665,7 +662,7 @@ __kernel void FA_TILE_NAME( // allow bypassing decode kernels to avoid compiler crash for DK=512 on Adreno GPUs #ifndef FA_PREFILL_ONLY -#ifndef FA_MQ_ONLY // q1 excluded from the MQ-only (g8) program +#if !defined(FA_MQ_ONLY) && !defined(FA_VEC_ONLY) REQD_FA_SG __kernel void flash_attn_f32_f16_q1( const global void * q_void, ulong q_offset, @@ -748,13 +745,13 @@ __kernel void flash_attn_f32_f16_q1( dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } m_i = max(m_i, score); } @@ -776,13 +773,13 @@ __kernel void flash_attn_f32_f16_q1( dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE p = exp(score - m_final); l_i += p; FA_UNROLL @@ -932,14 +929,14 @@ __kernel void flash_attn_f32_f16_q1_vec( } ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; score += slope * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } // FA-2 online update. All threads in the subgroup see the same score, // so m_i and l_i evolve identically across lanes within the subgroup. @@ -1122,13 +1119,13 @@ __kernel void flash_attn_f32_f16_q1_local_tile( if (tid == 0) { float s = red[0] * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; s += slope * (float) mask_ptr[kv_idx]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score_shared = s; } barrier(CLK_LOCAL_MEM_FENCE); @@ -1323,13 +1320,13 @@ __kernel void flash_attn_f32_f16_q1_local_mq_split( contrib += q_shared[h * DK + d] * (float) k_tile[j * DK + d]; } float s = sub_group_reduce_add(contrib) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (float) mask_ptr[kv_idx]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = s; } @@ -1385,6 +1382,7 @@ __kernel void flash_attn_f32_f16_q1_local_mq_split( #endif #define MQ_WG_SIZE (Q1_WG_SIZE * MQ_NSG) +#ifndef FA_MQ_SPLIT_ONLY REQD_SUBGROUP_SIZE_64 __kernel void flash_attn_f32_f16_q1_vec_mq( const global void * q_void, ulong q_offset, @@ -1509,13 +1507,13 @@ __kernel void flash_attn_f32_f16_q1_vec_mq( for (int h = 0; h < MQ_GQA; ++h) { const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = s; } @@ -1606,6 +1604,8 @@ __kernel void flash_attn_f32_f16_q1_vec_mq( } } +#endif // !FA_MQ_SPLIT_ONLY + #ifndef MQ_NSG_SPLIT #define MQ_NSG_SPLIT 4 #endif @@ -1615,6 +1615,10 @@ __kernel void flash_attn_f32_f16_q1_vec_mq( #define FA_PARTIAL_FLOATS (2 + DV) #endif +#ifndef FA_HEAD_SUB +#define FA_HEAD_SUB 1 +#endif + REQD_SUBGROUP_SIZE_64 __kernel void flash_attn_f32_f16_q1_vec_mq_split( const global void * q_void, ulong q_offset, @@ -1652,8 +1656,12 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split( const int split_idx = split_q_idx % n_splits; const int q_idx = split_q_idx / n_splits; - const int batch_idx = kvhead_batch_idx / n_head_kv; - const int head_kv_idx = kvhead_batch_idx % n_head_kv; + const int hgroups = n_head_kv * FA_HEAD_SUB; + const int batch_idx = kvhead_batch_idx / hgroups; + const int hg = kvhead_batch_idx % hgroups; + const int head_kv_idx = hg / FA_HEAD_SUB; + const int head_sub = hg % FA_HEAD_SUB; +#define FA_MQS_HEAD_IDX(h) (head_kv_idx * (MQ_GQA * FA_HEAD_SUB) + head_sub * MQ_GQA + (h)) const int kv_start = split_idx * kv_per_split; const int kv_end = min(kv_start + kv_per_split, n_kv); @@ -1666,7 +1674,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split( if (tid == 0) { #pragma unroll for (int h = 0; h < MQ_GQA; ++h) { - const int head_idx = head_kv_idx * MQ_GQA + h; + const int head_idx = FA_MQS_HEAD_IDX(h); const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits + split_idx); global float * rec = partial_void + rec_idx * record_stride; @@ -1681,22 +1689,33 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split( const global char * k_base = (const global char *) k_void + k_offset; const global char * v_base = (const global char *) v_void + v_offset; +#ifdef FA_Q1_Q_REG + ACC_TYPE4 q_reg[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = FA_MQS_HEAD_IDX(h); + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_reg[h] = (tid_sg < DK_VEC) ? CONVERT_Q_ACC4(q_ptr[tid_sg]) : (ACC_TYPE4)(0.0f); + } +#else // stage MQ_GQA Q rows in __local once (uniform across WG) __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) { const int h = i / DK_VEC; const int k = i % DK_VEC; - const int head_idx = head_kv_idx * MQ_GQA + h; + const int head_idx = FA_MQS_HEAD_IDX(h); const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); } barrier(CLK_LOCAL_MEM_FENCE); +#endif float slope[MQ_GQA]; #pragma unroll for (int h = 0; h < MQ_GQA; ++h) { - slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + slope[h] = get_alibi_slope(max_bias, FA_MQS_HEAD_IDX(h), n_head_log2, m0, m1); } const global char * mask_base[MQ_GQA]; @@ -1707,7 +1726,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split( (ulong) q_idx * mask_nb1; #pragma unroll for (int h = 0; h < MQ_GQA; ++h) { - const int head_idx = head_kv_idx * MQ_GQA + h; + const int head_idx = FA_MQS_HEAD_IDX(h); const int mask_head_idx = head_idx % mask_ne2; mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; } @@ -1742,6 +1761,15 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split( ACC_TYPE4 dot4[MQ_GQA]; #pragma unroll for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); +#ifdef FA_Q1_Q_REG + if (tid_sg < DK_VEC) { + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[tid_sg]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_reg[h], k_vec, dot4[h]); + } + } +#else for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]); #pragma unroll @@ -1749,19 +1777,20 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split( dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]); } } +#endif ACC_TYPE score[MQ_GQA]; #pragma unroll for (int h = 0; h < MQ_GQA; ++h) { const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = s; } @@ -1810,7 +1839,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split( barrier(CLK_LOCAL_MEM_FENCE); if (sgid == 0) { - const int head_idx = head_kv_idx * MQ_GQA + h; + const int head_idx = FA_MQS_HEAD_IDX(h); // fold per-subgroup (m, l) into split-level (m_c, l_c) ACC_TYPE m_c = sg_m[h][0]; @@ -1848,6 +1877,9 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split( } } +#undef FA_MQS_HEAD_IDX + +#ifndef FA_MQ_SPLIT_ONLY // Cluster-parallel variant of _q1_vec_mq_split // // Tthe baseline keeps one 256B K row in flight per subgroup (32 lanes cooperate @@ -1936,8 +1968,12 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( const int split_idx = split_q_idx % n_splits; const int q_idx = split_q_idx / n_splits; - const int batch_idx = kvhead_batch_idx / n_head_kv; - const int head_kv_idx = kvhead_batch_idx % n_head_kv; + const int hgroups = n_head_kv * FA_HEAD_SUB; + const int batch_idx = kvhead_batch_idx / hgroups; + const int hg = kvhead_batch_idx % hgroups; + const int head_kv_idx = hg / FA_HEAD_SUB; + const int head_sub = hg % FA_HEAD_SUB; +#define FA_HEAD_IDX(h) (head_kv_idx * (MQ_GQA * FA_HEAD_SUB) + head_sub * MQ_GQA + (h)) const int kv_start = split_idx * kv_per_split; const int kv_end = min(kv_start + kv_per_split, n_kv); @@ -1948,7 +1984,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( if (tid == 0) { #pragma unroll for (int h = 0; h < MQ_GQA; ++h) { - const int head_idx = head_kv_idx * MQ_GQA + h; + const int head_idx = FA_HEAD_IDX(h); const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits + split_idx); global float * rec = partial_void + rec_idx * record_stride; @@ -1968,7 +2004,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) { const int h = i / DK_VEC; const int k = i % DK_VEC; - const int head_idx = head_kv_idx * MQ_GQA + h; + const int head_idx = FA_HEAD_IDX(h); const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); @@ -1978,9 +2014,17 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( float slope[MQ_GQA]; #pragma unroll for (int h = 0; h < MQ_GQA; ++h) { - slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + slope[h] = get_alibi_slope(max_bias, FA_HEAD_IDX(h), n_head_log2, m0, m1); } +#ifdef FA_CL_MASK_BCAST + const global char * mask_base_b = NULL; + if (mask_void != NULL) { + mask_base_b = (const global char *) mask_void + mask_offset + + (batch_idx % mask_ne3) * mask_nb3 + (ulong) q_idx * mask_nb1; + } + const int mask_bcast = mask_base_b != NULL && mask_ne2 == 1; +#else const global char * mask_base[MQ_GQA]; if (mask_void != NULL) { const int mask_batch_idx = batch_idx % mask_ne3; @@ -1989,7 +2033,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( (ulong) q_idx * mask_nb1; #pragma unroll for (int h = 0; h < MQ_GQA; ++h) { - const int head_idx = head_kv_idx * MQ_GQA + h; + const int head_idx = FA_HEAD_IDX(h); const int mask_head_idx = head_idx % mask_ne2; mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; } @@ -1997,6 +2041,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( #pragma unroll for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; } +#endif // Per-CLUSTER online-softmax state (uniform across the cluster's lanes); // o_acc holds this lane's DV slice {lic + FA_CL_C*i}. @@ -2031,6 +2076,73 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + kv_row_base + (ulong) k_safe * k_nb1); const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_base + (ulong) k_safe * v_nb1); +#if defined(FA_CL_MHRED) && MQ_GQA == 4 && FA_CL_C == 16 && FA_CL_DK == 1 && FA_CL_DV == 1 + ACC_TYPE mask_val = 0.0f; + if (mask_bcast) { + mask_val = (ACC_TYPE) ((const global MASK_DATA_TYPE *) mask_base_b)[k_safe]; + } + const ACC_TYPE4 k_vec_1 = CONVERT_KV_ACC4(k_ptr[lic]); + const ACC_TYPE4 v_vec_1 = CONVERT_KV_ACC4(v_ptr[lic]); + + // Reduce four heads with eight shuffles and keep each head's summation order. + const int mh_b0 = lic & 1; + const int mh_b1 = lic & 2; + + ACC_TYPE mh_p[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE4 d4 = mad(q_shared[h * DK_VEC + lic], k_vec_1, (ACC_TYPE4)(0.0f)); + mh_p[h] = d4.s0 + d4.s1 + d4.s2 + d4.s3; + } + + ACC_TYPE mh_r2[2]; + #pragma unroll + for (int j = 0; j < 2; ++j) { + const ACC_TYPE keep = mh_b0 ? mh_p[j + 2] : mh_p[j]; + const ACC_TYPE send = mh_b0 ? mh_p[j] : mh_p[j + 2]; + mh_r2[j] = keep + sub_group_shuffle_xor(send, 1); + } + ACC_TYPE mh_r1 = (mh_b1 ? mh_r2[1] : mh_r2[0]) + + sub_group_shuffle_xor(mh_b1 ? mh_r2[0] : mh_r2[1], 2); + mh_r1 += sub_group_shuffle_xor(mh_r1, 4); + mh_r1 += sub_group_shuffle_xor(mh_r1, 8); + + ACC_TYPE mh_e2[2]; + { + const ACC_TYPE other = sub_group_shuffle_xor(mh_r1, 2); + mh_e2[0] = mh_b1 ? other : mh_r1; + mh_e2[1] = mh_b1 ? mh_r1 : other; + } + ACC_TYPE mh_s[MQ_GQA]; + #pragma unroll + for (int j = 0; j < 2; ++j) { + const ACC_TYPE other = sub_group_shuffle_xor(mh_e2[j], 1); + mh_s[j] = mh_b0 ? other : mh_e2[j]; + mh_s[j + 2] = mh_b0 ? mh_e2[j] : other; + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE s = mh_s[h] * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_bcast) { + s += slope[h] * mask_val; + } else if (mask_base_b != NULL) { + const int mask_head_idx = FA_HEAD_IDX(h) % mask_ne2; + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base_b + mask_head_idx * mask_nb2); + s += slope[h] * (ACC_TYPE) mask_ptr[k_safe]; + } + const ACC_TYPE sc = valid ? s : FA_M_INIT; + const ACC_TYPE m_new = max(m_i[h], sc); + const ACC_TYPE sp = native_exp(m_i[h] - m_new); + const ACC_TYPE p = native_exp(sc - m_new); + l_i[h] = l_i[h] * sp + p; + m_i[h] = m_new; + o_acc[h][0] = mad(p, v_vec_1, o_acc[h][0] * sp); + } +#else // Dot: this lane covers DK elements {lic + FA_CL_C*i} of the cluster's row. ACC_TYPE4 dot4[MQ_GQA]; #pragma unroll @@ -2055,13 +2167,13 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( s += sub_group_shuffle_xor(s, step); } s *= scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_safe]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = valid ? s : FA_M_INIT; } @@ -2087,6 +2199,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( o_acc[h][i] = mad(p_h[h], v_vec, o_acc[h][i] * sp_h[h]); } } +#endif } // Merge stage 1: fold the FA_CL_NCL cluster partials inside the subgroup. @@ -2148,7 +2261,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( barrier(CLK_LOCAL_MEM_FENCE); if (sgid == 0) { - const int head_idx = head_kv_idx * MQ_GQA + h; + const int head_idx = FA_HEAD_IDX(h); ACC_TYPE m_c = sg_m[h][0]; #pragma unroll @@ -2184,6 +2297,8 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( } } +#undef FA_HEAD_IDX + #endif // DK_VEC/DV_VEC divisible by FA_CL_C #endif // HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8) @@ -2330,13 +2445,13 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_k_img( for (int h = 0; h < MQ_GQA; ++h) { const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = s; } @@ -2419,9 +2534,11 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_k_img( barrier(CLK_LOCAL_MEM_FENCE); } } +#endif // !FA_MQ_SPLIT_ONLY #endif // !FA_DECODE_ONLY #ifndef FA_MQ_ONLY // q1_split + merge excluded from the MQ-only (g8) program +#ifndef FA_VEC_ONLY __kernel void flash_attn_f32_f16_q1_split( const global void * q_void, ulong q_offset, const global void * k_void, ulong k_offset, @@ -2513,13 +2630,13 @@ __kernel void flash_attn_f32_f16_q1_split( dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); score += slope * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } m_i = max(m_i, score); } @@ -2542,13 +2659,13 @@ __kernel void flash_attn_f32_f16_q1_split( dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); score += slope * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE p = exp(score - m_c); l_i += p; #pragma unroll @@ -2578,6 +2695,8 @@ __kernel void flash_attn_f32_f16_q1_split( } } +#endif // !FA_VEC_ONLY + // FD Pass 2: merge per-split partials into final O // empty splits drop via exp(-INF)=0. __kernel void flash_attn_f32_merge( diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl index 48adba4f7..3e75dc942 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl @@ -287,14 +287,14 @@ __kernel void flash_attn_f32_q4_0_q1( #endif } score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } // Online softmax step. const ACC_TYPE m_new = max(m_i, score); @@ -554,14 +554,14 @@ __kernel void flash_attn_f32_q4_0_q1_vec( ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; #endif + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; score += slope * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE m_new = max(m_i, score); const ACC_TYPE scale_prev = native_exp(m_i - m_new); @@ -745,13 +745,13 @@ __kernel void flash_attn_f32_q4_0_q1_split( #endif } score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); score += slope * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } // Online softmax step. const ACC_TYPE m_new = max(m_i, score); @@ -1037,13 +1037,13 @@ __kernel void flash_attn_f32_q4_0_q1_vec_mq_split( #pragma unroll for (int h = 0; h < MQ_GQA; ++h) { ACC_TYPE s = sub_group_reduce_add(lane_contrib[h]) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = s; } #else @@ -1067,13 +1067,13 @@ __kernel void flash_attn_f32_q4_0_q1_vec_mq_split( for (int h = 0; h < MQ_GQA; ++h) { const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = s; } #endif @@ -1385,13 +1385,13 @@ __kernel void flash_attn_f32_q4_0_q1_vec_mq_split_c8( s += sub_group_shuffle_xor(s, step); } s *= scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_safe]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = valid ? s : FA_M_INIT; } @@ -1852,9 +1852,15 @@ __kernel void flash_attn_f32_q4_0( s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2); s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2); #endif - if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; } #endif s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } + if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; } if (is_causal) { const int causal_limit = n_kv - n_q + my_query_row; @@ -1876,12 +1882,6 @@ __kernel void flash_attn_f32_q4_0( if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2]; if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3]; } - if (logit_softcap > 0.0f) { - s0 = logit_softcap * tanh(s0 / logit_softcap); - s1 = logit_softcap * tanh(s1 / logit_softcap); - s2 = logit_softcap * tanh(s2 / logit_softcap); - s3 = logit_softcap * tanh(s3 / logit_softcap); - } const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); // Whole tile masked (m_new == FA_M_INIT): force the exp() args diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl index f50912d21..75d0d9b0a 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl @@ -232,14 +232,14 @@ __kernel void flash_attn_f32_q8_0_q1( #endif } score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } // Online softmax step. const ACC_TYPE m_new = max(m_i, score); @@ -437,14 +437,14 @@ __kernel void flash_attn_f32_q8_0_q1_vec( } ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; score += slope * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE m_new = max(m_i, score); const ACC_TYPE scale_prev = native_exp(m_i - m_new); @@ -627,13 +627,13 @@ __kernel void flash_attn_f32_q8_0_q1_split( #endif } score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); score += slope * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } // Online softmax step. const ACC_TYPE m_new = max(m_i, score); @@ -882,13 +882,13 @@ __kernel void flash_attn_f32_q8_0_q1_vec_mq_split( for (int h = 0; h < MQ_GQA; ++h) { const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = s; } @@ -1151,13 +1151,13 @@ __kernel void flash_attn_f32_q8_0_q1_vec_mq_split_c8( s += sub_group_shuffle_xor(s, step); } s *= scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } if (mask_base[h] != NULL) { const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; s += slope[h] * (ACC_TYPE) mask_ptr[k_safe]; } - if (logit_softcap > 0.0f) { - s = logit_softcap * tanh(s / logit_softcap); - } score[h] = valid ? s : FA_M_INIT; } @@ -1652,9 +1652,15 @@ __kernel void flash_attn_f32_q8_0( s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2); s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2); #endif - if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; } #endif s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } + if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; } if (is_causal) { const int causal_limit = n_kv - n_q + my_query_row; @@ -1676,12 +1682,6 @@ __kernel void flash_attn_f32_q8_0( if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2]; if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3]; } - if (logit_softcap > 0.0f) { - s0 = logit_softcap * tanh(s0 / logit_softcap); - s1 = logit_softcap * tanh(s1 / logit_softcap); - s2 = logit_softcap * tanh(s2 / logit_softcap); - s3 = logit_softcap * tanh(s3 / logit_softcap); - } const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); // Whole tile masked (m_new == FA_M_INIT): force the exp() args diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl index 88ead4bcb..74aa67b08 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl @@ -154,3 +154,123 @@ __kernel void flash_attn_blk_f16( blk[((ulong) mask_slice * (ulong) n_q_blocks + (ulong) q_block_idx) * (ulong) n_kv_blocks + (ulong) kv_block_idx] = res; } + +// V^T for the decomposed flash attention: [dv, n_kv, n_head_kv] (strided) -> [n_kv, dv, n_head_kv] +// (contiguous). This is the layout the Adreno KQV image GEMM reads. A local tile keeps both the +// read and the write coalesced, and the +1 column avoids bank conflicts. +#define FA_VT_TILE 32 +#define FA_VT_ROWS 8 +__kernel void flash_attn_v_transpose_f16( + const global void * v_void, ulong v_offset, + global void * vt_void, + const int n_kv, + const int dv, + const int n_head_kv, + const ulong v_nb1, + const ulong v_nb2 +) { + __local half tile[FA_VT_TILE][FA_VT_TILE + 1]; + + const int d0 = get_group_id(0) * FA_VT_TILE; + const int k0 = get_group_id(1) * FA_VT_TILE; + const int head_kv_idx = get_group_id(2); + + const int lx = get_local_id(0); + const int ly = get_local_id(1); + + const global char * v_head = (const global char *) v_void + v_offset + (ulong) head_kv_idx * v_nb2; + + for (int r = 0; r < FA_VT_TILE; r += FA_VT_ROWS) { + const int d = d0 + lx; + const int kv = k0 + ly + r; + half val = (half) 0.0f; + if (d < dv && kv < n_kv) { + val = ((const global half *) (v_head + (ulong) kv * v_nb1))[d]; + } + tile[ly + r][lx] = val; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + global half * vt_head = (global half *) vt_void + (ulong) head_kv_idx * (ulong) n_kv * (ulong) dv; + + for (int r = 0; r < FA_VT_TILE; r += FA_VT_ROWS) { + const int kv = k0 + lx; + const int d = d0 + ly + r; + if (d < dv && kv < n_kv) { + vt_head[(ulong) d * (ulong) n_kv + (ulong) kv] = tile[lx][ly + r]; + } + } +} + +// Transpose V and quantize each block of 32 KV rows for the int8 KQV GEMM. +__kernel void flash_attn_v_transpose_q8( + const global void * v_void, ulong v_offset, + global void * vt_q_void, // char, [n_head_kv][dv][kv_pitch] + global void * vt_d_void, // half, [n_head_kv][dv][kv_pitch/32] + const int n_kv, + const int dv, + const int n_head_kv, + const ulong v_nb1, + const ulong v_nb2, + const ulong v_nb3, + const int kv_pitch // bytes per V^T row, >= n_kv, multiple of 32 +) { + __local half tile[FA_VT_TILE][FA_VT_TILE + 1]; + + const int d0 = get_group_id(0) * FA_VT_TILE; + const int k0 = get_group_id(1) * FA_VT_TILE; + const int head_kv_idx = get_group_id(2) % n_head_kv; + const int batch_idx = get_group_id(2) / n_head_kv; + + const int lx = get_local_id(0); + const int ly = get_local_id(1); + + const global char * v_head = (const global char *) v_void + v_offset + + (ulong) batch_idx * v_nb3 + (ulong) head_kv_idx * v_nb2; + + for (int r = 0; r < FA_VT_TILE; r += FA_VT_ROWS) { + const int d = d0 + lx; + const int kv = k0 + ly + r; + half val = (half) 0.0f; + if (d < dv && kv < n_kv) { + val = ((const global half *) (v_head + (ulong) kv * v_nb1))[d]; + } + tile[ly + r][lx] = val; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + const int lid = ly * FA_VT_TILE + lx; + const int nblk = kv_pitch / FA_VT_TILE; + + if (lid < FA_VT_TILE) { + const int d = d0 + lid; + if (d < dv) { + const ulong hbase = ((ulong) batch_idx * (ulong) n_head_kv + (ulong) head_kv_idx); + + global char * qrow = (global char *) vt_q_void + hbase*(ulong)kv_pitch*(ulong)dv + (ulong) d * (ulong) kv_pitch; + global half * drow = (global half *) vt_d_void + hbase*(ulong)nblk*(ulong)dv + (ulong) d * (ulong) nblk; + + float v[FA_VT_TILE]; + float amax = 0.0f; + + for (int i = 0; i < FA_VT_TILE; ++i) { + v[i] = (float) tile[i][lid]; + amax = fmax(amax, fabs(v[i])); + } + + const float dq = amax / 127.0f; + const float id = amax > 0.0f ? 127.0f / amax : 0.0f; + + drow[k0 / FA_VT_TILE] = (half) dq; + + for (int i = 0; i < FA_VT_TILE; ++i) { + const int kv = k0 + i; + if (kv < n_kv) { + qrow[kv] = (char) clamp((int) rint(v[i] * id), -127, 127); + } + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_pv_direct.cl b/ggml/src/ggml-opencl/kernels/flash_attn_pv_direct.cl new file mode 100644 index 000000000..201fe4443 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_pv_direct.cl @@ -0,0 +1,106 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define FA_KQVD_MAXC 32 + +// Read F16 V in its cache layout. Share each V row across the queries and heads of one KV group. +// P8 and its block scales are expanded in local memory; KV splits write separate partial sums. +__attribute__((qcom_reqd_sub_group_size("half"))) +kernel void kernel_fa_kqv_direct_f16( + global const char * v, + ulong off_v, + ulong v_nb1, + ulong v_nb2, + global const uchar * pq, + global const float * pd, + global float * part, // [split][head][n_q][dv] + int dv, + int n_q, + int gqa, + int n_head, + int nblk, + int blk_per_split, + int p_pitch +) { + const int lid = get_local_id(0); + const int d = get_group_id(0)*128 + lid*2; + const int head_kv = get_group_id(1); + const int split = get_group_id(2); + const int ncol = n_q*gqa; + const int b0 = split*blk_per_split; + const int b1 = min(b0 + blk_per_split, nblk); + const int nbp = p_pitch / 32; + + __local float sh_p[32][FA_KQVD_MAXC]; + + float2 acc[FA_KQVD_MAXC]; + #pragma unroll + for (int c = 0; c < FA_KQVD_MAXC; ++c) { + acc[c] = (float2)(0.0f); + } + + global const char * vbase = v + off_v + (size_t)head_kv*v_nb2 + (size_t)d*sizeof(half); + + for (int b = b0; b < b1; ++b) { + barrier(CLK_LOCAL_MEM_FENCE); + // Each lane expands 16 KV entries for one column. + const int c = lid % FA_KQVD_MAXC; + const int r0 = (lid / FA_KQVD_MAXC)*16; + float16 f = (float16)(0.0f); + if (c < ncol) { + const int n = c / gqa; + const int h = head_kv*gqa + c % gqa; + const size_t row = (size_t)h*n_q + n; + const float sc = pd[row*nbp + b]; + const uchar16 u = vload16(0, pq + row*p_pitch + (size_t)b*32 + r0); + f = convert_float16(u)*sc; + } + sh_p[r0+ 0][c] = f.s0; sh_p[r0+ 1][c] = f.s1; sh_p[r0+ 2][c] = f.s2; sh_p[r0+ 3][c] = f.s3; + sh_p[r0+ 4][c] = f.s4; sh_p[r0+ 5][c] = f.s5; sh_p[r0+ 6][c] = f.s6; sh_p[r0+ 7][c] = f.s7; + sh_p[r0+ 8][c] = f.s8; sh_p[r0+ 9][c] = f.s9; sh_p[r0+10][c] = f.sa; sh_p[r0+11][c] = f.sb; + sh_p[r0+12][c] = f.sc; sh_p[r0+13][c] = f.sd; sh_p[r0+14][c] = f.se; sh_p[r0+15][c] = f.sf; + barrier(CLK_LOCAL_MEM_FENCE); + + global const char * vr = vbase + (size_t)b*32*v_nb1; + for (int r = 0; r < 32; ++r) { + const float2 x = vload_half2(0, (global const half *)(vr + (size_t)r*v_nb1)); + #pragma unroll + for (int c = 0; c < FA_KQVD_MAXC; c += 4) { + if (c < ncol) { + const float4 pv = vload4(0, &sh_p[r][c]); + acc[c+0] = mad((float2)(pv.x), x, acc[c+0]); + acc[c+1] = mad((float2)(pv.y), x, acc[c+1]); + acc[c+2] = mad((float2)(pv.z), x, acc[c+2]); + acc[c+3] = mad((float2)(pv.w), x, acc[c+3]); + } + } + } + } + + #pragma unroll + for (int c = 0; c < FA_KQVD_MAXC; ++c) { + if (c < ncol) { + const int n = c / gqa; + const int h = head_kv*gqa + c % gqa; + vstore2(acc[c], 0, part + (((size_t)split*n_head + h)*n_q + n)*dv + d); + } + } +} + +kernel void kernel_fa_kqv_direct_reduce( + global const float * part, + global float * dst, + int n_elem, + int n_split +) { + const int i = get_global_id(0); + if (i >= n_elem) { + return; + } + float s = 0.0f; + for (int k = 0; k < n_split; ++k) { + s += part[(size_t)k*n_elem + i]; + } + dst[i] = s; +} diff --git a/ggml/src/ggml-opencl/kernels/fwht.cl b/ggml/src/ggml-opencl/kernels/fwht.cl new file mode 100644 index 000000000..b1fafc34f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/fwht.cl @@ -0,0 +1,44 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +kernel void kernel_fwht( + global const char * src, + ulong offset_src, + global char * dst, + ulong offset_dst, + int n, + ulong ne1, + ulong ne2, + ulong nb1, + ulong nb2, + ulong nb3, + float scale, + local float * work) { + const ulong row = get_group_id(1); + const ulong i1 = row % ne1; + const ulong i2 = row / ne1 % ne2; + const ulong i3 = row / ne1 / ne2; + global const SRC_TYPE * x = (global const SRC_TYPE *)(src + offset_src + i1*nb1 + i2*nb2 + i3*nb3); + global float * y = (global float *)(dst + offset_dst) + row*n; + const int tid = get_local_id(0); + const int nth = get_local_size(0); + + for (int i = tid; i < n; i += nth) { + work[i] = (float)x[i] * scale; + } + barrier(CLK_LOCAL_MEM_FENCE); + + for (int h = 1; h < n; h *= 2) { + for (int i = tid; i < n/2; i += nth) { + const int j = 2*i - (i & (h - 1)); + const float a = work[j]; + const float b = work[j + h]; + work[j] = a + b; + work[j + h] = a - b; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + for (int i = tid; i < n; i += nth) { + y[i] = work[i]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl new file mode 100644 index 000000000..a35957409 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl @@ -0,0 +1,205 @@ +// Cooperative-K DP4A GEMM for 2 to 4 columns with Q4_0 weights and Q8_1 activations. +// Named vector components avoid dynamic private-array indexing, which can cause register spills. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#ifndef COOPK_NSG +#define COOPK_NSG 4 +#endif +#define COOPK_SG 64 + +#ifndef COOPK_ROWS +#define COOPK_ROWS 4 +#endif +#ifndef COOPK_COLS +#define COOPK_COLS 4 +#endif + +#if COOPK_COLS == 4 +typedef float4 coopk_accv; +typedef int4 coopk_dotv; +#define COOPK_CONVF convert_float4 +#else +typedef float2 coopk_accv; +typedef int2 coopk_dotv; +#define COOPK_CONVF convert_float2 +#endif + +// Pack four Q4_0 weights as signed bytes after subtracting 8. +#define EXP40(l) \ + ( (((uint)((int)(( (l) & 0x000Fu) - 8) & 0xFF))) \ + | (((uint)((int)((((l) & 0x00F0u) >> 4) - 8) & 0xFF)) << 8) \ + | (((uint)((int)((((l) & 0x0F00u) >> 8) - 8) & 0xFF)) << 16) \ + | (((uint)((int)((((l) & 0xF000u) >> 12) - 8) & 0xFF)) << 24) ) + +#if COOPK_ROWS == 4 +#define COOPK_DOT(ci, t) \ + d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \ + d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); \ + d2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, d2.s##ci); \ + d3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, d3.s##ci); +#else +#define COOPK_DOT(ci, t) \ + d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \ + d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); +#endif + +#if COOPK_COLS == 4 +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t) +#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3) +#else +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) +#define COOPK_FOR_COLS(F) F(0) F(1) +#endif + +#if COOPK_ROWS == 4 +#define COOPK_KSTEP(t) \ + { \ + ushort4 bl = vload4(0, src0_q + row0 + (ku0 + t) * m); \ + const uint w0 = EXP40(bl.s0); \ + const uint w1 = EXP40(bl.s1); \ + const uint w2 = EXP40(bl.s2); \ + const uint w3 = EXP40(bl.s3); \ + COOPK_DOTS_AT(t) \ + } +#else +#define COOPK_KSTEP(t) \ + { \ + ushort2 bl = vload2(0, src0_q + row0 + (ku0 + t) * m); \ + const uint w0 = EXP40(bl.s0); \ + const uint w1 = EXP40(bl.s1); \ + COOPK_DOTS_AT(t) \ + } +#endif + +// Weights use the untiled Adreno noshuffle SoA layout, with the output row contiguous in each plane. +// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used. +kernel void kernel_gemm_coopk_q4_0_q8_1_dp4a( + global const ushort * src0_q, // four consecutive K nibbles per ushort, [k/4][m] + global const half * src0_d, // block scales, [k/32][m] + global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4] + global const half * src1_da, // activation scales, [n][k/32] + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + + const int gx = get_global_id(0); + const int sg = get_local_id(1); + const int lane = get_local_id(0); + + const int row0 = gx * COOPK_ROWS; + const int num_32blk = k / 32; + const int k_u = k >> 2; + const int k_b = k >> 5; + + // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding. + const int c0 = 0, c1 = 1; +#if COOPK_COLS == 4 + const int c2 = 2, c3 = 3; +#endif + + coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f); +#endif + + for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) { +#if COOPK_ROWS == 4 + half4 scale = vload4(0, src0_d + row0 + blk * m); +#else + half2 scale = vload2(0, src0_d + row0 + blk * m); +#endif + + coopk_accv da; + da.s0 = (float)src1_da[c0*k_b + blk]; + da.s1 = (float)src1_da[c1*k_b + blk]; +#if COOPK_COLS == 4 + da.s2 = (float)src1_da[c2*k_b + blk]; + da.s3 = (float)src1_da[c3*k_b + blk]; +#endif + + // Q4_0 uses one weight scale per 32 K values. + coopk_dotv d0 = (coopk_dotv)(0), d1 = (coopk_dotv)(0); +#if COOPK_ROWS == 4 + coopk_dotv d2 = (coopk_dotv)(0), d3 = (coopk_dotv)(0); +#endif + for (int half_idx = 0; half_idx < 2; ++half_idx) { + const int ku0 = (blk << 3) + half_idx * 4; + + uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0); + uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0); +#if COOPK_COLS == 4 + uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0); + uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0); +#endif + COOPK_KSTEP(0) + COOPK_KSTEP(1) + COOPK_KSTEP(2) + COOPK_KSTEP(3) + } + + acc0 += (float)scale.s0 * da * COOPK_CONVF(d0); + acc1 += (float)scale.s1 * da * COOPK_CONVF(d1); +#if COOPK_ROWS == 4 + acc2 += (float)scale.s2 * da * COOPK_CONVF(d2); + acc3 += (float)scale.s3 * da * COOPK_CONVF(d3); +#endif + } + + // Reuse one local buffer for all rows; the first barrier protects the previous row's reads. + local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)]; + coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f); +#endif + +#define COOPK_REDUCE(accv, outv) \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg == 0) { \ + coopk_accv sum = (accv); \ + for (int s = 0; s < COOPK_NSG - 1; s++) { \ + sum += reduceLM[s * COOPK_SG + lane]; \ + } \ + (outv) = sum; \ + } + + COOPK_REDUCE(acc0, out0) + COOPK_REDUCE(acc1, out1) +#if COOPK_ROWS == 4 + COOPK_REDUCE(acc2, out2) + COOPK_REDUCE(acc3, out3) +#endif + +#undef COOPK_REDUCE + +#if COOPK_ROWS == 4 +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \ + idx += m; \ + } +#else +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \ + idx += m; \ + } +#endif + + if (sg == 0) { + int idx = row0; + COOPK_FOR_COLS(COOPK_STORE_COL) + } + +#undef COOPK_STORE_COL +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl new file mode 100644 index 000000000..31dce626f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl @@ -0,0 +1,264 @@ +// Cooperative-K DP4A GEMM for 2 to 4 columns with Q4_K weights and Q8_1 activations. +// Named vector components avoid dynamic private-array indexing, which can cause register spills. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 +#define K_SCALE_SIZE 12 + +#ifndef COOPK_NSG +#define COOPK_NSG 4 +#endif +#define COOPK_SG 64 + +#ifndef COOPK_ROWS +#define COOPK_ROWS 4 +#endif +#ifndef COOPK_COLS +#define COOPK_COLS 4 +#endif + +#if COOPK_COLS == 4 +typedef float4 coopk_accv; +typedef int4 coopk_dotv; +#define COOPK_CONVF convert_float4 +#else +typedef float2 coopk_accv; +typedef int2 coopk_dotv; +#define COOPK_CONVF convert_float2 +#endif + +// Expand four Q4_K nibbles into bytes for DP4A. +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +#if COOPK_ROWS == 4 +#define COOPK_DOT(ci, t) \ + s0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, s0.s##ci); \ + s1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, s1.s##ci); \ + s2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, s2.s##ci); \ + s3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, s3.s##ci); +#else +#define COOPK_DOT(ci, t) \ + s0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, s0.s##ci); \ + s1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, s1.s##ci); +#endif + +#if COOPK_COLS == 4 +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t) +#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3) +#else +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) +#define COOPK_FOR_COLS(F) F(0) F(1) +#endif + +// The bin layout packs eight K values in each uint, with four nibbles in each 16-bit half. +// Since ku0 is a multiple of four, t / 2 selects the uint and t & 1 selects its half. +#ifdef COOPK_Q_BIN +#define COOPK_QBITS4(t) convert_ushort4(((t) & 1) ? (vload4(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) >> 16) \ + : (vload4(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) & 0xFFFFu)) +#define COOPK_QBITS2(t) convert_ushort2(((t) & 1) ? (vload2(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) >> 16) \ + : (vload2(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) & 0xFFFFu)) +#else +#define COOPK_QBITS4(t) vload4(0, src0_q + row0 + (ku0 + t) * m) +#define COOPK_QBITS2(t) vload2(0, src0_q + row0 + (ku0 + t) * m) +#endif + +#if COOPK_ROWS == 4 +#define COOPK_KSTEP(t) \ + { \ + ushort4 bits = COOPK_QBITS4(t); \ + const uint w0 = EXP4(bits.s0); \ + const uint w1 = EXP4(bits.s1); \ + const uint w2 = EXP4(bits.s2); \ + const uint w3 = EXP4(bits.s3); \ + COOPK_DOTS_AT(t) \ + } +#else +#define COOPK_KSTEP(t) \ + { \ + ushort2 bits = COOPK_QBITS2(t); \ + const uint w0 = EXP4(bits.s0); \ + const uint w1 = EXP4(bits.s1); \ + COOPK_DOTS_AT(t) \ + } +#endif + +inline void get_scale_min_k4_c(int j, global const uchar * q, int stride, + uchar * d, uchar * m, + uchar mask_d6, uchar mask_d4, uchar mask_hi2) { + if (j < 4) { + *d = q[j*stride] & mask_d6; + *m = q[(j + 4)*stride] & mask_d6; + } else { + *d = (q[(j + 4)*stride] & mask_d4) | ((q[(j - 4)*stride] >> 6) << 4); + *m = (q[(j + 4)*stride] >> 4) | ((q[(j )*stride] >> 6) << 4); + } +} + +// Weights use the untiled Adreno noshuffle SoA layout; COOPK_Q_BIN selects uint-transposed q instead of ushort-transposed q. +// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used. +kernel void kernel_gemm_coopk_q4_k_q8_1_dp4a( +#ifdef COOPK_Q_BIN + global const uint * src0_q, // eight consecutive K nibbles per uint, [k/8][m] +#else + global const ushort * src0_q, // four consecutive K nibbles per ushort, [k/4][m] +#endif + global const uchar * src0_s, // packed 6-bit scales/mins, [k/256][12][m] + global const half * src0_d, // super-block scales, [k/256][m] + global const half * src0_dm, // super-block mins, [k/256][m] + global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4] + global const half * src1_da, // activation scales, [n][k/32] + global const half * src1_sa, // scaled activation sums, [n][k/32] + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const int gx = get_global_id(0); // row group + const int sg = get_local_id(1); // K-split subgroup + const int lane = get_local_id(0); + + const int row0 = gx * COOPK_ROWS; + const int num_32blk = k / 32; + const int k_u = k >> 2; // K in uint (int8x4) units + const int k_b = k >> 5; // 32-blocks along K + + // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding. + const int nl = n - 1; + const int c0 = 0; + const int c1 = (1 < n) ? 1 : nl; +#if COOPK_COLS >= 4 + const int c2 = (2 < n) ? 2 : nl; + const int c3 = (3 < n) ? 3 : nl; +#endif + + coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f); +#endif + + for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) { + const int i = blk << 5; + const int sb_idx = blk >> 3; + const int sub_idx = blk & 7; + + global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * m + row0; + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4_c(sub_idx, sc + 0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4_c(sub_idx, sc + 1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); +#if COOPK_ROWS == 4 + half4 dd = vload4(0, src0_d + row0 + sb_idx * m); + half4 dmm = vload4(0, src0_dm + row0 + sb_idx * m); + uchar sv2, mn2, sv3, mn3; + get_scale_min_k4_c(sub_idx, sc + 2, m, &sv2, &mn2, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4_c(sub_idx, sc + 3, m, &sv3, &mn3, mask_d6, mask_d4, mask_hi2); + const float sc2 = (float)dd.s2 * (float)sv2, mv2 = (float)dmm.s2 * (float)mn2; + const float sc3 = (float)dd.s3 * (float)sv3, mv3 = (float)dmm.s3 * (float)mn3; +#else + half2 dd = vload2(0, src0_d + row0 + sb_idx * m); + half2 dmm = vload2(0, src0_dm + row0 + sb_idx * m); +#endif + const float sc0 = (float)dd.s0 * (float)sv0, mv0 = (float)dmm.s0 * (float)mn0; + const float sc1 = (float)dd.s1 * (float)sv1, mv1 = (float)dmm.s1 * (float)mn1; + + // Q4_K scale and min values change every 32 K values. + coopk_dotv s0 = (coopk_dotv)(0), s1 = (coopk_dotv)(0); +#if COOPK_ROWS == 4 + coopk_dotv s2 = (coopk_dotv)(0), s3 = (coopk_dotv)(0); +#endif + + for (int uq = 0; uq < 2; ++uq) { + const int ku0 = (i >> 2) + uq * 4; + + uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0); + uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0); +#if COOPK_COLS >= 4 + uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0); + uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0); +#endif + COOPK_KSTEP(0) + COOPK_KSTEP(1) + COOPK_KSTEP(2) + COOPK_KSTEP(3) + } + + // Q4_K stores q * scale - min for each 32-value block. + // src1_sa already includes the activation scale, so the min term needs no extra factor of da. + coopk_accv da, sa; + da.s0 = (float)src1_da[c0*k_b + blk]; sa.s0 = (float)src1_sa[c0*k_b + blk]; + da.s1 = (float)src1_da[c1*k_b + blk]; sa.s1 = (float)src1_sa[c1*k_b + blk]; +#if COOPK_COLS >= 4 + da.s2 = (float)src1_da[c2*k_b + blk]; sa.s2 = (float)src1_sa[c2*k_b + blk]; + da.s3 = (float)src1_da[c3*k_b + blk]; sa.s3 = (float)src1_sa[c3*k_b + blk]; +#endif + + acc0 += sc0 * da * COOPK_CONVF(s0) - mv0 * sa; + acc1 += sc1 * da * COOPK_CONVF(s1) - mv1 * sa; +#if COOPK_ROWS == 4 + acc2 += sc2 * da * COOPK_CONVF(s2) - mv2 * sa; + acc3 += sc3 * da * COOPK_CONVF(s3) - mv3 * sa; +#endif + } + + // Reuse one local buffer for all rows; the first barrier protects the previous row's reads. + local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)]; + coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f); +#endif + +#define COOPK_REDUCE(accv, outv) \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg == 0) { \ + coopk_accv sum = (accv); \ + for (int s = 0; s < COOPK_NSG - 1; s++) { \ + sum += reduceLM[s * COOPK_SG + lane]; \ + } \ + (outv) = sum; \ + } + + COOPK_REDUCE(acc0, out0) + COOPK_REDUCE(acc1, out1) +#if COOPK_ROWS == 4 + COOPK_REDUCE(acc2, out2) + COOPK_REDUCE(acc3, out3) +#endif + +#undef COOPK_REDUCE + +#if COOPK_ROWS == 4 +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \ + idx += m; \ + } +#else +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \ + idx += m; \ + } +#endif + + if (sg == 0) { + int idx = row0; + COOPK_FOR_COLS(COOPK_STORE_COL) + } + +#undef COOPK_STORE_COL +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl new file mode 100644 index 000000000..4410a230d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl @@ -0,0 +1,216 @@ +// Cooperative-K DP4A GEMM for 2 to 4 columns with Q6_K weights and Q8_1 activations. +// Named vector components avoid dynamic private-array indexing, which can cause register spills. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#ifndef COOPK_NSG +#define COOPK_NSG 4 +#endif +#define COOPK_SG 64 + +#ifndef COOPK_ROWS +#define COOPK_ROWS 4 +#endif +#ifndef COOPK_COLS +#define COOPK_COLS 4 +#endif + +#if COOPK_COLS == 4 +typedef float4 coopk_accv; +typedef int4 coopk_dotv; +#define COOPK_CONVF convert_float4 +#else +typedef float2 coopk_accv; +typedef int2 coopk_dotv; +#define COOPK_CONVF convert_float2 +#endif + +// Combine low nibbles and high bits, subtract 32, and pack four signed bytes for DP4A. +#define EXP6(l, h, mf, mc) \ + ( (((uint)((int)((( (l) & 0x000Fu) | (((h) & 0x03u) << 4)) - 32) & 0xFF))) \ + | (((uint)((int)(((((l) & 0x00F0u) >> 4) | (((h) & 0x0Cu) << 2)) - 32) & 0xFF)) << 8) \ + | (((uint)((int)(((((l) & 0x0F00u) >> 8) | ((h) & 0x30u)) - 32) & 0xFF)) << 16) \ + | (((uint)((int)(((((l) & (mf)) >> 12) | (((h) & (mc)) >> 2)) - 32) & 0xFF)) << 24) ) + +#if COOPK_ROWS == 4 +#define COOPK_DOT(ci, t) \ + d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \ + d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); \ + d2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, d2.s##ci); \ + d3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, d3.s##ci); +#else +#define COOPK_DOT(ci, t) \ + d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \ + d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); +#endif + +#if COOPK_COLS == 4 +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t) +#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3) +#else +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) +#define COOPK_FOR_COLS(F) F(0) F(1) +#endif + +#if COOPK_ROWS == 4 +#define COOPK_KSTEP(t) \ + { \ + ushort4 bl = vload4(0, src0_ql + row0 + (ku0 + t) * m); \ + uchar4 bh = vload4(0, src0_qh + row0 + (ku0 + t) * m); \ + const uint w0 = EXP6(bl.s0, bh.s0, mask_f000, mask_c0); \ + const uint w1 = EXP6(bl.s1, bh.s1, mask_f000, mask_c0); \ + const uint w2 = EXP6(bl.s2, bh.s2, mask_f000, mask_c0); \ + const uint w3 = EXP6(bl.s3, bh.s3, mask_f000, mask_c0); \ + COOPK_DOTS_AT(t) \ + } +#else +#define COOPK_KSTEP(t) \ + { \ + ushort2 bl = vload2(0, src0_ql + row0 + (ku0 + t) * m); \ + uchar2 bh = vload2(0, src0_qh + row0 + (ku0 + t) * m); \ + const uint w0 = EXP6(bl.s0, bh.s0, mask_f000, mask_c0); \ + const uint w1 = EXP6(bl.s1, bh.s1, mask_f000, mask_c0); \ + COOPK_DOTS_AT(t) \ + } +#endif + +// Weights use the untiled Adreno noshuffle SoA layout, with the output row contiguous in each plane. +// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used. +kernel void kernel_gemm_coopk_q6_k_q8_1_dp4a( + global const ushort * src0_ql, // four consecutive low nibbles per ushort, [k/4][m] + global const uchar * src0_qh, // four consecutive high 2-bit codes per uchar, [k/4][m] + global const ushort * src0_s, // two signed int8 scales per ushort, [k/32][m] + global const half * src0_d, // super-block scales, [k/256][m] + global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4] + global const half * src1_da, // activation scales, [n][k/32] + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + ushort mask_f000, + uchar mask_c0 +) { + dst = (global float *)((global char *)dst + offsetd); + + const int gx = get_global_id(0); + const int sg = get_local_id(1); + const int lane = get_local_id(0); + + const int row0 = gx * COOPK_ROWS; + const int num_32blk = k / 32; + const int k_u = k >> 2; + const int k_b = k >> 5; + + // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding. + const int c0 = 0, c1 = 1; +#if COOPK_COLS == 4 + const int c2 = 2, c3 = 3; +#endif + + coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f); +#endif + + for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) { + const int i = blk << 5; + + // Each ushort packs two signed scale bytes, one for each 16-value half of this 32-value block. + // The half-precision super-block scale is shared by 256 K values. + ushort4 spk = vload4(0, src0_s + row0 + blk * m); + half4 scd = vload4(0, src0_d + row0 + (i >> 8) * m); + + coopk_accv da; + da.s0 = (float)src1_da[c0*k_b + blk]; + da.s1 = (float)src1_da[c1*k_b + blk]; +#if COOPK_COLS == 4 + da.s2 = (float)src1_da[c2*k_b + blk]; + da.s3 = (float)src1_da[c3*k_b + blk]; +#endif + + // Apply each signed weight scale to its 16-value dot product. + for (int half_idx = 0; half_idx < 2; ++half_idx) { + const int ku0 = (i >> 2) + half_idx * 4; + + coopk_dotv d0 = (coopk_dotv)(0), d1 = (coopk_dotv)(0); +#if COOPK_ROWS == 4 + coopk_dotv d2 = (coopk_dotv)(0), d3 = (coopk_dotv)(0); +#endif + uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0); + uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0); +#if COOPK_COLS == 4 + uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0); + uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0); +#endif + COOPK_KSTEP(0) + COOPK_KSTEP(1) + COOPK_KSTEP(2) + COOPK_KSTEP(3) + + const float s0f = (float)(half_idx ? as_char2(spk.s0).s1 : as_char2(spk.s0).s0) * (float)scd.s0; + const float s1f = (float)(half_idx ? as_char2(spk.s1).s1 : as_char2(spk.s1).s0) * (float)scd.s1; + acc0 += s0f * da * COOPK_CONVF(d0); + acc1 += s1f * da * COOPK_CONVF(d1); +#if COOPK_ROWS == 4 + const float s2f = (float)(half_idx ? as_char2(spk.s2).s1 : as_char2(spk.s2).s0) * (float)scd.s2; + const float s3f = (float)(half_idx ? as_char2(spk.s3).s1 : as_char2(spk.s3).s0) * (float)scd.s3; + acc2 += s2f * da * COOPK_CONVF(d2); + acc3 += s3f * da * COOPK_CONVF(d3); +#endif + } + } + + // Reuse one local buffer for all rows; the first barrier protects the previous row's reads. + local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)]; + coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f); +#endif + +#define COOPK_REDUCE(accv, outv) \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg == 0) { \ + coopk_accv sum = (accv); \ + for (int s = 0; s < COOPK_NSG - 1; s++) { \ + sum += reduceLM[s * COOPK_SG + lane]; \ + } \ + (outv) = sum; \ + } + + COOPK_REDUCE(acc0, out0) + COOPK_REDUCE(acc1, out1) +#if COOPK_ROWS == 4 + COOPK_REDUCE(acc2, out2) + COOPK_REDUCE(acc3, out3) +#endif + +#undef COOPK_REDUCE + +#if COOPK_ROWS == 4 +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \ + idx += m; \ + } +#else +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \ + idx += m; \ + } +#endif + + if (sg == 0) { + int idx = row0; + COOPK_FOR_COLS(COOPK_STORE_COL) + } + +#undef COOPK_STORE_COL +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl index d0b191e18..0fdd8d203 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl @@ -100,6 +100,13 @@ inline int dp4a4(uint w0,uint w1,uint w2,uint w3,uint a0,uint a1,uint a2,uint a3 int r=0; r=dot_acc_sat_4x8packed_ss_int(w0,a0,r); r=dot_acc_sat_4x8packed_ss_int(w1,a1,r); r=dot_acc_sat_4x8packed_ss_int(w2,a2,r); r=dot_acc_sat_4x8packed_ss_int(w3,a3,r); return r; } +// Q8_0 and MXFP4 have no min term. Do not multiply zero by an overflowing half activation sum. +#if MOE_QT == 80 || MOE_QT == 82 +#define MOE_MIN_TERM(t) 0.0f +#else +#define MOE_MIN_TERM(t) (mn*(float)sh_s[t]) +#endif + // One token's two-half dp4a + uniform scale/min epilogue into acc[t]. #define MOE_DP4A_T(t) do { \ uint4 a0 = vload4(0, &sh_qa[t][0]); \ @@ -107,7 +114,7 @@ inline int dp4a4(uint w0,uint w1,uint w2,uint w3,uint a0,uint a1,uint a2,uint a3 const int raw1 = dp4a4(qw[0],qw[1],qw[2],qw[3], a0.s0,a0.s1,a0.s2,a0.s3); \ const int raw2 = dp4a4(qw[4],qw[5],qw[6],qw[7], a1.s0,a1.s1,a1.s2,a1.s3); \ const float a_d = (float)sh_d[t]; \ - acc[t] += sc0*a_d*(float)raw1 + sc1*a_d*(float)raw2 - mn*(float)sh_s[t]; \ + acc[t] += sc0*a_d*(float)raw1 + sc1*a_d*(float)raw2 - MOE_MIN_TERM(t); \ } while (0) __attribute__((qcom_wave_pair_mode(1))) diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_f32.cl new file mode 100644 index 000000000..cc9e3d45a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_f32.cl @@ -0,0 +1,93 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq1_m_f32( + global const uchar * src0_qs, + global const uchar * src0_qh, + global const ushort * src0_sc, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uint4 w0 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 0) * m)); + const uint4 w1 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 1) * m)); + const uint4 w2 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 2) * m)); + const uint4 w3 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 3) * m)); + const ushort4 bits = convert_ushort4((w0 >> (uint4)(12)) | ((w1 >> (uint4)(8)) & (uint4)(0x00f0)) + | ((w2 >> (uint4)(4)) & (uint4)(0x0f00)) | (w3 & (uint4)(0xf000))); + const half4 d = as_half4(bits); + const uint4 sc = convert_uint4(vload4(0, src0_sc + row + (i / 64) * m)); + const half4 scale = d * convert_half4(2*((sc >> (uint4)(3*((i / 16) & 3))) & (uint4)(7)) + 1); + const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (i / 16) * m)); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m)) + | (((qh >> (uint4)(4*((ki / 8) & 1))) & (uint4)(7)) << (uint4)(8)); + const uint4 grid = (uint4)(iq1_s_grid[gi.s0], iq1_s_grid[gi.s1], iq1_s_grid[gi.s2], iq1_s_grid[gi.s3]); + const float4 delta = select((float4)(-0.875f), (float4)(-1.125f), (qh & (uint4)(0x08 << (4*((ki / 8) & 1)))) != 0); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const float4 magnitude = convert_float4((grid >> (uint4)(8*j + (ki & 4))) & (uint4)(15)); + const half4 w = convert_half4(magnitude + delta) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_q8_1_dp4a.cl new file mode 100644 index 000000000..fb5d58a88 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_q8_1_dp4a.cl @@ -0,0 +1,144 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq1_m_pack(uint g, uint shift, uint negative_delta) { + // 8*(grid + delta) is an exact signed integer in [-9, 9]. + const int bias = negative_delta ? -9 : -7; + const int4 values = 8*convert_int4(as_uchar4((g >> shift) & 0x0F0F0F0Fu)) + bias; + return as_uint(convert_char4(values)); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq1_m_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_qh, + __global const ushort * src0_sc, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define IQ1_M_GRID(i) iq1_s_grid[(i)] + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + float dsuper = 0.f; + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; + + if ((sub & 7u) == 0u) { + const uint sb = rrow + 4u*ib*(uint)m; + const uint w0 = src0_sc[sb + 0u*(uint)m]; + const uint w1 = src0_sc[sb + 1u*(uint)m]; + const uint w2 = src0_sc[sb + 2u*(uint)m]; + const uint w3 = src0_sc[sb + 3u*(uint)m]; + const ushort bits = (ushort)((w0 >> 12) | ((w1 >> 8) & 0x00f0u) + | ((w2 >> 4) & 0x0f00u) | (w3 & 0xf000u)); + dsuper = (float)as_half(bits); + } + const uint sc = src0_sc[rrow + (4u*ib + (sub & 7u)/2u)*(uint)m]; + const uint sh = 6u*(sub & 1u); + const float dl0 = dsuper * (float)(2u*((sc >> sh) & 7u) + 1u) * 0.125f; + const float dl1 = dsuper * (float)(2u*((sc >> (sh + 3u)) & 7u) + 1u) * 0.125f; + const uint h0 = src0_qh[rrow + (2u*sub + 0u)*(uint)m]; + const uint h1 = src0_qh[rrow + (2u*sub + 1u)*(uint)m]; + const uint gb = rrow + 4u*sub*(uint)m; + const uint g0 = IQ1_M_GRID((uint)src0_qs[gb + 0u*(uint)m] | (( h0 & 7u) << 8)); + const uint g1 = IQ1_M_GRID((uint)src0_qs[gb + 1u*(uint)m] | (((h0 >> 4) & 7u) << 8)); + const uint g2 = IQ1_M_GRID((uint)src0_qs[gb + 2u*(uint)m] | (( h1 & 7u) << 8)); + const uint g3 = IQ1_M_GRID((uint)src0_qs[gb + 3u*(uint)m] | (((h1 >> 4) & 7u) << 8)); + const uint4 qlo = (uint4)(iq1_m_pack(g0, 0, h0 & 0x08u), iq1_m_pack(g0, 4, h0 & 0x08u), + iq1_m_pack(g1, 0, h0 & 0x80u), iq1_m_pack(g1, 4, h0 & 0x80u)); + const uint4 qhi = (uint4)(iq1_m_pack(g2, 0, h1 & 0x08u), iq1_m_pack(g2, 4, h1 & 0x08u), + iq1_m_pack(g3, 0, h1 & 0x80u), iq1_m_pack(g3, 4, h1 & 0x80u)); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) +#define IQ1_M_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \ + + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1]))) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = IQ1_M_COL(b+0); rf.s1 = IQ1_M_COL(b+1); + rf.s2 = IQ1_M_COL(b+2); rf.s3 = IQ1_M_COL(b+3); + acc[g] += LD4(sh_d, b) * rf; + } +#undef IQ1_M_COL +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +#undef IQ1_M_GRID +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_f32.cl new file mode 100644 index 000000000..55fdba620 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_f32.cl @@ -0,0 +1,86 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq1_s_f32( + global const uchar * src0_qs, + global const ushort * src0_qh, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (i / 32) * m)); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const half4 scale = d * convert_half4(2*((qh >> (uint4)(12)) & (uint4)(7)) + 1); + const float4 delta = select((float4)(-0.875f), (float4)(-1.125f), (qh & (uint4)(0x8000)) != 0); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m)) + | (((qh >> (uint4)(3*((ki / 8) & 3))) & (uint4)(7)) << (uint4)(8)); + const uint4 grid = (uint4)(iq1_s_grid[gi.s0], iq1_s_grid[gi.s1], iq1_s_grid[gi.s2], iq1_s_grid[gi.s3]); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const float4 magnitude = convert_float4((grid >> (uint4)(8*j + (ki & 4))) & (uint4)(15)); + const half4 w = convert_half4(magnitude + delta) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_q8_1_dp4a.cl new file mode 100644 index 000000000..92acb89b9 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_q8_1_dp4a.cl @@ -0,0 +1,138 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq1_s_q8_1_dp4a( + __global const uchar * src0_qs, + __global const ushort * src0_qh, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_da[TILESIZE_N]; + __local half sh_sa[TILESIZE_N]; + +#define IQ1_S_GRID(i) iq1_s_grid[(i)] + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; + + const uint qhv = (uint)src0_qh[rrow + sub * (uint)m]; + const float dl = (float)src0_d[rrow + ib * (uint)m] + * (float)(2u * ((qhv >> 12) & 7u) + 1u); + + // The codebook stores grid + 1; sa includes the Q8_1 activation scale. + const float dlt = ((qhv & 0x8000u) ? -0.125f : 0.125f) - 1.0f; + + const uint gb = rrow + (step >> 3) * (uint)m; + + uint8 qw; + { + const uint g0 = IQ1_S_GRID((uint)src0_qs[gb + 0u * (uint)m] | ((( qhv & 7u) << 8))); + const uint g1 = IQ1_S_GRID((uint)src0_qs[gb + 1u * (uint)m] | ((((qhv >> 3) & 7u) << 8))); + const uint g2 = IQ1_S_GRID((uint)src0_qs[gb + 2u * (uint)m] | ((((qhv >> 6) & 7u) << 8))); + const uint g3 = IQ1_S_GRID((uint)src0_qs[gb + 3u * (uint)m] | ((((qhv >> 9) & 7u) << 8))); + qw.s0 = g0 & 0x0F0F0F0Fu; qw.s1 = (g0 >> 4) & 0x0F0F0F0Fu; + qw.s2 = g1 & 0x0F0F0F0Fu; qw.s3 = (g1 >> 4) & 0x0F0F0F0Fu; + qw.s4 = g2 & 0x0F0F0F0Fu; qw.s5 = (g2 >> 4) & 0x0F0F0F0Fu; + qw.s6 = g3 & 0x0F0F0F0Fu; qw.s7 = (g3 >> 4) & 0x0F0F0F0Fu; + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + const bool ok = c < (uint)n_no_padding; + sh_da[lid] = ok ? src1_da[c * k_b + sub] : (half)0; + sh_sa[lid] = ok ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += dl * (LD4(sh_da, b) * rf) + (dl * dlt) * LD4(sh_sa, b); + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +#undef IQ1_S_GRID +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_f32.cl new file mode 100644 index 000000000..757a5ebd1 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_f32.cl @@ -0,0 +1,93 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq2_s_f32( + global const uchar * src0_qs, + global const uchar * src0_sg, + global const uchar * src0_qh, + global const uchar * src0_sc, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uchar4 sc = vload4(0, src0_sc + row + (i / 32) * m); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4((sc >> (uchar4)(4*((i / 16) & 1))) & (uchar4)(15))); + const half4 scale = convert_half4(scale_f); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m)); + const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (ki / 32) * m)); + const uchar4 sg = vload4(0, src0_sg + row + (ki / 8) * m); + const uint4 gi = qs | (((qh >> (uint4)(2*((ki / 8) & 3))) & (uint4)(3)) << (uint4)(8)); + const uint half_grid = (ki & 7) / 4; + const uint4 grid = (uint4)(iq2_s_grid[2*gi.s0 + half_grid], iq2_s_grid[2*gi.s1 + half_grid], + iq2_s_grid[2*gi.s2 + half_grid], iq2_s_grid[2*gi.s3 + half_grid]); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255)); + const int4 signs = convert_int4((sg >> (uchar4)((ki & 7) + j)) & (uchar4)(1)); + const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_q8_1_dp4a.cl new file mode 100644 index 000000000..a8f1bf9d0 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_q8_1_dp4a.cl @@ -0,0 +1,149 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq2_s_pack(uint gv, uint sg, uint base) { + int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; } + int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; } + int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; } + int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; } + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq2_s_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_sg, + __global const uchar * src0_qh, + __global const uchar * src0_sc, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define IQ2_S_GRID(i) iq2_s_grid[(i)] + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; + + const uint qhv = (uint)src0_qh[rrow + sub * (uint)m]; + const uint scv = (uint)src0_sc[rrow + sub * (uint)m]; + const float dv = (float)src0_d[rrow + ib * (uint)m]; + const float dl0 = dv * (0.5f + (float)( scv & 0xFu)) * 0.25f; + const float dl1 = dv * (0.5f + (float)( scv >> 4) ) * 0.25f; + + const uint gb = rrow + (step >> 3) * (uint)m; + + uint4 qlo, qhi; + { + const uint gi0 = (uint)src0_qs[gb + 0u * (uint)m] | ((( qhv & 3u) << 8)); + const uint gi1 = (uint)src0_qs[gb + 1u * (uint)m] | ((((qhv >> 2) & 3u) << 8)); + const uint gi2 = (uint)src0_qs[gb + 2u * (uint)m] | ((((qhv >> 4) & 3u) << 8)); + const uint gi3 = (uint)src0_qs[gb + 3u * (uint)m] | ((((qhv >> 6) & 3u) << 8)); + const uint s0 = (uint)src0_sg[gb + 0u * (uint)m]; + const uint s1 = (uint)src0_sg[gb + 1u * (uint)m]; + const uint s2 = (uint)src0_sg[gb + 2u * (uint)m]; + const uint s3 = (uint)src0_sg[gb + 3u * (uint)m]; + qlo.s0 = iq2_s_pack(IQ2_S_GRID(2u*gi0 + 0u), s0, 0u); + qlo.s1 = iq2_s_pack(IQ2_S_GRID(2u*gi0 + 1u), s0, 4u); + qlo.s2 = iq2_s_pack(IQ2_S_GRID(2u*gi1 + 0u), s1, 0u); + qlo.s3 = iq2_s_pack(IQ2_S_GRID(2u*gi1 + 1u), s1, 4u); + qhi.s0 = iq2_s_pack(IQ2_S_GRID(2u*gi2 + 0u), s2, 0u); + qhi.s1 = iq2_s_pack(IQ2_S_GRID(2u*gi2 + 1u), s2, 4u); + qhi.s2 = iq2_s_pack(IQ2_S_GRID(2u*gi3 + 0u), s3, 0u); + qhi.s3 = iq2_s_pack(IQ2_S_GRID(2u*gi3 + 1u), s3, 4u); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) +#define IQ2_S_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \ + + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1]))) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = IQ2_S_COL(b+0); rf.s1 = IQ2_S_COL(b+1); + rf.s2 = IQ2_S_COL(b+2); rf.s3 = IQ2_S_COL(b+3); + acc[g] += LD4(sh_d, b) * rf; + } +#undef IQ2_S_COL +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +#undef IQ2_S_GRID +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_f32.cl new file mode 100644 index 000000000..f4a075063 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_f32.cl @@ -0,0 +1,91 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq2_xs_f32( + global const ushort * src0_qs, + global const uchar * src0_sc, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uchar4 sc = vload4(0, src0_sc + row + (i / 32) * m); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4((sc >> (uchar4)(4*((i / 16) & 1))) & (uchar4)(15))); + const half4 scale = convert_half4(scale_f); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m)); + const uint4 code = qs >> (uint4)(9); + const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7)); + const uint4 gi = qs & (uint4)(511); + const uint half_grid = (ki & 7) / 4; + const uint4 grid = (uint4)(iq2_xs_grid[2*gi.s0 + half_grid], iq2_xs_grid[2*gi.s1 + half_grid], + iq2_xs_grid[2*gi.s2 + half_grid], iq2_xs_grid[2*gi.s3 + half_grid]); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255)); + const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1)); + const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_q8_1_dp4a.cl new file mode 100644 index 000000000..c322803b9 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_q8_1_dp4a.cl @@ -0,0 +1,150 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq2_xs_pack(uint gv, uint sg, uint base) { + int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; } + int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; } + int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; } + int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; } + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a( + __global const ushort * src0_qs, + __global const uchar * src0_sc, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define IQ2_XS_GRID(i) iq2_xs_grid[(i)] + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; + + const uint scv = (uint)src0_sc[rrow + sub * (uint)m]; + const float dv = (float)src0_d[rrow + ib * (uint)m]; + const float dl0 = dv * (0.5f + (float)( scv & 0xFu)) * 0.25f; + const float dl1 = dv * (0.5f + (float)( scv >> 4) ) * 0.25f; + + const uint gb = rrow + (step >> 3) * (uint)m; + + uint4 qlo, qhi; + { + const uint q0 = (uint)src0_qs[gb + 0u * (uint)m]; + const uint q1 = (uint)src0_qs[gb + 1u * (uint)m]; + const uint q2 = (uint)src0_qs[gb + 2u * (uint)m]; + const uint q3 = (uint)src0_qs[gb + 3u * (uint)m]; + const uint gi0 = (q0 & 511u) << 1; + const uint gi1 = (q1 & 511u) << 1; + const uint gi2 = (q2 & 511u) << 1; + const uint gi3 = (q3 & 511u) << 1; + const uint s0 = iq2_xs_signs(q0 >> 9); + const uint s1 = iq2_xs_signs(q1 >> 9); + const uint s2 = iq2_xs_signs(q2 >> 9); + const uint s3 = iq2_xs_signs(q3 >> 9); + qlo.s0 = iq2_xs_pack(IQ2_XS_GRID(gi0 + 0u), s0, 0u); + qlo.s1 = iq2_xs_pack(IQ2_XS_GRID(gi0 + 1u), s0, 4u); + qlo.s2 = iq2_xs_pack(IQ2_XS_GRID(gi1 + 0u), s1, 0u); + qlo.s3 = iq2_xs_pack(IQ2_XS_GRID(gi1 + 1u), s1, 4u); + qhi.s0 = iq2_xs_pack(IQ2_XS_GRID(gi2 + 0u), s2, 0u); + qhi.s1 = iq2_xs_pack(IQ2_XS_GRID(gi2 + 1u), s2, 4u); + qhi.s2 = iq2_xs_pack(IQ2_XS_GRID(gi3 + 0u), s3, 0u); + qhi.s3 = iq2_xs_pack(IQ2_XS_GRID(gi3 + 1u), s3, 4u); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) +#define IQ2_XS_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \ + + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1]))) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = IQ2_XS_COL(b+0); rf.s1 = IQ2_XS_COL(b+1); + rf.s2 = IQ2_XS_COL(b+2); rf.s3 = IQ2_XS_COL(b+3); + acc[g] += LD4(sh_d, b) * rf; + } +#undef IQ2_XS_COL +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +#undef IQ2_XS_GRID +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_f32.cl new file mode 100644 index 000000000..5d6cc0dea --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_f32.cl @@ -0,0 +1,90 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq2_xxs_f32( + global const uchar * src0_qs, + global const uint * src0_sas, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uint4 sas = vload4(0, src0_sas + row + (i / 32) * m); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4(sas >> (uint4)(28))); + const half4 scale = convert_half4(scale_f); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m)); + const uint4 code = (sas >> (uint4)(7*((ki / 8) & 3))) & (uint4)(127); + const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7)); + const uint half_grid = (ki & 7) / 4; + const uint4 grid = (uint4)(iq2_xxs_grid[2*gi.s0 + half_grid], iq2_xxs_grid[2*gi.s1 + half_grid], + iq2_xxs_grid[2*gi.s2 + half_grid], iq2_xxs_grid[2*gi.s3 + half_grid]); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255)); + const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1)); + const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl new file mode 100644 index 000000000..60cd268f3 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl @@ -0,0 +1,145 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq2_xxs_pack(uint gv, uint sg, uint base) { + int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; } + int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; } + int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; } + int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; } + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uint * src0_sas, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define IQ2_XXS_GRID(i) iq2_xxs_grid[(i)] + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; + + const uint aux = src0_sas[rrow + sub * (uint)m]; + const float dv = (float)src0_d[rrow + ib * (uint)m]; + const float dl = dv * (0.5f + (float)(aux >> 28)) * 0.25f; + + const uint gb = rrow + (step >> 3) * (uint)m; + + uint4 qlo, qhi; + { + const uint gi0 = (uint)src0_qs[gb + 0u * (uint)m] << 1; + const uint gi1 = (uint)src0_qs[gb + 1u * (uint)m] << 1; + const uint gi2 = (uint)src0_qs[gb + 2u * (uint)m] << 1; + const uint gi3 = (uint)src0_qs[gb + 3u * (uint)m] << 1; + const uint s0 = iq2_xxs_signs((aux >> 0) & 127u); + const uint s1 = iq2_xxs_signs((aux >> 7) & 127u); + const uint s2 = iq2_xxs_signs((aux >> 14) & 127u); + const uint s3 = iq2_xxs_signs((aux >> 21) & 127u); + qlo.s0 = iq2_xxs_pack(IQ2_XXS_GRID(gi0 + 0u), s0, 0u); + qlo.s1 = iq2_xxs_pack(IQ2_XXS_GRID(gi0 + 1u), s0, 4u); + qlo.s2 = iq2_xxs_pack(IQ2_XXS_GRID(gi1 + 0u), s1, 0u); + qlo.s3 = iq2_xxs_pack(IQ2_XXS_GRID(gi1 + 1u), s1, 4u); + qhi.s0 = iq2_xxs_pack(IQ2_XXS_GRID(gi2 + 0u), s2, 0u); + qhi.s1 = iq2_xxs_pack(IQ2_XXS_GRID(gi2 + 1u), s2, 4u); + qhi.s2 = iq2_xxs_pack(IQ2_XXS_GRID(gi3 + 0u), s3, 0u); + qhi.s3 = iq2_xxs_pack(IQ2_XXS_GRID(gi3 + 1u), s3, 4u); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) +#define IQ2_XXS_COL(b) (dl * (float)(dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \ + + dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])))) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = IQ2_XXS_COL(b+0); rf.s1 = IQ2_XXS_COL(b+1); + rf.s2 = IQ2_XXS_COL(b+2); rf.s3 = IQ2_XXS_COL(b+3); + acc[g] += LD4(sh_d, b) * rf; + } +#undef IQ2_XXS_COL +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +#undef IQ2_XXS_GRID +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl new file mode 100644 index 000000000..1a15ff86c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl @@ -0,0 +1,92 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq3_s_f32( + global const uchar * src0_qs, + global const uchar * src0_sg, + global const uchar * src0_qh, + global const uchar * src0_sc, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uchar4 sc = vload4(0, src0_sc + row + (i / 64) * m); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const float4 scale_f = convert_float4(d) * (1.0f + 2.0f*convert_float4((sc >> (uchar4)(4*((i / 32) & 1))) & (uchar4)(15))); + const half4 scale = convert_half4(scale_f); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 4) * m)); + const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (ki / 32) * m)); + const uchar4 sg = vload4(0, src0_sg + row + (ki / 8) * m); + const uint4 gi = qs | (((qh >> (uint4)(((ki / 4) & 7))) & (uint4)(1)) << (uint4)(8)); + const uint4 grid = (uint4)(iq3_s_grid[gi.s0], iq3_s_grid[gi.s1], + iq3_s_grid[gi.s2], iq3_s_grid[gi.s3]); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255)); + const int4 signs = convert_int4((sg >> (uchar4)((ki & 7) + j)) & (uchar4)(1)); + const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl new file mode 100644 index 000000000..43cb38f72 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl @@ -0,0 +1,154 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq3_s_pack(uint gv, uint sg, uint base) { + int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; } + int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; } + int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; } + int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; } + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +#define IQ3_S_GEMM_GRID(i) iq3_s_grid[(i)] + +inline uint iq3_s_load(__global const uchar * qs, uint qsb, uint qhb, uint m, + uint u, uint sgv, uint base) { + const uint gidx = (uint)qs[qsb + u * m] | (((qhb >> u) & 1u) << 8); + return iq3_s_pack(IQ3_S_GEMM_GRID(gidx), sgv, base); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq3_s_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_sg, + __global const uchar * src0_qh, + __global const uchar * src0_sc, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const uint ib = sub >> 3; + const uint sb = sub & 7u; + + const uint scb = (uint)src0_sc[rrow + (ib*(QK_K/64) + (sb >> 1)) * (uint)m]; + const uint nib = (sb & 1u) ? (scb >> 4) : (scb & 0xFu); + const float d_w = (float)src0_d[rrow + ib * (uint)m] * (float)(1 + 2*nib); + + const uint qhb = (uint)src0_qh[rrow + (ib*(QK_K/32) + sb) * (uint)m]; + const uint qsb = rrow + (ib*(QK_K/4) + 8*sb) * (uint)m; + const uint sgb = rrow + (ib*(QK_K/8) + 4*sb) * (uint)m; + + const uint sg0 = (uint)src0_sg[sgb + 0 * (uint)m]; + const uint sg1 = (uint)src0_sg[sgb + 1 * (uint)m]; + const uint sg2 = (uint)src0_sg[sgb + 2 * (uint)m]; + const uint sg3 = (uint)src0_sg[sgb + 3 * (uint)m]; + + uint8 qw; + qw.s0 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 0u, sg0, 0u); + qw.s1 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 1u, sg0, 4u); + qw.s2 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 2u, sg1, 0u); + qw.s3 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 3u, sg1, 4u); + qw.s4 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 4u, sg2, 0u); + qw.s5 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 5u, sg2, 4u); + qw.s6 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 6u, sg3, 0u); + qw.s7 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 7u, sg3, 4u); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl new file mode 100644 index 000000000..af58e850c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl @@ -0,0 +1,89 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq3_xxs_f32( + global const uchar * src0_qs, + global const uint * src0_sas, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uint4 sas = vload4(0, src0_sas + row + (i / 32) * m); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const float4 scale_f = convert_float4(d) * 0.5f * (0.5f + convert_float4(sas >> (uint4)(28))); + const half4 scale = convert_half4(scale_f); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 4) * m)); + const uint4 code = (sas >> (uint4)(7*((ki / 8) & 3))) & (uint4)(127); + const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7)); + const uint4 grid = (uint4)(iq3_xxs_grid[gi.s0], iq3_xxs_grid[gi.s1], + iq3_xxs_grid[gi.s2], iq3_xxs_grid[gi.s3]); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255)); + const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1)); + const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl new file mode 100644 index 000000000..2ade6e2cf --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl @@ -0,0 +1,137 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq3_xxs_pack(uint gv, uint sg, uint base) { + int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; } + int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; } + int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; } + int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; } + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uint * src0_sas, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const uint ib = sub >> 3; + const uint aux = src0_sas[rrow + sub*(uint)m]; + const float d_w = (float)src0_d[rrow + ib*(uint)m] * (0.5f + (float)(aux >> 28)) * 0.5f; + const uint sg0 = iq3_xxs_signs( aux & 127u); + const uint sg1 = iq3_xxs_signs((aux >> 7) & 127u); + const uint sg2 = iq3_xxs_signs((aux >> 14) & 127u); + const uint sg3 = iq3_xxs_signs((aux >> 21) & 127u); + const uint qsb = rrow + sub*8u*(uint)m; + + uint8 qw; + qw.s0 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 0u*(uint)m]], sg0, 0u); + qw.s1 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 1u*(uint)m]], sg0, 4u); + qw.s2 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 2u*(uint)m]], sg1, 0u); + qw.s3 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 3u*(uint)m]], sg1, 4u); + qw.s4 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 4u*(uint)m]], sg2, 0u); + qw.s5 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 5u*(uint)m]], sg2, 4u); + qw.s6 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 6u*(uint)m]], sg3, 0u); + qw.s7 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 7u*(uint)m]], sg3, 4u); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl new file mode 100644 index 000000000..284868008 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl @@ -0,0 +1,88 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq4_xs_f32( + global const ushort * src0_qs, + global const uint * src0_sl, + global const ushort * src0_sh, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uint4 sl = vload4(0, src0_sl + row + (i / 256) * m); + const uint4 sh = convert_uint4(vload4(0, src0_sh + row + (i / 256) * m)); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const int sb = (i / 32) & 7; + const int4 ls = convert_int4(((sl >> (uint4)(4*sb)) & (uint4)(15)) | (((sh >> (uint4)(2*sb)) & (uint4)(3)) << (uint4)(4))) - (int4)(32); + const half4 scale = convert_half4(convert_float4(d) * convert_float4(ls)); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const ushort4 q = vload4(0, src0_qs + row + (ki / 4) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const ushort4 indices = (q >> (ushort4)(4*j)) & (ushort4)(15); + const half4 w = (half4)(iq4_xs_values[indices.s0], iq4_xs_values[indices.s1], + iq4_xs_values[indices.s2], iq4_xs_values[indices.s3]) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl new file mode 100644 index 000000000..f23fb1dd5 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl @@ -0,0 +1,135 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq4_xs_pack(ushort q) { + return iq4_xs_code(q & 15) + | (iq4_xs_code((q >> 4) & 15) << 8) + | (iq4_xs_code((q >> 8) & 15) << 16) + | (iq4_xs_code(q >> 12) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a( + __global const ushort * src0_qs, + __global const uint * src0_sl, + __global const ushort * src0_sh, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const uint ib = sub >> 3; + const uint sb = sub & 7u; + const uint sl = src0_sl[rrow + ib*(uint)m]; + const uint sh = src0_sh[rrow + ib*(uint)m]; + const int ls = (int)(((sl >> (4*sb)) & 15u) | (((sh >> (2*sb)) & 3u) << 4)) - 32; + const float d_w = (float)src0_d[rrow + ib*(uint)m] * (float)ls; + const uint qsb = rrow + sub*8u*(uint)m; + + uint8 qw; + qw.s0 = iq4_xs_pack(src0_qs[qsb + 0u*(uint)m]); + qw.s1 = iq4_xs_pack(src0_qs[qsb + 1u*(uint)m]); + qw.s2 = iq4_xs_pack(src0_qs[qsb + 2u*(uint)m]); + qw.s3 = iq4_xs_pack(src0_qs[qsb + 3u*(uint)m]); + qw.s4 = iq4_xs_pack(src0_qs[qsb + 4u*(uint)m]); + qw.s5 = iq4_xs_pack(src0_qs[qsb + 5u*(uint)m]); + qw.s6 = iq4_xs_pack(src0_qs[qsb + 6u*(uint)m]); + qw.s7 = iq4_xs_pack(src0_qs[qsb + 7u*(uint)m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_0_f32.cl new file mode 100644 index 000000000..b7885841f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_0_f32.cl @@ -0,0 +1,80 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_q2_0_f32( + global const uchar * src0_qs, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + float8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const float4 scale = convert_float4(vload4(0, src0_d + row + (i / 64) * m)); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uchar4 q = vload4(0, src0_qs + row + (ki / 4) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + float8 b; + b.s0123 = convert_float4(read_imageh(src1, gy * 2 + (ki + j) * n_4)); + b.s4567 = convert_float4(read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4)); + const int4 v = convert_int4((q >> (uchar4)(2*j)) & (uchar4)(3)) - (int4)(1); + const float4 w = convert_float4(v) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_0_q8_1_dp4a.cl new file mode 100644 index 000000000..d99e166f1 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_0_q8_1_dp4a.cl @@ -0,0 +1,127 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint q2_0_pack(uchar q) { + return (((uint)(q & 3) - 1u) & 255u) + | (((((uint)q >> 2) & 3u) - 1u) & 255u) << 8 + | (((((uint)q >> 4) & 3u) - 1u) & 255u) << 16 + | (((((uint)q >> 6) & 3u) - 1u) & 255u) << 24; +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q2_0_q8_1_dp4a( + __global const uchar * src0_qs, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const uint ib = sub >> 1; + const float d_w = (float)src0_d[rrow + ib*(uint)m]; + const uint qsb = rrow + sub*8u*(uint)m; + + uint8 qw; + qw.s0 = q2_0_pack(src0_qs[qsb + 0u*(uint)m]); + qw.s1 = q2_0_pack(src0_qs[qsb + 1u*(uint)m]); + qw.s2 = q2_0_pack(src0_qs[qsb + 2u*(uint)m]); + qw.s3 = q2_0_pack(src0_qs[qsb + 3u*(uint)m]); + qw.s4 = q2_0_pack(src0_qs[qsb + 4u*(uint)m]); + qw.s5 = q2_0_pack(src0_qs[qsb + 5u*(uint)m]); + qw.s6 = q2_0_pack(src0_qs[qsb + 6u*(uint)m]); + qw.s7 = q2_0_pack(src0_qs[qsb + 7u*(uint)m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl new file mode 100644 index 000000000..2749e1c14 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl @@ -0,0 +1,83 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_q2_k_f32( + global const uchar * src0_qs, + global const uchar * src0_sc, + global const half * src0_dm, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const half8 dm = vload8(0, src0_dm + 2 * (row + (i / 256) * m)); + const uchar4 sc = vload4(0, src0_sc + row + (i / 16) * m); + const half4 scale = convert_half4(convert_float4(dm.s0246) * convert_float4(sc & (uchar4)(15))); + const half4 mval = convert_half4(convert_float4(dm.s1357) * convert_float4(sc >> (uchar4)(4))); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const half4 w = convert_half4((bits >> (uchar4)(2 * j)) & (uchar4)(3)) * scale - mval; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl new file mode 100644 index 000000000..7db7e6d71 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl @@ -0,0 +1,146 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint q2_k_pack(uint pk) { + return ( (pk ) & 3u) + | (((pk >> 2) & 3u) << 8) + | (((pk >> 4) & 3u) << 16) + | (((pk >> 6) & 3u) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +#define KQ_STAGE(p) vload4(0, (p)) + +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q2_k_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_sc, + __global const half * src0_dm, // half pairs: d, dmin + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local float sh_s [TILESIZE_N][2]; // per-16 activation sums, in qa units + __local half sh_d [TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; + + const half2 dm = vload2(rrow + ib * (uint)m, src0_dm); + const float dv = (float)dm.s0; + const float dmv = (float)dm.s1; + + const uint scb = rrow + (sub << 1) * (uint)m; + const uint sc0 = (uint)src0_sc[scb + 0u * (uint)m]; + const uint sc1 = (uint)src0_sc[scb + 1u * (uint)m]; + const float dl0 = dv * (float)( sc0 & 0xFu); + const float ml0 = dmv * (float)( sc0 >> 4); + const float dl1 = dv * (float)( sc1 & 0xFu); + const float ml1 = dmv * (float)( sc1 >> 4); + + const uint qsb = rrow + (step >> 2) * (uint)m; + uint4 qlo, qhi; + qlo.s0 = q2_k_pack((uint)src0_qs[qsb + 0u * (uint)m]); + qlo.s1 = q2_k_pack((uint)src0_qs[qsb + 1u * (uint)m]); + qlo.s2 = q2_k_pack((uint)src0_qs[qsb + 2u * (uint)m]); + qlo.s3 = q2_k_pack((uint)src0_qs[qsb + 3u * (uint)m]); + qhi.s0 = q2_k_pack((uint)src0_qs[qsb + 4u * (uint)m]); + qhi.s1 = q2_k_pack((uint)src0_qs[qsb + 5u * (uint)m]); + qhi.s2 = q2_k_pack((uint)src0_qs[qsb + 6u * (uint)m]); + qhi.s3 = q2_k_pack((uint)src0_qs[qsb + 7u * (uint)m]); + + for (uint idx = lid; idx < TILESIZE_N * 2u; idx += 64u) { + const uint t = idx >> 1; + const uint h = idx & 1u; + const uint c = col_base + t; + const bool ok = c < (uint)n_no_padding; + const uint4 w = ok ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (h << 2)) + : (uint4)(0u); + sh_qa4[t][h] = w; + int s = 0; + s = dot_acc_sat_4x8packed_ss_int(w.x, 0x01010101u, s); + s = dot_acc_sat_4x8packed_ss_int(w.y, 0x01010101u, s); + s = dot_acc_sat_4x8packed_ss_int(w.z, 0x01010101u, s); + s = dot_acc_sat_4x8packed_ss_int(w.w, 0x01010101u, s); + sh_s[t][h] = (float)s; + if (h == 0u) { + sh_d[t] = ok ? src1_da[c * k_b + sub] : (half)0; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) +#define Q2_K_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) - ml0 * sh_s[b][0] + \ + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])) - ml1 * sh_s[b][1]) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = Q2_K_COL(b+0); rf.s1 = Q2_K_COL(b+1); + rf.s2 = Q2_K_COL(b+2); rf.s3 = Q2_K_COL(b+3); + acc[g] += LD4(sh_d, b) * rf; + } +#undef Q2_K_COL +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl new file mode 100644 index 000000000..285114678 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl @@ -0,0 +1,92 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_q3_k_f32( + global const uchar * src0_qs, + global const uchar * src0_hm, + global const uint * src0_sc, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const int is = (i / 16) & 15; + const int word = is >> 2; + const int shift = 8 * (is & 3); + const int sc_base = row + 3 * (i / 256) * m; + const uint4 low = vload4(0, src0_sc + sc_base + (word & 1) * m) >> (uint4)(4 * (word >> 1)); + const uint4 high = vload4(0, src0_sc + sc_base + 2 * m) >> (uint4)(2 * word); + const int4 sc = convert_int4(((low >> (uint4)(shift)) & (uint4)(15)) | (((high >> (uint4)(shift)) & (uint4)(3)) << (uint4)(4))) - (int4)(32); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const half4 scale = convert_half4(convert_float4(d) * convert_float4(sc)); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m); + const uchar4 hbits = vload4(0, src0_hm + row + (ki / 8) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 low = convert_int4((bits >> (uchar4)(2 * j)) & (uchar4)(3)); + const int4 high = convert_int4((hbits >> (uchar4)((ki & 7) + j)) & (uchar4)(1)); + const half4 w = convert_half4(low - (int4)(4) + (int4)(4) * high) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl new file mode 100644 index 000000000..04fcca0b3 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl @@ -0,0 +1,160 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) { + const uint w = is >> 2; + const uint sh = 8u * (is & 3u); + const uint tb = (sc2 >> sh) & 0xFFu; + uint v; + if (w == 0u) { v = ( (sc0 >> sh) & 0xFu) | (((tb >> 0) & 3u) << 4); } + else if (w == 1u) { v = ( (sc1 >> sh) & 0xFu) | (((tb >> 2) & 3u) << 4); } + else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); } + else { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); } + return (int)v - 32; +} + +inline uint q3_k_pack(uint pk, uint hb) { + int v0 = (int)((pk ) & 3u) - 4 + 4*(int)((hb ) & 1u); + int v1 = (int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u); + int v2 = (int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u); + int v3 = (int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u); + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +#define KQ_STAGE(p) vload4(0, (p)) + +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q3_k_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_hm, + __global const uint * src0_sc, + __global const half * src0_d, + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; // super-block along K + const uint sb = sub & 7u; + + const uint scb = rrow + (3u * ib) * (uint)m; + const uint sc0 = src0_sc[scb + 0u * (uint)m]; + const uint sc1 = src0_sc[scb + 1u * (uint)m]; + const uint sc2 = src0_sc[scb + 2u * (uint)m]; + + const float d_w = (float)src0_d[rrow + ib * (uint)m]; + const float dl0 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 0u); + const float dl1 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 1u); + + const uint qsb = rrow + (step >> 2) * (uint)m; + const uint hmb = rrow + (step >> 3) * (uint)m; + + uint4 qlo, qhi; + { + const uint h0 = (uint)src0_hm[hmb + 0u * (uint)m]; + const uint h1 = (uint)src0_hm[hmb + 1u * (uint)m]; + const uint h2 = (uint)src0_hm[hmb + 2u * (uint)m]; + const uint h3 = (uint)src0_hm[hmb + 3u * (uint)m]; + qlo.s0 = q3_k_pack((uint)src0_qs[qsb + 0u * (uint)m], h0 & 0xFu); + qlo.s1 = q3_k_pack((uint)src0_qs[qsb + 1u * (uint)m], (h0 >> 4) & 0xFu); + qlo.s2 = q3_k_pack((uint)src0_qs[qsb + 2u * (uint)m], h1 & 0xFu); + qlo.s3 = q3_k_pack((uint)src0_qs[qsb + 3u * (uint)m], (h1 >> 4) & 0xFu); + qhi.s0 = q3_k_pack((uint)src0_qs[qsb + 4u * (uint)m], h2 & 0xFu); + qhi.s1 = q3_k_pack((uint)src0_qs[qsb + 5u * (uint)m], (h2 >> 4) & 0xFu); + qhi.s2 = q3_k_pack((uint)src0_qs[qsb + 6u * (uint)m], h3 & 0xFu); + qhi.s3 = q3_k_pack((uint)src0_qs[qsb + 7u * (uint)m], (h3 >> 4) & 0xFu); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define Q3_K_COL(T) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[T][0])) \ + + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[T][1]))) + rf.s0 = Q3_K_COL(b+0); rf.s1 = Q3_K_COL(b+1); + rf.s2 = Q3_K_COL(b+2); rf.s3 = Q3_K_COL(b+3); +#undef Q3_K_COL + acc[g] += LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl index 446a8eb18..6418d0f8f 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl @@ -125,3 +125,129 @@ kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + const uint qsbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[qsbase + 0 * m]); + qw.s1 = EXP4(src0_q[qsbase + 1 * m]); + qw.s2 = EXP4(src0_q[qsbase + 2 * m]); + qw.s3 = EXP4(src0_q[qsbase + 3 * m]); + qw.s4 = EXP4(src0_q[qsbase + 4 * m]); + qw.s5 = EXP4(src0_q[qsbase + 5 * m]); + qw.s6 = EXP4(src0_q[qsbase + 6 * m]); + qw.s7 = EXP4(src0_q[qsbase + 7 * m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + + acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b)); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_K_f32_tiled_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_K_f32_tiled_dp4a.cl new file mode 100644 index 000000000..af642b4df --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_K_f32_tiled_dp4a.cl @@ -0,0 +1,131 @@ +// DP4A GEMM for the canonical 64-row Q4_K layout. +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK_K 256 +#define TILE_ROWS 64 +#define NTILES 4 +#define BN_DP 8 +#define WG_THREADS (NTILES * TILE_ROWS) + +inline void q4k_scale_min(int j, __global const uchar * q, uchar * d, uchar * m) { + if (j < 4) { + *d = q[j] & 0x3F; + *m = q[j+4] & 0x3F; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +// dp4a GEMM over the tiled q4_K layout (ne1 > 1) with a q8_1 activation; per 32-element sub-block: +// acc += d*sd*qad*dp4a(q, qa) - dm*sm*qas +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a( + __global uint4 * src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __global uint * qa, // q8_1 int8 activation, 4/uint [ne11][ne00/4] + __global half * qad, // q8_1 per-32-block scale [ne11][ne00/32] + __global half * qas, // q8_1 per-32-block sum (qad*Sum)[ne11][ne00/32] + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + int rit = get_local_id(0); + int sg = get_local_id(1); + int lid = sg * TILE_ROWS + rit; + int row = get_group_id(0) * WG_THREADS + lid; + int rt = row / TILE_ROWS; + int col0 = get_global_id(2) * BN_DP; + int nb = ne00 / QK_K; + int qcs = ne00 / 4; + int dcs = ne00 / 32; + + const bool row_ok = row < ne01; + __local uint sh_qa[BN_DP * 64]; + __local half sh_qad[BN_DP * 8]; + __local half sh_qas[BN_DP * 8]; + + float acc[BN_DP]; + #pragma unroll + for (int j = 0; j < BN_DP; ++j) acc[j] = 0.0f; + + for (int sb = 0; sb < nb; ++sb) { + for (int p = lid; p < BN_DP * 64; p += WG_THREADS) { + int j = p >> 6, u = p & 63, c = col0 + j; + sh_qa[p] = (c < ne11) ? qa[c * qcs + sb * 64 + u] : 0u; + } + for (int p = lid; p < BN_DP * 8; p += WG_THREADS) { + int j = p >> 3, b = p & 7, c = col0 + j; + sh_qad[p] = (c < ne11) ? qad[c * dcs + sb * 8 + b] : (half)0; + sh_qas[p] = (c < ne11) ? qas[c * dcs + sb * 8 + b] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (row_ok) { + int tile_blk = rt * nb + sb; + float dval = (float)src0_d [tile_blk * TILE_ROWS + rit]; + float dmval = (float)src0_dm[tile_blk * TILE_ROWS + rit]; + __global uchar * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 12; + + uint q[32]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_q[(tile_blk * 8 + g) * TILE_ROWS + rit]; + q[g*4+0] = v.x; q[g*4+1] = v.y; q[g*4+2] = v.z; q[g*4+3] = v.w; + } + + for (int is = 0; is < 8; ++is) { + uchar sd, sm; + q4k_scale_min(is, sc, &sd, &sm); + float scl = dval * (float)sd, mnv = dmval * (float)sm; + + uint cw[8]; + #pragma unroll + for (int t = 0; t < 8; ++t) { + uint pk = 0; + #pragma unroll + for (int u = 0; u < 4; ++u) { + int e = is * 32 + t * 4 + u; + uint code = (q[e >> 3] >> ((e & 7) * 4)) & 0xF; + pk |= (code & 0xFF) << (u * 8); + } + cw[t] = pk; + } + #pragma unroll + for (int j = 0; j < BN_DP; ++j) { + __local uint * a = sh_qa + j * 64 + is * 8; + int raw = 0; + #pragma unroll + for (int t = 0; t < 8; ++t) raw = dot_acc_sat_4x8packed_ss_int(cw[t], a[t], raw); + acc[j] += scl * (float)sh_qad[j*8 + is] * (float)raw + - mnv * (float)sh_qas[j*8 + is]; + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_ok) { + dst = (global float*)((global char*)dst + offsetd); + #pragma unroll + for (int j = 0; j < BN_DP; ++j) { + int c = col0 + j; + if (c < ne11) dst[(ulong)c * ne01 + row] = acc[j]; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl index a3b39b6aa..69cabb946 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl @@ -279,3 +279,141 @@ kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const uchar * src0_s, + __global const half * src0_d, + __global const half * src0_dm, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * (uint)m + rrow; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, (uint)m, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[wbase + 0 * m]); + qw.s1 = EXP4(src0_q[wbase + 1 * m]); + qw.s2 = EXP4(src0_q[wbase + 2 * m]); + qw.s3 = EXP4(src0_q[wbase + 3 * m]); + qw.s4 = EXP4(src0_q[wbase + 4 * m]); + qw.s5 = EXP4(src0_q[wbase + 5 * m]); + qw.s6 = EXP4(src0_q[wbase + 6 * m]); + qw.s7 = EXP4(src0_q[wbase + 7 * m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl index aaeed68f6..eef474b47 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl @@ -162,3 +162,141 @@ kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const uchar * src0_qh, + __global const uchar * src0_s, + __global const half * src0_d, + __global const half * src0_dm, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint num_superblocks = (uint)k / QK_K; + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + const uint wbase = rrow + (step >> 2) * (uint)m; + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; +#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \ + | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) ) + qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3); + qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7); +#undef QWU + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl index 382d79fdd..f17403a72 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl @@ -142,3 +142,117 @@ kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_ql, + __global const uchar * src0_qh, + __global const ushort * src0_s, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + + const float dd = (float)src0_d[rrow + sb_idx * m]; + const char2 sc = as_char2(src0_s[rrow + sub * m]); + const float scale0 = dd * (float)sc.s0; + const float scale1 = dd * (float)sc.s1; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint qw[8]; + #pragma unroll + for (int u = 0; u < 8; ++u) { + const uint o = wbase + (uint)u * (uint)m; + qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o])); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + #define DOT_TOK(j) { \ + const uint4 a0 = sh_qa4[b + (j)][0]; \ + const uint4 a1 = sh_qa4[b + (j)][1]; \ + const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a0.x, a0.y, a0.z, a0.w); \ + const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a1.x, a1.y, a1.z, a1.w); \ + rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \ + } + DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3); + #undef DOT_TOK + const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]); + acc[g] += ad * rf; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl index a481636c2..32ff7b4f2 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl @@ -210,3 +210,125 @@ kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4( + __global const uint * src0_q, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = src0_q[wbase + 0 * m]; + qw.s1 = src0_q[wbase + 1 * m]; + qw.s2 = src0_q[wbase + 2 * m]; + qw.s3 = src0_q[wbase + 3 * m]; + qw.s4 = src0_q[wbase + 4 * m]; + qw.s5 = src0_q[wbase + 5 * m]; + qw.s6 = src0_q[wbase + 6 * m]; + qw.s7 = src0_q[wbase + 7 * m]; + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl b/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl index df9d9aed0..0f17df47a 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl @@ -101,7 +101,7 @@ __kernel void kernel_gemm_xmem_f16_f32_os8( const int X = get_group_id(1)*get_local_size(0) + get_local_id(0); const int Z = get_group_id(0)*get_local_size(2) + get_local_id(2); - if (X >= N || Z*8 >= npack) { + if (Z*8 >= npack) { return; } @@ -198,6 +198,11 @@ __kernel void kernel_gemm_xmem_f16_f32_os8( r7 += src1.w * weights_cache[15].scdef; } while (coord_s < kpack); + // Keep all lanes active until the subgroup loads and syncs are done. + if (X >= N) { + return; + } + int coord_s_out = Z*8; if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r0); coord_s_out++; } if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r1); coord_s_out++; } diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl index ee8b94f44..3f21c3523 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl @@ -89,6 +89,9 @@ __kernel void kernel_gemv_moe_mxfp4_f32_ns( uint i11 = i20 % ne11; uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } uint expert_offset = expert_id * ne00 * ne01 / 32; __private float sum = 0.0f; // each thread calculate partial sum of one output @@ -188,6 +191,9 @@ __kernel void kernel_gemv_moe_mxfp4_f32_ns_wimg( uint i11 = i20 % ne11; uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } uint expert_offset = expert_id * ne00 * ne01 / 32; __private float sum = 0.0f; diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl index 2d28db63e..47ffed112 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl @@ -30,7 +30,9 @@ __kernel void kernel_gemv_moe_q4_0_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -41,9 +43,13 @@ __kernel void kernel_gemv_moe_q4_0_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } uint expert_offset = expert_id * ne00 * ne01 / 32; __private float sum = 0.0f; // each thread calculate partial sum of one output diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl index b98bdc0f1..4041887f0 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl @@ -31,7 +31,9 @@ __kernel void kernel_gemv_moe_q4_1_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -42,9 +44,13 @@ __kernel void kernel_gemv_moe_q4_1_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } uint expert_offset = expert_id * ne00 * ne01 / 32; __private float sum = 0.0f; // each thread calculate partial sum of one output diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl index d3a3c7db8..13787568e 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl @@ -47,7 +47,9 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -58,9 +60,13 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int num_superblocks = ne00 / QK_K; int num_subblocks = ne00 / 32; @@ -166,7 +172,9 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns_wimg( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -177,9 +185,13 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns_wimg( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int num_superblocks = ne00 / QK_K; int num_subblocks = ne00 / 32; diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl index b43613638..f98e7bc39 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl @@ -31,7 +31,9 @@ __kernel void kernel_gemv_moe_q5_0_f32_ns( ulong offsetd, uint ne00, uint ne01, - uint ne11 + uint ne11, + uint ne20, + uint ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -42,9 +44,13 @@ __kernel void kernel_gemv_moe_q5_0_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } uint expert_offset = expert_id * ne00 * ne01 / 32; __private float sum = 0.0f; // each thread calculate partial sum of one output diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl index 7a666006e..d5ec3c34d 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl @@ -32,7 +32,9 @@ __kernel void kernel_gemv_moe_q5_1_f32_ns( ulong offsetd, uint ne00, uint ne01, - uint ne11 + uint ne11, + uint ne20, + uint ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -43,9 +45,13 @@ __kernel void kernel_gemv_moe_q5_1_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } uint expert_offset = expert_id * ne00 * ne01 / 32; __private float sum = 0.0f; // each thread calculate partial sum of one output diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl index 7d868d7ab..cb41cd1d0 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl @@ -48,7 +48,9 @@ __kernel void kernel_gemv_moe_q5_k_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -59,9 +61,13 @@ __kernel void kernel_gemv_moe_q5_k_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int num_superblocks = ne00 / QK_K; int num_subblocks = ne00 / 32; diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl index c166bad5b..66cd9a6cc 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl @@ -31,7 +31,9 @@ __kernel void kernel_gemv_moe_q6_k_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -42,9 +44,13 @@ __kernel void kernel_gemv_moe_q6_k_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int num_superblocks = ne00 / QK_K; int num_subblocks = ne00 / 32; // 8 sub-blocks of 32 per super-block diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_m_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_m_f32.cl new file mode 100644 index 000000000..e1f4484fe --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_m_f32.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 +#define IQ1_M_DELTA 0.125f + +#ifndef IQ1_M_MV_NSG +#define IQ1_M_MV_NSG 4 +#endif + +inline float4 iq1_m_vals(uint g) { + return (float4)((float)( g & 0xFu), (float)((g >> 8) & 0xFu), + (float)((g >> 16) & 0xFu), (float)((g >> 24) & 0xFu)); +} + +inline float iq1_m_super(uint w0, uint w1, uint w2, uint w3) { + const ushort u16 = (ushort)(((w0 >> 12)) | ((w1 >> 8) & 0x00f0u) + | ((w2 >> 4) & 0x0f00u) | (w3 & 0xf000u)); + return (float)as_half(u16); +} + +kernel void kernel_gemv_noshuffle_iq1_m_f32( + __read_only image1d_buffer_t grid_img, + global const uchar * src0_qs, + global const uchar * src0_qh, + global const ushort * src0_sc, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ1_M_GRID(i) (read_imageui(grid_img, (int)(i)).x) + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f, sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * qhu = (global const ushort *)src0_qh; + global const uint * scu = (global const uint *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += IQ1_M_MV_NSG) { + const uint sb = j + (ib * 4u) * mh; + const uint w0 = scu[sb + 0u * mh]; + const uint w1 = scu[sb + 1u * mh]; + const uint w2 = scu[sb + 2u * mh]; + const uint w3 = scu[sb + 3u * mh]; + const float d0 = iq1_m_super( w0 & 0xFFFFu, w1 & 0xFFFFu, + w2 & 0xFFFFu, w3 & 0xFFFFu); + const float d1 = iq1_m_super((w0 >> 16) & 0xFFFFu, (w1 >> 16) & 0xFFFFu, + (w2 >> 16) & 0xFFFFu, (w3 >> 16) & 0xFFFFu); + + float acc0 = 0.f, acc1 = 0.f; + + for (uint sub_l = 0; sub_l < 8u; ++sub_l) { + const uint sub = ib * 8u + sub_l; + const uint scw = scu[j + (ib * 4u + (sub_l >> 1)) * mh]; + const uint grp = sub * 8u; + const uint shb = 3u * (2u * (sub_l & 1u)); + + for (uint l = 0; l < 4u; ++l) { + const uint sh = shb + 3u * (l >> 1); + const float s0 = (float)(2u * (((scw ) >> sh) & 7u) + 1u); + const float s1 = (float)(2u * (((scw >> 16) >> sh) & 7u) + 1u); + + const uint qhv = (uint)qhu[j + (sub * 2u + (l >> 1)) * mh]; + const uint h0 = qhv & 0xFFu; + const uint h1 = (qhv >> 8) & 0xFFu; + const uint qsv = (uint)qsu[j + (sub * 4u + l) * mh]; + + const uint msk = 0x08u << (4u * (l & 1u)); + const float t0 = (h0 & msk) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA); + const float t1 = (h1 & msk) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA); + + const uint g0 = IQ1_M_GRID(( qsv & 0xFFu) + | ((((h0 >> (4u*(l & 1u))) & 7u) << 8))); + const uint g1 = IQ1_M_GRID(((qsv >> 8) & 0xFFu) + | ((((h1 >> (4u*(l & 1u))) & 7u) << 8))); + + const float4 y0 = vload4(grp + 2u*l + 0u, y); + const float4 y1 = vload4(grp + 2u*l + 1u, y); + + const float4 ys = y0 + y1; + const float as = ys.s0 + ys.s1 + ys.s2 + ys.s3; + + const float a0 = dot(y0, iq1_m_vals(g0)) + dot(y1, iq1_m_vals(g0 >> 4)); + const float a1 = dot(y0, iq1_m_vals(g1)) + dot(y1, iq1_m_vals(g1 >> 4)); + acc0 += s0 * (a0 + t0 * as); + acc1 += s1 * (a1 + t1 * as); + } + } + sumf += d0 * acc0; + sumf1 += d1 * acc1; + } + } + +#if IQ1_M_MV_NSG > 1 + __local float2 part[IQ1_M_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ1_M_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#undef IQ1_M_GRID +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_s_f32.cl new file mode 100644 index 000000000..6a6eb6e9c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_s_f32.cl @@ -0,0 +1,117 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ1_S_MV_NSG +#define IQ1_S_MV_NSG 4 +#endif + +inline float4 iq1_s_vals(uint g) { + return (float4)((float)( g & 0xFu), (float)((g >> 8) & 0xFu), + (float)((g >> 16) & 0xFu), (float)((g >> 24) & 0xFu)); +} + +kernel void kernel_gemv_noshuffle_iq1_s_f32( + __read_only image1d_buffer_t grid_img, + global const uchar * src0_qs, + global const ushort * src0_qh, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ1_S_GRID(i) (read_imageui(grid_img, (int)(i)).x) + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f, sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + + for (uint ib = sgi; ib < nsb; ib += IQ1_S_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint sub = ib * 8u + sb; + const ushort2 qhv = vload2(j + sub * mh, src0_qh); + const uint h0 = (uint)qhv.s0, h1 = (uint)qhv.s1; + + const float s0 = (float)(2u * ((h0 >> 12) & 7u) + 1u); + const float s1 = (float)(2u * ((h1 >> 12) & 7u) + 1u); + const float t0 = ((h0 & 0x8000u) ? -0.125f : 0.125f) - 1.0f; + const float t1 = ((h1 & 0x8000u) ? -0.125f : 0.125f) - 1.0f; + + const uint gb = j + (sub * 4u) * mh; + const uint grp = sub * 8u; + + float a0 = 0.f, a1 = 0.f; + float4 as = (float4)(0.f); + for (uint l = 0; l < 4u; ++l) { + const uint qsv = (uint)qsu[gb + l * mh]; + const uint g0 = IQ1_S_GRID(( qsv & 0xFFu) | ((((h0 >> (3u*l)) & 7u) << 8))); + const uint g1 = IQ1_S_GRID(((qsv >> 8) & 0xFFu) | ((((h1 >> (3u*l)) & 7u) << 8))); + + const float4 y0 = vload4(grp + 2u*l + 0u, y); + const float4 y1 = vload4(grp + 2u*l + 1u, y); + as += y0 + y1; + a0 += dot(y0, iq1_s_vals(g0)) + dot(y1, iq1_s_vals(g0 >> 4)); + a1 += dot(y0, iq1_s_vals(g1)) + dot(y1, iq1_s_vals(g1 >> 4)); + } + + const float asum = as.s0 + as.s1 + as.s2 + as.s3; + acc0 += s0 * (a0 + t0 * asum); + acc1 += s1 * (a1 + t1 * asum); + } + sumf += (float)dh.s0 * acc0; + sumf1 += (float)dh.s1 * acc1; + } + } + +#if IQ1_S_MV_NSG > 1 + __local float2 part[IQ1_S_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ1_S_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#undef IQ1_S_GRID +} + +kernel void kernel_iq1_s_grid_export(global uint * out) { + const uint i = get_global_id(0); + if (i < 2048u) { + out[i] = iq1_s_grid[i]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_s_f32.cl new file mode 100644 index 000000000..25de1cb91 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_s_f32.cl @@ -0,0 +1,122 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ2_S_MV_NSG +#define IQ2_S_MV_NSG 8 +#endif + +inline float4 iq2_s_vals(uint gv, uint sg, uint base) { + + const uint s = sg >> base; + const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u; + return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn); +} + +kernel void kernel_iq2_s_grid_export(global uint * out) { + out[get_global_id(0)] = iq2_s_grid[get_global_id(0)]; +} + +kernel void kernel_gemv_noshuffle_iq2_s_f32( + read_only image1d_buffer_t grid_img, + global const uchar * src0_qs, + global const uchar * src0_sg, + global const uchar * src0_qh, + global const uchar * src0_sc, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ2_S_GRID(i) read_imageui(grid_img, (int)(i)).x + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f, sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * sgu = (global const ushort *)src0_sg; + global const ushort * qhu = (global const ushort *)src0_qh; + global const ushort * scu = (global const ushort *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += IQ2_S_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint sub = ib * 8u + sb; + const uint qhv = (uint)qhu[j + sub * mh]; + const uint scv = (uint)scu[j + sub * mh]; + const uint gb = j + (sub * 4u) * mh; + + for (uint h = 0; h < 2u; ++h) { + const uint n0 = (h == 0u) ? ( scv & 0xFu) : (( scv >> 4) & 0xFu); + const uint n1 = (h == 0u) ? ((scv >> 8) & 0xFu) : (((scv >> 8) >> 4) & 0xFu); + + float a0 = 0.f, a1 = 0.f; + for (uint t = 0; t < 2u; ++t) { + const uint l = 2u*h + t; + const uint qsv = (uint)qsu[gb + l * mh]; + const uint sgv = (uint)sgu[gb + l * mh]; + const uint gi0 = ( qsv & 0xFFu) | ((((qhv ) >> (2u*l)) & 3u) << 8); + const uint gi1 = ((qsv >> 8) & 0xFFu) | ((((qhv >> 8) >> (2u*l)) & 3u) << 8); + const uint s0 = sgv & 0xFFu; + const uint s1 = (sgv >> 8) & 0xFFu; + + const uint grp = (ib * 64u + sb * 8u) + l * 2u; + const float4 y0 = vload4(grp + 0u, y); + const float4 y1 = vload4(grp + 1u, y); + a0 += dot(y0, iq2_s_vals(IQ2_S_GRID(2u*gi0 + 0u), s0, 0u)); + a0 += dot(y1, iq2_s_vals(IQ2_S_GRID(2u*gi0 + 1u), s0, 4u)); + a1 += dot(y0, iq2_s_vals(IQ2_S_GRID(2u*gi1 + 0u), s1, 0u)); + a1 += dot(y1, iq2_s_vals(IQ2_S_GRID(2u*gi1 + 1u), s1, 4u)); + } + acc0 += (0.5f + (float)n0) * a0; + acc1 += (0.5f + (float)n1) * a1; + } + } + sumf += (float)dh.s0 * 0.25f * acc0; + sumf1 += (float)dh.s1 * 0.25f * acc1; + } + } + +#if IQ2_S_MV_NSG > 1 + __local float2 part[IQ2_S_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ2_S_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#undef IQ2_S_GRID +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xs_f32.cl new file mode 100644 index 000000000..52d626ec8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xs_f32.cl @@ -0,0 +1,128 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ2_XS_MV_NSG +#define IQ2_XS_MV_NSG 4 +#endif + +inline float4 iq2_xs_vals(uint gv, uint sgv, uint base) { + const uint s = sgv >> base; + const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u; + return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn); +} + +kernel void kernel_gemv_noshuffle_iq2_xs_f32( + __read_only image1d_buffer_t grid_img, + global const ushort * src0_qs, + global const uchar * src0_sc, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ2_XS_GRID(i) (read_imageui(grid_img, (int)(i)).x) + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const uint * qsu = (global const uint *)src0_qs; + global const ushort * scu = (global const ushort *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += IQ2_XS_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const float d0 = (float)dh.s0; + const float d1 = (float)dh.s1; + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint sub = ib * 8u + sb; + const uint scv = (uint)scu[j + sub * mh]; + + const float lo0 = 0.5f + (float)( scv & 0xFu); + const float hi0 = 0.5f + (float)((scv >> 4) & 0xFu); + const float lo1 = 0.5f + (float)((scv >> 8) & 0xFu); + const float hi1 = 0.5f + (float)((scv >> 12) & 0xFu); + + const uint grp = sub * 4u; + const uint qsb = j + grp * mh; + + float alo0 = 0.f, ahi0 = 0.f, alo1 = 0.f, ahi1 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint qsv = qsu[qsb + u * mh]; + const uint q0 = qsv & 0xFFFFu; + const uint q1 = qsv >> 16; + + const uint g0 = (q0 & 511u) << 1; + const uint g1 = (q1 & 511u) << 1; + const uint sg0 = iq2_xs_signs(q0 >> 9); + const uint sg1 = iq2_xs_signs(q1 >> 9); + + const float4 yl = vload4(sub * 8u + u * 2u + 0u, y); + const float4 yh = vload4(sub * 8u + u * 2u + 1u, y); + + const float a0 = dot(yl, iq2_xs_vals(IQ2_XS_GRID(g0 ), sg0, 0u)) + + dot(yh, iq2_xs_vals(IQ2_XS_GRID(g0 + 1), sg0, 4u)); + const float a1 = dot(yl, iq2_xs_vals(IQ2_XS_GRID(g1 ), sg1, 0u)) + + dot(yh, iq2_xs_vals(IQ2_XS_GRID(g1 + 1), sg1, 4u)); + + if (u < 2u) { alo0 += a0; alo1 += a1; } + else { ahi0 += a0; ahi1 += a1; } + } + acc0 += lo0 * alo0 + hi0 * ahi0; + acc1 += lo1 * alo1 + hi1 * ahi1; + } + sumf += d0 * 0.25f * acc0; + sumf1 += d1 * 0.25f * acc1; + } + } + +#if IQ2_XS_MV_NSG > 1 + __local float2 part[IQ2_XS_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ2_XS_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#undef IQ2_XS_GRID +} + +kernel void kernel_iq2_xs_grid_export(global uint * out) { + const uint i = get_global_id(0); + if (i < 1024u) { + out[i] = iq2_xs_grid[i]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xxs_f32.cl new file mode 100644 index 000000000..405a8144f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xxs_f32.cl @@ -0,0 +1,117 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ2_XXS_MV_NSG +#define IQ2_XXS_MV_NSG 4 +#endif + +inline float4 iq2_xxs_vals(uint gv, uint sgv, uint base) { + const uint s = sgv >> base; + const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u; + return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn); +} + +kernel void kernel_gemv_noshuffle_iq2_xxs_f32( + __read_only image1d_buffer_t grid_img, + global const uchar * src0_qs, + global const uint * src0_sas, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ2_XXS_GRID(i) (read_imageui(grid_img, (int)(i)).x) + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + + for (uint ib = sgi; ib < nsb; ib += IQ2_XXS_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const float d0 = (float)dh.s0; + const float d1 = (float)dh.s1; + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint sub = ib * 8u + sb; + const uint2 aux = vload2(j + sub * mh, src0_sas); + + const uint grp = sub * 4u; + const uint qsb = j + grp * mh; + + float a0 = 0.f, a1 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint qsv = (uint)qsu[qsb + u * mh]; + const uint sg0 = iq2_xxs_signs((aux.s0 >> (7u * u)) & 127u); + const uint sg1 = iq2_xxs_signs((aux.s1 >> (7u * u)) & 127u); + + const uint g0 = ( qsv & 0xFFu) << 1; + const uint g1 = ((qsv >> 8) & 0xFFu) << 1; + + const float4 yl = vload4(sub * 8u + u * 2u + 0u, y); + const float4 yh = vload4(sub * 8u + u * 2u + 1u, y); + + a0 += dot(yl, iq2_xxs_vals(IQ2_XXS_GRID(g0 ), sg0, 0u)) + + dot(yh, iq2_xxs_vals(IQ2_XXS_GRID(g0 + 1), sg0, 4u)); + a1 += dot(yl, iq2_xxs_vals(IQ2_XXS_GRID(g1 ), sg1, 0u)) + + dot(yh, iq2_xxs_vals(IQ2_XXS_GRID(g1 + 1), sg1, 4u)); + } + acc0 += (0.5f + (float)(aux.s0 >> 28)) * a0; + acc1 += (0.5f + (float)(aux.s1 >> 28)) * a1; + } + sumf += d0 * 0.25f * acc0; + sumf1 += d1 * 0.25f * acc1; + } + } + +#if IQ2_XXS_MV_NSG > 1 + __local float2 part[IQ2_XXS_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ2_XXS_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#undef IQ2_XXS_GRID +} + +kernel void kernel_iq2_xxs_grid_export(global uint * out) { + const uint i = get_global_id(0); + if (i < 512u) { + out[i] = iq2_xxs_grid[i]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl new file mode 100644 index 000000000..e94b84a86 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl @@ -0,0 +1,128 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ3_S_MV_NSG +#define IQ3_S_MV_NSG 4 +#endif + +inline float4 iq3_s_vals(uint gv, uint sg, uint base) { + + const uint s = sg >> base; + const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u; + return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn); +} + +kernel void kernel_gemv_noshuffle_iq3_s_f32( + __read_only image1d_buffer_t grid_img, + global const uchar * src0_qs, + global const uchar * src0_sg, + global const uchar * src0_qh, + global const uchar * src0_sc, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ3_S_GRID(i) (read_imageui(grid_img, (int)(i)).x) + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * qhu = (global const ushort *)src0_qh; + global const ushort * sgu = (global const ushort *)src0_sg; + global const ushort * scu = (global const ushort *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += IQ3_S_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const float d0 = (float)dh.s0; + const float d1 = (float)dh.s1; + + const uint scbase = j + ib * 4u * mh; + ushort sc4[4]; + sc4[0] = scu[scbase + 0u * mh]; + sc4[1] = scu[scbase + 1u * mh]; + sc4[2] = scu[scbase + 2u * mh]; + sc4[3] = scu[scbase + 3u * mh]; + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint scv = (uint)sc4[sb >> 1]; + const uint nib0 = (sb & 1u) ? ((scv >> 4) & 0xFu) : ( scv & 0xFu); + const uint nib1 = (sb & 1u) ? ((scv >> 12) & 0xFu) : ((scv >> 8) & 0xFu); + + const uint qhv = (uint)qhu[j + (ib * 8u + sb) * mh]; + const uint qh0 = qhv & 0xFFu; + const uint qh1 = qhv >> 8; + + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mh; + const uint sgb = j + (ib * 32u + sb * 4u) * mh; + + float a0 = 0.f, a1 = 0.f; + for (uint u = 0; u < 8u; ++u) { + const uint qsv = (uint)qsu[qsb + u * mh]; + const uint sgv = (uint)sgu[sgb + (u >> 1) * mh]; + const uint g0 = ( qsv & 0xFFu) | (((qh0 >> u) & 1u) << 8); + const uint g1 = ((qsv >> 8) & 0xFFu) | (((qh1 >> u) & 1u) << 8); + const uint base = (u & 1u) * 4u; + const float4 yv = vload4(grp + u, y); + const uint gw0 = IQ3_S_GRID(g0); + const uint gw1 = IQ3_S_GRID(g1); + a0 += dot(yv, iq3_s_vals(gw0, sgv & 0xFFu, base)); + a1 += dot(yv, iq3_s_vals(gw1, (sgv >> 8) & 0xFFu, base)); + } + acc0 += (float)(1u + 2u * nib0) * a0; + acc1 += (float)(1u + 2u * nib1) * a1; + } + sumf += d0 * acc0; + sumf1 += d1 * acc1; + } + } + +#if IQ3_S_MV_NSG > 1 + __local float2 part[IQ3_S_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ3_S_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +} + +kernel void kernel_iq3_s_grid_export(global uint * out) { + out[get_global_id(0)] = iq3_s_grid[get_global_id(0)]; +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl new file mode 100644 index 000000000..c6a5dbeac --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl @@ -0,0 +1,120 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ3_XXS_MV_NSG +#define IQ3_XXS_MV_NSG 4 +#endif + +inline float4 iq3_xxs_vals(uint gv, uint sgv, uint base) { + const uint s = sgv >> base; + const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u; + return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn); +} + +kernel void kernel_gemv_noshuffle_iq3_xxs_f32( + __read_only image1d_buffer_t grid_img, + global const uchar * src0_qs, + global const uint * src0_sas, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ3_XXS_GRID(i) (read_imageui(grid_img, (int)(i)).x) + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + + for (uint ib = sgi; ib < nsb; ib += IQ3_XXS_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const float d0 = (float)dh.s0; + const float d1 = (float)dh.s1; + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint sub = ib * 8u + sb; + const uint2 aux = vload2(j + sub * mh, src0_sas); + + const uint grp = sub * 8u; + const uint qsb = j + grp * mh; + + float a0 = 0.f, a1 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint qsv0 = (uint)qsu[qsb + (2u*u)*mh]; + const uint qsv1 = (uint)qsu[qsb + (2u*u + 1u)*mh]; + const uint sg0 = iq3_xxs_signs((aux.s0 >> (7u * u)) & 127u); + const uint sg1 = iq3_xxs_signs((aux.s1 >> (7u * u)) & 127u); + + const uint g00 = qsv0 & 0xFFu; + const uint g01 = qsv1 & 0xFFu; + const uint g10 = (qsv0 >> 8) & 0xFFu; + const uint g11 = (qsv1 >> 8) & 0xFFu; + + const float4 yl = vload4(sub * 8u + u * 2u + 0u, y); + const float4 yh = vload4(sub * 8u + u * 2u + 1u, y); + + a0 += dot(yl, iq3_xxs_vals(IQ3_XXS_GRID(g00), sg0, 0u)) + + dot(yh, iq3_xxs_vals(IQ3_XXS_GRID(g01), sg0, 4u)); + a1 += dot(yl, iq3_xxs_vals(IQ3_XXS_GRID(g10), sg1, 0u)) + + dot(yh, iq3_xxs_vals(IQ3_XXS_GRID(g11), sg1, 4u)); + } + acc0 += (0.5f + (float)(aux.s0 >> 28)) * a0; + acc1 += (0.5f + (float)(aux.s1 >> 28)) * a1; + } + sumf += d0 * 0.5f * acc0; + sumf1 += d1 * 0.5f * acc1; + } + } + +#if IQ3_XXS_MV_NSG > 1 + __local float2 part[IQ3_XXS_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ3_XXS_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#undef IQ3_XXS_GRID +} + +kernel void kernel_iq3_xxs_grid_export(global uint * out) { + const uint i = get_global_id(0); + if (i < 256u) { + out[i] = iq3_xxs_grid[i]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl index 1f832cb25..e17ca09ba 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl @@ -230,6 +230,8 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; private uint4 regA; @@ -240,7 +242,7 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32( // loop along K in block granularity, skip 4 blocks every iter for (uint k = groupId; k < (K / QK4_NL); k += NSUBGROUPS) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -248,20 +250,20 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32( } // load half weights for two blocks in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB); #endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB); #else diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl new file mode 100644 index 000000000..3f71add46 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl @@ -0,0 +1,85 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ4_XS_MV_NSG +#define IQ4_XS_MV_NSG 8 +#endif + +inline float4 iq4_xs_vals(ushort q) { + return (float4)((char)iq4_xs_code(q & 15), (char)iq4_xs_code((q >> 4) & 15), + (char)iq4_xs_code((q >> 8) & 15), (char)iq4_xs_code(q >> 12)); +} + +kernel void kernel_gemv_noshuffle_iq4_xs_f32( + global const ushort * src0_qs, + global const uint * src0_sl, + global const ushort * src0_sh, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint mh = m >> 1; + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + const uint j = get_group_id(0)*64u + lid; + const uint row = j << 1; + global const float * y = src1 + (ulong)col * (uint)ne10; + + float sum0 = 0.0f, sum1 = 0.0f; + if (j < mh) { + global const uint * qs = (global const uint *)src0_qs; + global const uint * sh = (global const uint *)src0_sh; + for (uint ib = sgi; ib < (uint)ne00/QK_K; ib += IQ4_XS_MV_NSG) { + const uint base = j + ib*mh; + const uint2 sl = vload2(base, src0_sl); + const uint shp = sh[base]; + const half2 d = vload2(base, src0_d); + float acc0 = 0.0f, acc1 = 0.0f; + for (uint sb = 0; sb < 8; ++sb) { + const int ls0 = (int)(((sl.s0 >> (4*sb)) & 15u) | (((shp >> (2*sb)) & 3u) << 4)) - 32; + const int ls1 = (int)(((sl.s1 >> (4*sb)) & 15u) | (((shp >> (16 + 2*sb)) & 3u) << 4)) - 32; + float a0 = 0.0f, a1 = 0.0f; + for (uint u = 0; u < 8; ++u) { + const uint group = ib*64 + sb*8 + u; + const uint q = qs[j + group*mh]; + const float4 yv = vload4(group, y); + a0 += dot(yv, iq4_xs_vals((ushort)q)); + a1 += dot(yv, iq4_xs_vals((ushort)(q >> 16))); + } + acc0 += (float)ls0*a0; + acc1 += (float)ls1*a1; + } + sum0 += (float)d.s0*acc0; + sum1 += (float)d.s1*acc1; + } + } + +#if IQ4_XS_MV_NSG > 1 + local float2 part[IQ4_XS_MV_NSG][64]; + part[sgi][lid] = (float2)(sum0, sum1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ4_XS_MV_NSG; ++s) { + const float2 v = part[s][lid]; + sum0 += v.s0; + sum1 += v.s1; + } +#endif + if (j < mh) { + vstore2((float2)(sum0, sum1), 0, dst + (ulong)col*(uint)ne0 + row); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl index 9efede294..572f11637 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl @@ -8,7 +8,9 @@ #endif #define QK1_0 128 +#ifndef N_SIMDGROUP #define N_SIMDGROUP 4 +#endif #define dequantizeBlockAccum_q1(total, bits, scale, regB, lb) \ total += (2.0f*(float)((bits >> 0) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+0); \ @@ -73,6 +75,8 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32( uint M = ne01; uint LINE_STRIDE_A = M; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = 4 * M; uint4 regA; @@ -83,7 +87,7 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32( #pragma unroll 1 for (uint kb = groupId; kb < (K / QK1_0); kb += N_SIMDGROUP) { - regS = src0_d[gid + kb * LINE_STRIDE_A]; // each fiber loads its row's scale + regS = src0_d[gid_s + kb * LINE_STRIDE_A]; // each fiber loads its row's scale // first 16 fibers load 8 B values each -> 128 activations for this block if (slid < 16) { @@ -92,10 +96,10 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32( } // load this row's 4 uint32 (128 sign bits) - regA.s0 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; float scale = (float)regS; dequantizeBlockAccum_q1(totalSum, regA.s0, scale, regB, 0); @@ -104,15 +108,19 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32( dequantizeBlockAccum_q1(totalSum, regA.s3, scale, regB, 12); } - // reduction in local memory, assumes #wave = N_SIMDGROUP = 4 - local float reduceLM[SIMDGROUP_WIDTH * 3]; - if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum; - if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum; - if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum; +#if N_SIMDGROUP > 1 + // Sum the K slices from the other subgroups. + local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (groupId > 0) { + reduceLM[SIMDGROUP_WIDTH * (groupId - 1) + slid] = totalSum; + } barrier(CLK_LOCAL_MEM_FENCE); - if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; - if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; - if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + if (groupId == 0) { + for (uint w = 0; w < N_SIMDGROUP - 1; ++w) { + totalSum += reduceLM[SIMDGROUP_WIDTH * w + slid]; + } + } +#endif if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); @@ -120,6 +128,8 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32( // so when ne01 is not a multiple of the wave size the tail work-items run past // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical // when ne01 is wave-aligned (no padding). - if (gid < M) dst[gid] = totalSum; + if (gid < M) { + dst[gid] = totalSum; + } } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl new file mode 100644 index 000000000..a46293c24 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl @@ -0,0 +1,77 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#if Q2_0_MV_WIDE +#define Q2_0_MV_ROWS 8 +#define Q2_0_MV_LOAD_D vload8 +#define Q2_0_MV_STORE vstore8 +#define Q2_0_MV_FLOAT convert_float8 +typedef float8 q2_0_mv_acc_t; +typedef uchar8 q2_0_mv_quant_t; +#else +#define Q2_0_MV_ROWS 4 +#define Q2_0_MV_LOAD_D vload4 +#define Q2_0_MV_STORE vstore4 +#define Q2_0_MV_FLOAT convert_float4 +typedef float4 q2_0_mv_acc_t; +typedef uchar4 q2_0_mv_quant_t; +#endif + +kernel void kernel_gemv_noshuffle_q2_0_f32( + global const uchar * src0_qs, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint mr = (uint)ne01/Q2_0_MV_ROWS; + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint lanes = get_local_size(0); + const uint nsg = get_local_size(1); + const uint col = get_group_id(1); + const uint j = get_group_id(0)*lanes + lid; + global const float * y = src1 + (ulong)col * (uint)ne10; + + q2_0_mv_acc_t sum = 0.0f; + if (j < mr) { + for (uint ib = sgi; ib < (uint)ne00/64; ib += nsg) { + const q2_0_mv_acc_t d = Q2_0_MV_FLOAT(Q2_0_MV_LOAD_D(j + ib*mr, src0_d)); + q2_0_mv_acc_t acc = 0.0f; + for (uint u = 0; u < 16; ++u) { + const uint group = ib*16 + u; +#if Q2_0_MV_WIDE + const uchar8 q = as_uchar8(((global const uint2 *)src0_qs)[j + group*mr]); +#else + const uchar4 q = as_uchar4(((global const uint *)src0_qs)[j + group*mr]); +#endif + const float4 yv = vload4(group, y); + acc += (Q2_0_MV_FLOAT( q & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s0; + acc += (Q2_0_MV_FLOAT((q >> (q2_0_mv_quant_t)(2)) & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s1; + acc += (Q2_0_MV_FLOAT((q >> (q2_0_mv_quant_t)(4)) & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s2; + acc += (Q2_0_MV_FLOAT( q >> (q2_0_mv_quant_t)(6)) - 1.0f)*yv.s3; + } + sum += d*acc; + } + } + + local q2_0_mv_acc_t part[512]; + part[sgi*lanes + lid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < nsg; ++s) { + sum += part[s*lanes + lid]; + } + if (j < mr) { + Q2_0_MV_STORE(sum, 0, dst + (ulong)col*(uint)ne0 + j*Q2_0_MV_ROWS); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl new file mode 100644 index 000000000..efcc31077 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl @@ -0,0 +1,231 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef Q2_K_MV_NSG +#define Q2_K_MV_NSG 8 +#endif + +#ifndef Q2_K_MV_R +#define Q2_K_MV_R 4 +#endif + +inline float4 q2_k_vals(uint pk) { + return (float4)((float)( pk & 3u), (float)((pk >> 2) & 3u), + (float)((pk >> 4) & 3u), (float)((pk >> 6) & 3u)); +} + +kernel void kernel_gemv_noshuffle_q2_k_f32( + global const uchar * src0_qs, + global const uchar * src0_sc, + global const half * src0_dm, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, // K + int ne01, // M + int ne10, // activation row stride, == K + int ne0 // dst row stride +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + + const uint mr = m / Q2_K_MV_R; // row groups + const uint j = get_group_id(0) * 64u + lid; // row group this lane owns + const uint row = j * Q2_K_MV_R; + +#if Q2_K_MV_R == 4 + float sumf0 = 0.f; + float sumf1 = 0.f; + float sumf2 = 0.f; + float sumf3 = 0.f; + + if (j < mr) { + global const uint * qsu = (global const uint *)src0_qs; + global const uint * scu = (global const uint *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) { + const half4 dma = vload4(2u*(j + ib*mr) + 0u, src0_dm); // d0, dmin0, d1, dmin1 + const half4 dmb = vload4(2u*(j + ib*mr) + 1u, src0_dm); // d2, dmin2, d3, dmin3 + + float ad0 = 0.f, am0 = 0.f, ad1 = 0.f, am1 = 0.f; + float ad2 = 0.f, am2 = 0.f, ad3 = 0.f, am3 = 0.f; + + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mr; + + for (uint h = 0; h < 2u; ++h) { + const uint scv = scu[j + (2u * (ib * 8u + sb) + h) * mr]; + + float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f; + float4 as = (float4)(0.f); + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = qsu[qsb + gg * mr]; // four rows, one load + const float4 yv = vload4(grp + gg, y); + as += yv; + a0 += dot(yv, q2_k_vals( qsv & 0xFFu)); + a1 += dot(yv, q2_k_vals((qsv >> 8) & 0xFFu)); + a2 += dot(yv, q2_k_vals((qsv >> 16) & 0xFFu)); + a3 += dot(yv, q2_k_vals((qsv >> 24) & 0xFFu)); + } + const float asum = as.s0 + as.s1 + as.s2 + as.s3; + + const uint s0 = scv & 0xFFu; + const uint s1 = (scv >> 8) & 0xFFu; + const uint s2 = (scv >> 16) & 0xFFu; + const uint s3 = (scv >> 24) & 0xFFu; + ad0 += (float)(s0 & 0xFu) * a0; am0 += (float)(s0 >> 4) * asum; + ad1 += (float)(s1 & 0xFu) * a1; am1 += (float)(s1 >> 4) * asum; + ad2 += (float)(s2 & 0xFu) * a2; am2 += (float)(s2 >> 4) * asum; + ad3 += (float)(s3 & 0xFu) * a3; am3 += (float)(s3 >> 4) * asum; + } + } + sumf0 += (float)dma.s0 * ad0 - (float)dma.s1 * am0; + sumf1 += (float)dma.s2 * ad1 - (float)dma.s3 * am1; + sumf2 += (float)dmb.s0 * ad2 - (float)dmb.s1 * am2; + sumf3 += (float)dmb.s2 * ad3 - (float)dmb.s3 * am3; + } + } + +#if Q2_K_MV_NSG > 1 + __local float4 part[Q2_K_MV_NSG][64]; + part[sgi][lid] = (float4)(sumf0, sumf1, sumf2, sumf3); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q2_K_MV_NSG; ++s) { + const float4 p = part[s][lid]; + sumf0 += p.s0; + sumf1 += p.s1; + sumf2 += p.s2; + sumf3 += p.s3; + } +#endif + + if (j < mr) { + global float * o = dst + (ulong)col * (uint)ne0 + row; + o[0] = sumf0; o[1] = sumf1; o[2] = sumf2; o[3] = sumf3; + } + +#elif Q2_K_MV_R == 2 + float sumf0 = 0.f; + float sumf1 = 0.f; + + if (j < mr) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * scu = (global const ushort *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) { + const half4 dm = vload4(j + ib * mr, src0_dm); // d0, dmin0, d1, dmin1 + + float ad0 = 0.f, am0 = 0.f, ad1 = 0.f, am1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mr; + + for (uint h = 0; h < 2u; ++h) { + const uint scv = (uint)scu[j + (2u * (ib * 8u + sb) + h) * mr]; + const uint s0 = scv & 0xFFu; + const uint s1 = scv >> 8; + + float a0 = 0.f, a1 = 0.f; + float4 as = (float4)(0.f); + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = (uint)qsu[qsb + gg * mr]; + const float4 yv = vload4(grp + gg, y); + as += yv; + a0 += dot(yv, q2_k_vals( qsv & 0xFFu)); + a1 += dot(yv, q2_k_vals((qsv >> 8) & 0xFFu)); + } + const float asum = as.s0 + as.s1 + as.s2 + as.s3; + ad0 += (float)(s0 & 0xFu) * a0; am0 += (float)(s0 >> 4) * asum; + ad1 += (float)(s1 & 0xFu) * a1; am1 += (float)(s1 >> 4) * asum; + } + } + sumf0 += (float)dm.s0 * ad0 - (float)dm.s1 * am0; + sumf1 += (float)dm.s2 * ad1 - (float)dm.s3 * am1; + } + } + +#if Q2_K_MV_NSG > 1 + __local float2 part[Q2_K_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf0, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q2_K_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf0 += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mr) { + vstore2((float2)(sumf0, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } + +#else + float sumf0 = 0.f; + + if (j < mr) { + for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) { + const half2 dm = vload2(row + ib * m, src0_dm); + + float ad = 0.f, am = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = row + grp * m; + + for (uint h = 0; h < 2u; ++h) { + const uint sc = (uint)src0_sc[row + (2u * (ib * 8u + sb) + h) * m]; + + float a = 0.f; + float4 as = (float4)(0.f); + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const float4 yv = vload4(grp + gg, y); + as += yv; + const uint pkv = (uint)src0_qs[qsb + gg * m]; + a += dot(yv, q2_k_vals(pkv)); + } + const float asum = as.s0 + as.s1 + as.s2 + as.s3; + ad += (float)(sc & 0xFu) * a; am += (float)(sc >> 4) * asum; + } + } + sumf0 += (float)dm.s0 * ad - (float)dm.s1 * am; + } + } + +#if Q2_K_MV_NSG > 1 + __local float part[Q2_K_MV_NSG][64]; + part[sgi][lid] = sumf0; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q2_K_MV_NSG; ++s) { + sumf0 += part[s][lid]; + } +#endif + + if (j < mr) { + dst[(ulong)col * (uint)ne0 + row] = sumf0; + } +#endif +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl new file mode 100644 index 000000000..f44354154 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl @@ -0,0 +1,249 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef Q3_K_MV_NSG +#define Q3_K_MV_NSG 8 +#endif + +#ifndef Q3_K_MV_R +#define Q3_K_MV_R 2 +#endif + +inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) { + const uint w = is >> 2; + const uint sh = 8u * (is & 3u); + const uint tb = (sc2 >> sh) & 0xFFu; + uint v; + if (w == 0u) { v = ( (sc0 >> sh) & 0xFu) | (((tb >> 0) & 3u) << 4); } + else if (w == 1u) { v = ( (sc1 >> sh) & 0xFu) | (((tb >> 2) & 3u) << 4); } + else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); } + else { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); } + return (int)v - 32; +} + +inline float4 q3_k_vals(uint pk, uint hb) { + float4 v; + v.s0 = (float)((int)((pk ) & 3u) - 4 + 4*(int)((hb ) & 1u)); + v.s1 = (float)((int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u)); + v.s2 = (float)((int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u)); + v.s3 = (float)((int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u)); + return v; +} + +kernel void kernel_gemv_noshuffle_q3_k_f32( + global const uchar * src0_qs, + global const uchar * src0_hm, + global const uint * src0_sc, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, // K + int ne01, // M + int ne10, // activation row stride + int ne0 // dst row stride +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#if Q3_K_MV_R == 4 + const uint mq = m >> 2; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 2; + + float sumf = 0.f, sumf1 = 0.f, sumf2 = 0.f, sumf3 = 0.f; + + if (j < mq) { + global const uint * qsw = (global const uint *)src0_qs; + global const uint * hmw = (global const uint *)src0_hm; + + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const half4 dh = vload4(j + ib * mq, src0_d); + const uint4 s0 = vload4(j + (3u * ib + 0u) * mq, src0_sc); + const uint4 s1 = vload4(j + (3u * ib + 1u) * mq, src0_sc); + const uint4 s2 = vload4(j + (3u * ib + 2u) * mq, src0_sc); + + float acc0 = 0.f, acc1 = 0.f, acc2 = 0.f, acc3 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mq; + const uint hmb = j + (grp >> 1) * mq; + + for (uint h = 0; h < 2u; ++h) { // two 16-weight halves + const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h); + const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h); + const int l2 = q3_k_scale(s0.s2, s1.s2, s2.s2, 2u*sb + h); + const int l3 = q3_k_scale(s0.s3, s1.s3, s2.s3, 2u*sb + h); + + float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = qsw[qsb + gg * mq]; // four rows, one load + const uint hmv = hmw[hmb + (gg >> 1) * mq]; + const uint hsh = 4u * (gg & 1u); + const float4 yv = vload4(grp + gg, y); // shared by all four + a0 += dot(yv, q3_k_vals((qsv ) & 0xFFu, ((hmv ) >> hsh) & 0xFu)); + a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8) >> hsh) & 0xFu)); + a2 += dot(yv, q3_k_vals((qsv >> 16) & 0xFFu, ((hmv >> 16) >> hsh) & 0xFu)); + a3 += dot(yv, q3_k_vals((qsv >> 24) & 0xFFu, ((hmv >> 24) >> hsh) & 0xFu)); + } + acc0 += (float)l0 * a0; + acc1 += (float)l1 * a1; + acc2 += (float)l2 * a2; + acc3 += (float)l3 * a3; + } + } + sumf += (float)dh.s0 * acc0; + sumf1 += (float)dh.s1 * acc1; + sumf2 += (float)dh.s2 * acc2; + sumf3 += (float)dh.s3 * acc3; + } + } +#elif Q3_K_MV_R == 2 + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * hmu = (global const ushort *)src0_hm; + + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const uint2 s0 = vload2(j + (3u * ib + 0u) * mh, src0_sc); + const uint2 s1 = vload2(j + (3u * ib + 1u) * mh, src0_sc); + const uint2 s2 = vload2(j + (3u * ib + 2u) * mh, src0_sc); + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mh; + const uint hmb = j + (grp >> 1) * mh; + + for (uint h = 0; h < 2u; ++h) { // two 16-weight halves + const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h); + const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h); + + float a0 = 0.f, a1 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = (uint)qsu[qsb + gg * mh]; + const uint hmv = (uint)hmu[hmb + (gg >> 1) * mh]; + const uint hsh = 4u * (gg & 1u); + const float4 yv = vload4(grp + gg, y); + a0 += dot(yv, q3_k_vals( qsv & 0xFFu, ( hmv >> hsh) & 0xFu)); + a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8) >> hsh) & 0xFu)); + } + acc0 += (float)l0 * a0; + acc1 += (float)l1 * a1; + } + } + sumf += (float)dh.s0 * acc0; + sumf1 += (float)dh.s1 * acc1; + } + } +#else + const uint row = get_group_id(0) * 64u + lid; + + float sumf = 0.f; + + if (row < m) { + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const float d = (float)src0_d[row + ib * m]; + const uint s0 = src0_sc[row + (3u * ib + 0u) * m]; + const uint s1 = src0_sc[row + (3u * ib + 1u) * m]; + const uint s2 = src0_sc[row + (3u * ib + 2u) * m]; + + float acc = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = row + grp * m; + const uint hmb = row + (grp >> 1) * m; + + for (uint h = 0; h < 2u; ++h) { + const int l = q3_k_scale(s0, s1, s2, 2u*sb + h); + float a = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = (uint)src0_qs[qsb + gg * m]; + const uint hmv = (uint)src0_hm[hmb + (gg >> 1) * m]; + const float4 yv = vload4(grp + gg, y); + a += dot(yv, q3_k_vals(qsv, (hmv >> (4u * (gg & 1u))) & 0xFu)); + } + acc += (float)l * a; + } + } + sumf += d * acc; + } + } +#endif + +#if Q3_K_MV_NSG > 1 +#if Q3_K_MV_R == 4 + __local float4 part[Q3_K_MV_NSG][64]; + part[sgi][lid] = (float4)(sumf, sumf1, sumf2, sumf3); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + const float4 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + sumf2 += p.s2; + sumf3 += p.s3; + } +#elif Q3_K_MV_R == 2 + __local float2 part[Q3_K_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#else + __local float part[Q3_K_MV_NSG][64]; + part[sgi][lid] = sumf; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + sumf += part[s][lid]; + } +#endif +#endif + +#if Q3_K_MV_R == 4 + if (j < mq) { + vstore4((float4)(sumf, sumf1, sumf2, sumf3), 0, dst + (ulong)col * (uint)ne0 + row); + } +#elif Q3_K_MV_R == 2 + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#else + if (row < m) { + dst[(ulong)col * (uint)ne0 + row] = sumf; + } +#endif +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl index 023e848f7..6066ae3a9 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl @@ -7,6 +7,14 @@ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) #endif +// A6X compiler incorrectly constant-folds get_local_size() results; +// force runtime materialization via a no-op ALU round-trip. +#ifdef GGML_CL_A6X_CONSTFOLD_FIX +#define MATERIALIZE_WG(x) do { (x) *= 2u; if ((x) > 1u) (x) /= 2u; } while(0) +#else +#define MATERIALIZE_WG(x) +#endif + // assume #define QK4_0 32 #define N_SIMDGROUP 4 @@ -216,6 +224,8 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = N_SIMDGROUP * M; __private uint4 regA; @@ -226,7 +236,7 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( // loop along K in block granularity, skip 4 blocks every iter for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -234,20 +244,20 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( } // load half weights for two blocks in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB); #endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB); #else @@ -321,12 +331,15 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32_mc3( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); // BLOCK_STRIDE_A is the LAYOUT stride between consecutive K-blocks = 4 uints // per q4_0 block * M (set by the trans4_ns convert). The "4" is uints/block, NOT // the subgroup count — keep it fixed so the K-split count (nsg) can vary. uint BLOCK_STRIDE_A = N_SIMDGROUP * M; // = 4 * M (N_SIMDGROUP is the #define 4) uint COL_STRIDE = K / 4; // float4 pixels per activation column uint nsg = get_local_size(1); // runtime K-split (4 default, 8 small-M) + MATERIALIZE_WG(nsg); __private uint4 regA_hi, regA_lo; __private half2 regS; @@ -338,17 +351,17 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32_mc3( __private float2 ts3 = (float2)(0.0f); for (uint k = groupId; k < (K / QK4_0); k += nsg) { - regS = src0_d[gid + k * LINE_STRIDE_A]; + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // weights loaded ONCE, reused across the columns - regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; - regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; MC_COL_Q40(ts0, 0); MC_COL_Q40(ts1, 1); @@ -372,10 +385,12 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32_mc3( } dst = (global float*)((global char*)dst + offsetd); // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row - vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); - vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); - if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); - if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + if (gid < LINE_STRIDE_A) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } } } #undef MC_COL_Q40 diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl index 0dca20f71..63990c414 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl @@ -210,6 +210,8 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( { uint groupId = get_local_id(1); uint gid = get_global_id(0); + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, (uint)LINE_STRIDE_A - 1u); ushort slid = get_sub_group_local_id(); __private uint4 regA; @@ -220,7 +222,7 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( // loop along K in block granularity, skip 4 blocks every iter for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -228,20 +230,20 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( } // load half weights for two blocks in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB); #endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB); #else diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl index 2ccf4214c..b0b4793d7 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl @@ -209,6 +209,8 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; private uint4 regA; @@ -220,8 +222,8 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( // loop along K in block granularity, skip 4 blocks every iter for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows - regM = src0_m[gid + k * LINE_STRIDE_A]; // each fiber loads min of two rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows + regM = src0_m[gid_s + k * LINE_STRIDE_A]; // each fiber loads min of two rows // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -229,20 +231,20 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( } // load half weights for two blocks in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAT dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB); #endif // VECTOR_SUB_GROUP_BROADCAT - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAT dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB); #else @@ -326,6 +328,8 @@ kernel void kernel_gemv_noshuffle_q4_1_f32_mc3( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; uint COL_STRIDE = K / 4; // float4 pixels per activation column @@ -339,18 +343,18 @@ kernel void kernel_gemv_noshuffle_q4_1_f32_mc3( private float2 ts3 = (float2)(0.0f); for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) { - regS = src0_d[gid + k * LINE_STRIDE_A]; - regM = src0_m[gid + k * LINE_STRIDE_A]; + regS = src0_d[gid_s + k * LINE_STRIDE_A]; + regM = src0_m[gid_s + k * LINE_STRIDE_A]; // weights loaded ONCE, reused across the columns - regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; - regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; MC_COL_Q41(ts0, 0); MC_COL_Q41(ts1, 1); @@ -373,10 +377,12 @@ kernel void kernel_gemv_noshuffle_q4_1_f32_mc3( acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; dst = (global float*)((global char*)dst + offsetd); // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row - vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); - vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); - if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); - if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + if (gid < LINE_STRIDE_A) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } } } #undef MC_COL_Q41 diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl index c0078131e..3dc30db58 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl @@ -11,6 +11,14 @@ #define NSUBGROUPS 4 #define SUBGROUP_SIZE 64 +// A6X compiler incorrectly constant-folds get_local_size() results; +// force runtime materialization via a no-op ALU round-trip. +#ifdef GGML_CL_A6X_CONSTFOLD_FIX +#define MATERIALIZE_WG(x) do { (x) *= 2u; if ((x) > 1u) (x) /= 2u; } while(0) +#else +#define MATERIALIZE_WG(x) +#endif + // scales are transposed: consecutive codes of a row are `stride` apart inline void get_scale_min_k4( int j, @@ -233,6 +241,7 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( // K-split (more waves/SP -> latency hiding) while large-M keeps 4. The // physical weight layout stride below is INDEPENDENT of this (see BLOCK_STRIDE_A). uint nsg = get_local_size(1); + MATERIALIZE_WG(nsg); uint K = ne00; uint M = ne01; @@ -400,11 +409,14 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_glu( uint gid = get_global_id(0); ushort slid = get_sub_group_local_id(); uint nsg = get_local_size(1); + MATERIALIZE_WG(nsg); uint K = ne00; uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = 4 * M; private uint4 regA; @@ -424,9 +436,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_glu( for (uint k = groupId; k < (K / 32); k += nsg) { \ uint sb = k / 8; \ uint j = k % 8; \ - half2 d = DD[gid + sb * LINE_STRIDE_A]; \ - half2 dm = MM[gid + sb * LINE_STRIDE_A]; \ - global const uchar * sc0 = SS + sb * 12 * M + 2 * gid; \ + half2 d = DD[gid_s + sb * LINE_STRIDE_A]; \ + half2 dm = MM[gid_s + sb * LINE_STRIDE_A]; \ + global const uchar * sc0 = SS + sb * 12 * M + 2 * gid_s; \ global const uchar * sc1 = sc0 + 1; \ uchar sv0, mn0, sv1, mn1; \ get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); \ @@ -437,15 +449,15 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_glu( regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); \ regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); \ } \ - regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; \ - regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; \ - regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; \ - regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; \ + regA.s0 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; \ + regA.s1 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; \ + regA.s2 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; \ + regA.s3 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; \ DEQ_HI(SUM, as_ushort8(regA), regS, regM, regB); \ - regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; \ - regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; \ - regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; \ - regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; \ + regA.s0 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; \ + regA.s1 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; \ + regA.s2 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; \ + regA.s3 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; \ DEQ_LO(SUM, as_ushort8(regA), regS, regM, regB); \ } @@ -479,8 +491,10 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_glu( upSum += p.zw; } dst = (global float*)((global char*)dst + offsetd); - dst[gid * 2 + 0] = glu_apply(glu_op, gateSum.s0, upSum.s0); - dst[gid * 2 + 1] = glu_apply(glu_op, gateSum.s1, upSum.s1); + if (gid < LINE_STRIDE_A) { + dst[gid * 2 + 0] = glu_apply(glu_op, gateSum.s0, upSum.s0); + dst[gid * 2 + 1] = glu_apply(glu_op, gateSum.s1, upSum.s1); + } } } @@ -514,6 +528,7 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk( uint gid = get_global_id(0); ushort slid = get_sub_group_local_id(); uint nsg = get_local_size(1); + MATERIALIZE_WG(nsg); uint ksplit = get_num_groups(1); uint kslice = get_group_id(1); @@ -522,6 +537,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk( uint LINE_STRIDE_A = M / 2; uint BLOCK_STRIDE_A = 4 * M; // physical, independent of the K-split + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + private uint4 regA; private half2 regS, regM; private float8 regB; @@ -531,9 +549,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk( for (uint k = kslice * nsg + groupId; k < (K / 32); k += ksplit * nsg) { uint sb = k / 8; uint j = k % 8; - half2 d = src0_d[gid + sb * LINE_STRIDE_A]; - half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; - global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid; + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; + global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid_s; global const uchar * sc1 = sc0 + 1; uchar sv0, mn0, sv1, mn1; get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); @@ -544,19 +562,19 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk( regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); } - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB); #endif - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB); #else @@ -573,7 +591,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk( for (uint i = 0; i < nsg - 1; ++i) { totalSum += reduceLM[SUBGROUP_SIZE * i + slid]; } - vstore2(totalSum, 0, &(partial[kslice * M + gid * 2])); + // Keep tail stores inside this K-slice. + if (gid * 2 + 0 < M) partial[kslice * M + gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) partial[kslice * M + gid * 2 + 1] = totalSum.s1; } } @@ -678,6 +698,8 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_mc3( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; uint COL_STRIDE = K / 4; // float4 pixels per activation column @@ -702,10 +724,10 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_mc3( uint sb = k / 8; uint j = k % 8; - half2 d = src0_d[gid + sb * LINE_STRIDE_A]; - half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; - global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid; + global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid_s; global const uchar * sc1 = sc0 + 1; uchar sv0, mn0, sv1, mn1; @@ -716,14 +738,14 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_mc3( regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); // weights loaded ONCE, reused across the 3 columns - regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; - regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef Q4K_MC3_DEQUANT_ONCE // Dequant the 32 weights/row (16 hi + 16 lo) ONCE into half2[] (byte- @@ -816,8 +838,10 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_mc3( acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; dst = (global float*)((global char*)dst + offsetd); // dst is column-major [M rows x 3 cols]: (row, col) at col*M + row - vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); - vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); - vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (gid < LINE_STRIDE_A) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + } } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl index 02916bb91..2c1c659a3 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl @@ -231,8 +231,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_o4( // Two consecutive pair-indices (each the same access pattern the 2-output // kernel uses); together they cover 4 consecutive output rows. - uint gid_a = gid * 2; - uint gid_b = gid * 2 + 1; + // Keep tail lanes active for subgroup ops, but read valid weight pairs. + uint gid_a = min(gid * 2, (uint)(ne01 / 2) - 1u); + uint gid_b = min(gid * 2 + 1, (uint)(ne01 / 2) - 1u); uint K = ne00; uint M = ne01; diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl index 7dbf5a3bb..d8537e158 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl @@ -209,6 +209,8 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; private uint4 regA; @@ -218,13 +220,13 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32( private float2 totalSum = (float2)(0.0f); for (uint k = groupId; k < (K / QK5_0); k += NSUBGROUPS) { - regS = src0_d[gid + k * LINE_STRIDE_A]; + regS = src0_d[gid_s + k * LINE_STRIDE_A]; ushort4 qh_raw; - qh_raw.s0 = src0_qh[gid + (4*k + 0) * LINE_STRIDE_A]; - qh_raw.s1 = src0_qh[gid + (4*k + 1) * LINE_STRIDE_A]; - qh_raw.s2 = src0_qh[gid + (4*k + 2) * LINE_STRIDE_A]; - qh_raw.s3 = src0_qh[gid + (4*k + 3) * LINE_STRIDE_A]; + qh_raw.s0 = src0_qh[gid_s + (4*k + 0) * LINE_STRIDE_A]; + qh_raw.s1 = src0_qh[gid_s + (4*k + 1) * LINE_STRIDE_A]; + qh_raw.s2 = src0_qh[gid_s + (4*k + 2) * LINE_STRIDE_A]; + qh_raw.s3 = src0_qh[gid_s + (4*k + 3) * LINE_STRIDE_A]; uchar8 raw = as_uchar8(qh_raw); uchar8 qh_bytes = (uchar8)(raw.s0, raw.s2, raw.s4, raw.s6, @@ -236,10 +238,10 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32( regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); } - regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regB); @@ -247,10 +249,10 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32( dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regB); #endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regB); #else diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl index ba0e2a711..82b4c741e 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl @@ -210,6 +210,8 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; __private uint4 regA; @@ -220,14 +222,14 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32( __private float2 totalSum = (float2)(0.0f); for (uint k = groupId; k < (K / QK5_1); k += NSUBGROUPS) { - regS = src0_d[gid + k * LINE_STRIDE_A]; - regM = src0_m[gid + k * LINE_STRIDE_A]; + regS = src0_d[gid_s + k * LINE_STRIDE_A]; + regM = src0_m[gid_s + k * LINE_STRIDE_A]; ushort4 qh_raw; - qh_raw.s0 = src0_qh[gid + (4*k + 0) * LINE_STRIDE_A]; - qh_raw.s1 = src0_qh[gid + (4*k + 1) * LINE_STRIDE_A]; - qh_raw.s2 = src0_qh[gid + (4*k + 2) * LINE_STRIDE_A]; - qh_raw.s3 = src0_qh[gid + (4*k + 3) * LINE_STRIDE_A]; + qh_raw.s0 = src0_qh[gid_s + (4*k + 0) * LINE_STRIDE_A]; + qh_raw.s1 = src0_qh[gid_s + (4*k + 1) * LINE_STRIDE_A]; + qh_raw.s2 = src0_qh[gid_s + (4*k + 2) * LINE_STRIDE_A]; + qh_raw.s3 = src0_qh[gid_s + (4*k + 3) * LINE_STRIDE_A]; uchar8 raw = as_uchar8(qh_raw); uchar8 qh_bytes = (uchar8)(raw.s0, raw.s2, raw.s4, raw.s6, @@ -239,10 +241,10 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32( regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); } - regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB); @@ -250,10 +252,10 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32( dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB); #endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB); #else diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl index ae864b19b..6570167c9 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl @@ -232,6 +232,8 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; uint LINE_STRIDE_A_QH = M / 2; @@ -250,11 +252,11 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( uint sb = k / 8; uint j = k % 8; - half2 d = src0_d[gid + sb * LINE_STRIDE_A]; - half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; - global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12; - global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12; + global const uchar * sc0 = src0_s + 2 * gid_s * scales_per_row + sb * 12; + global const uchar * sc1 = src0_s + (2 * gid_s + 1) * scales_per_row + sb * 12; uchar sv0, mn0, sv1, mn1; get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); @@ -268,25 +270,25 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); } - regH.s0 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x); - regH.s1 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x); - regH.s2 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x); - regH.s3 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x); + regH.s0 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x); + regH.s1 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x); + regH.s2 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x); + regH.s3 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x); - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB); #endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB); #else @@ -373,6 +375,8 @@ kernel void kernel_gemv_noshuffle_q5_k_f32_mc3( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; uint LINE_STRIDE_A_QH = M / 2; uint BLOCK_STRIDE_A_QH = NSUBGROUPS * M / 2; @@ -393,11 +397,11 @@ kernel void kernel_gemv_noshuffle_q5_k_f32_mc3( uint sb = k / 8; uint j = k % 8; - half2 d = src0_d[gid + sb * LINE_STRIDE_A]; - half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; - global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12; - global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12; + global const uchar * sc0 = src0_s + 2 * gid_s * scales_per_row + sb * 12; + global const uchar * sc1 = src0_s + (2 * gid_s + 1) * scales_per_row + sb * 12; uchar sv0, mn0, sv1, mn1; get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); @@ -407,19 +411,19 @@ kernel void kernel_gemv_noshuffle_q5_k_f32_mc3( regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); // high-bit plane + weights loaded ONCE, reused across the columns - regH.s0 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x); - regH.s1 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x); - regH.s2 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x); - regH.s3 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x); - - regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; - regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regH.s0 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x); + regH.s1 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x); + regH.s2 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x); + regH.s3 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x); + + regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; MC_COL_Q5K(ts0, 0); MC_COL_Q5K(ts1, 1); @@ -442,10 +446,12 @@ kernel void kernel_gemv_noshuffle_q5_k_f32_mc3( acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; dst = (global float*)((global char*)dst + offsetd); // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row - vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); - vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); - if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); - if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + if (gid < LINE_STRIDE_A) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } } } #undef MC_COL_Q5K diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl index 32624ac86..a6908a6a7 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl @@ -219,26 +219,28 @@ kernel void kernel_gemv_noshuffle_q6_K_f32( float2 total_sum = 0.0f; int line_stride_a = ne01 / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + int gid_s = min(gid, line_stride_a - 1); int block_stride_a = NSUBGROUPS * ne01; for (int k = grp; k < nb; k += NSUBGROUPS) { - reg_d = src0_d[gid + k/8 * line_stride_a]; - reg_s = as_char4(src0_s[gid + k * line_stride_a]); + reg_d = src0_d[gid_s + k/8 * line_stride_a]; + reg_s = as_char4(src0_s[gid_s + k * line_stride_a]); if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0 + slid*2 + k*8); reg_b.s4567 = read_imagef(src1, 1 + slid*2 + k*8); } - reg_a_l.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*0).x; - reg_a_l.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*1).x; - reg_a_l.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*2).x; - reg_a_l.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*3).x; + reg_a_l.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*0).x; + reg_a_l.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*1).x; + reg_a_l.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*2).x; + reg_a_l.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*3).x; - reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*0).x); - reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*1).x); - reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*2).x); - reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*3).x); + reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*0).x); + reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*1).x); + reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*2).x); + reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*3).x); #ifdef VECTOR_SUB_GROUP_BROADCAT dequantize_block_acc_bcast_8_hi(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b); @@ -246,15 +248,15 @@ kernel void kernel_gemv_noshuffle_q6_K_f32( dequantize_block_acc_bcast_1_hi(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b); #endif // VECTOR_SUB_GROUP_BROADCAT - reg_a_l.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*4).x; - reg_a_l.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*5).x; - reg_a_l.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*6).x; - reg_a_l.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*7).x; + reg_a_l.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*4).x; + reg_a_l.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*5).x; + reg_a_l.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*6).x; + reg_a_l.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*7).x; - reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*4).x); - reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*5).x); - reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*6).x); - reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*7).x); + reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*4).x); + reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*5).x); + reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*6).x); + reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*7).x); #ifdef VECTOR_SUB_GROUP_BROADCAT dequantize_block_acc_bcast_8_lo(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b); @@ -323,6 +325,8 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_mc3( int nb = ne00 / 32; int line_stride_a = ne01 / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + int gid_s = min(gid, line_stride_a - 1); int block_stride_a = NSUBGROUPS * ne01; int COL_STRIDE = ne00 / 4; // float4 pixels per activation column @@ -335,27 +339,27 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_mc3( float2 ts0 = 0.0f, ts1 = 0.0f, ts2 = 0.0f; for (int k = grp; k < nb; k += NSUBGROUPS) { - reg_d = src0_d[gid + k/8 * line_stride_a]; - reg_s = as_char4(src0_s[gid + k * line_stride_a]); + reg_d = src0_d[gid_s + k/8 * line_stride_a]; + reg_s = as_char4(src0_s[gid_s + k * line_stride_a]); // weights loaded ONCE (hi: blocks 0-3, lo: blocks 4-7), reused x3 cols - ql_hi.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*0).x; - ql_hi.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*1).x; - ql_hi.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*2).x; - ql_hi.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*3).x; - qh_hi.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*0).x); - qh_hi.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*1).x); - qh_hi.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*2).x); - qh_hi.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*3).x); - - ql_lo.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*4).x; - ql_lo.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*5).x; - ql_lo.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*6).x; - ql_lo.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*7).x; - qh_lo.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*4).x); - qh_lo.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*5).x); - qh_lo.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*6).x); - qh_lo.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*7).x); + ql_hi.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*0).x; + ql_hi.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*1).x; + ql_hi.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*2).x; + ql_hi.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*3).x; + qh_hi.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*0).x); + qh_hi.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*1).x); + qh_hi.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*2).x); + qh_hi.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*3).x); + + ql_lo.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*4).x; + ql_lo.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*5).x; + ql_lo.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*6).x; + ql_lo.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*7).x; + qh_lo.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*4).x); + qh_lo.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*5).x); + qh_lo.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*6).x); + qh_lo.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*7).x); // Per-column: load only this column's activation (single reg_b live) -> // 1/3 the activation register pressure, cutting the private-mem spill. @@ -402,8 +406,10 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_mc3( acc += reduce_lm[SUBGROUP_SIZE*2 + slid]; dst = (global float*)((global char*)dst + offsetd); // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row - vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0*ne01 + gid*2])); - vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1*ne01 + gid*2])); - vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2*ne01 + gid*2])); + if (gid < line_stride_a) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0*ne01 + gid*2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1*ne01 + gid*2])); + vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2*ne01 + gid*2])); + } } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl index 84447e61b..22efb6c3c 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl @@ -249,8 +249,9 @@ kernel void Q6K_O4_NAME( // ne01/4 apart) is slower because two distant cache-line streams have worse // locality than the adjacent pair whose reads interleave into the same lines // each iteration. - int gid_a = gid * 2; - int gid_b = gid * 2 + 1; + // Keep tail lanes active for subgroup ops, but read valid weight pairs. + int gid_a = min(gid * 2, ne01 / 2 - 1); + int gid_b = min(gid * 2 + 1, ne01 / 2 - 1); int nb = ne00 / 32; diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl index c5049f396..a9ddc1130 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl @@ -104,11 +104,11 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled( } } -// Multi-column (N=3) variant of the tiled q6_K decode GEMV, for the speculative/ -// MTP VERIFY lm_head/embed (ne1=3 = 2 drafts + 1 bonus). Identical tiled weight -// layout + unpack as the ne1=1 kernel above; each WI computes 3 output columns, +// Multi-column (ne1 in [2..4]) variant of the tiled q6_K decode GEMV, for the +// speculative/MTP VERIFY lm_head/embed (drafts + 1 bonus). Identical tiled weight +// layout + unpack as the ne1=1 kernel above; each WI computes up to 4 output columns, // streaming the (large) lm_head weight ONCE per superblock and reusing it across -// the 3 verify activation columns (dequant once per code, MAC into 3 accs). This +// the verify activation columns (dequant once per code, MAC into 4 accs). This // is the lm_head analogue of the per-layer mc3 GEMV; the multiply order matches // the ne1=1 kernel, so each column is byte-identical to a standalone tiled GEMV. #if defined(ADRENO_GPU) @@ -123,7 +123,8 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( global float * dst, ulong offsetd, int ne00, - int ne01 + int ne01, + int ne1 ) { int grp = get_local_id(1); int row = get_global_id(0); @@ -133,7 +134,12 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( int nb = ne00 / 256; int col_stride = ne00 / 4; // activation float4 pixels per column - float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f; + // columns past ne1 re-read the last column and are not stored + int col1 = min(1, ne1 - 1) * col_stride; + int col2 = min(2, ne1 - 1) * col_stride; + int col3 = min(3, ne1 - 1) * col_stride; + + float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f, acc3 = 0.0f; for (int sb = grp; sb < nb; sb += NSUBGROUPS) { int tile_blk = rt * nb + sb; @@ -157,9 +163,10 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( int act_base = sb * 64; #pragma unroll for (int e4 = 0; e4 < 64; ++e4) { - float4 a0 = read_imagef(src1, 0*col_stride + act_base + e4); - float4 a1 = read_imagef(src1, 1*col_stride + act_base + e4); - float4 a2 = read_imagef(src1, 2*col_stride + act_base + e4); + float4 a0 = read_imagef(src1, act_base + e4); + float4 a1 = read_imagef(src1, col1 + act_base + e4); + float4 a2 = read_imagef(src1, col2 + act_base + e4); + float4 a3 = read_imagef(src1, col3 + act_base + e4); #pragma unroll for (int t = 0; t < 4; ++t) { int e = e4 * 4 + t; @@ -171,15 +178,17 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( float av0 = (t == 0) ? a0.x : (t == 1) ? a0.y : (t == 2) ? a0.z : a0.w; float av1 = (t == 0) ? a1.x : (t == 1) ? a1.y : (t == 2) ? a1.z : a1.w; float av2 = (t == 0) ? a2.x : (t == 1) ? a2.y : (t == 2) ? a2.z : a2.w; + float av3 = (t == 0) ? a3.x : (t == 1) ? a3.y : (t == 2) ? a3.z : a3.w; acc0 += w * av0; acc1 += w * av1; acc2 += w * av2; + acc3 += w * av3; } } } local float4 reduce_lm[NSUBGROUPS * TILE_ROWS]; - reduce_lm[grp * TILE_ROWS + rit] = (float4)(acc0, acc1, acc2, 0.0f); + reduce_lm[grp * TILE_ROWS + rit] = (float4)(acc0, acc1, acc2, acc3); barrier(CLK_LOCAL_MEM_FENCE); if (grp == 0) { @@ -188,9 +197,10 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( + reduce_lm[2 * TILE_ROWS + rit] + reduce_lm[3 * TILE_ROWS + rit]; dst = (global float*)((global char*)dst + offsetd); - // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row + // dst column-major [ne01 rows x ne1 cols]: (row, col) at col*ne01 + row dst[0*ne01 + row] = total.x; dst[1*ne01 + row] = total.y; - dst[2*ne01 + row] = total.z; + if (ne1 > 2) dst[2*ne01 + row] = total.z; + if (ne1 > 3) dst[3*ne01 + row] = total.w; } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl index 6f6d7425c..3c854c32a 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl @@ -155,6 +155,8 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32_splitk( uint M = ne01; uint LINE_STRIDE_A = M; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = 8 * M; // physical, independent of the K-split __private uint8 regA; @@ -164,19 +166,19 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32_splitk( #pragma unroll 1 for (uint k = kslice * nsg + groupId; k < (K / QK8_0); k += ksplit * nsg) { - regS = src0_d[gid + k * LINE_STRIDE_A]; + regS = src0_d[gid_s + k * LINE_STRIDE_A]; if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); } - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; - regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s4 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s5 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s6 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s7 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB); } @@ -199,6 +201,88 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32_splitk( } } +// Multi-column (n_cols in [2..4]) variant of the decode GEMV below, for the spec/MTP +// verify batch: each block's weights are loaded once and reused across the columns. +// Per column the operand order matches the decode GEMV. +#define MC_COL_Q8_0(ts, c) \ + { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \ + regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \ + dequantizeBlockAccum_ns_sgbroadcast_1(ts, regA, scale, regB); } +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q8_0_f32_mc( + __read_only image1d_buffer_t src0_q, // quantized A + global half * src0_d, // A scales + __read_only image1d_buffer_t src1, // B, n_cols columns of K + global float * dst, // C, column-major [M x n_cols] + ulong offsetd, + int ne00, // K + int ne01, // M + int n_cols) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = 8 * M; + uint COL_STRIDE = K / 4; // float4 pixels per activation column + + __private uint8 regA; + __private float8 regB; + + __private float ts0 = 0.0f; + __private float ts1 = 0.0f; + __private float ts2 = 0.0f; + __private float ts3 = 0.0f; + + #pragma unroll 1 + for (uint k = groupId; k < (K / QK8_0); k += N_SIMDGROUP) { + float scale = convert_float(src0_d[gid_s + k * LINE_STRIDE_A]); + + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s4 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s5 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s6 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s7 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + MC_COL_Q8_0(ts0, 0); + MC_COL_Q8_0(ts1, 1); + if (n_cols > 2) MC_COL_Q8_0(ts2, 2); + if (n_cols > 3) MC_COL_Q8_0(ts3, 3); + } + + // reduction in local memory, assumes #wave=4 + __local float4 reduceLM[SIMDGROUP_WIDTH * 3]; + float4 acc = (float4)(ts0, ts1, ts2, ts3); + if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = acc; + if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = acc; + if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = acc; + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) { + acc += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + acc += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + acc += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + if (gid < M) { + dst[0 * M + gid] = acc.s0; + dst[1 * M + gid] = acc.s1; + if (n_cols > 2) dst[2 * M + gid] = acc.s2; + if (n_cols > 3) dst[3 * M + gid] = acc.s3; + } + } +} +#undef MC_COL_Q8_0 + #ifdef ADRENO_GPU REQD_SUBGROUP_SIZE_64 #endif @@ -227,6 +311,8 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32( uint M = ne01; uint LINE_STRIDE_A = M; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = 8 * M; // 32 / 4 = 8 __private uint8 regA; @@ -238,7 +324,7 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32( // loop along K in block granularity, skip 4 blocks every iter #pragma unroll 1 /* tell compiler not to unroll */ for (uint k = groupId; k < (K / QK8_0); k += N_SIMDGROUP) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of one rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads the scale of one row // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -246,14 +332,14 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32( } // load weights for one block in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; - regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s4 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s5 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s6 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s7 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB); } diff --git a/ggml/src/ggml-opencl/kernels/get_rows.cl b/ggml/src/ggml-opencl/kernels/get_rows.cl index 9ae4fff09..f060e46ae 100644 --- a/ggml/src/ggml-opencl/kernels/get_rows.cl +++ b/ggml/src/ggml-opencl/kernels/get_rows.cl @@ -106,6 +106,53 @@ kernel void kernel_get_rows_f32( } } +kernel void kernel_get_rows_i32( + global void * src0, + ulong offset0, + global int * src1, + ulong offset1, + global int * dst, + ulong offsetd, + int ne00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global void*)((global char*)src0 + offset0); + src1 = (global int*)((global char*)src1 + offset1); + dst = (global int*)((global char*)dst + offsetd); + + int nchunks = get_num_groups(0) / ne10; + int g = get_group_id(0); + int i10 = g / nchunks; + int chunk = g - i10 * nchunks; + int i11 = get_group_id(1); + int i12 = get_group_id(2); + + int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0]; + + int i02 = i11; + int i03 = i12; + + global int * dst_row = (global int *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1); + global int * src_row = (global int *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03); + + int span = (ne00 + nchunks - 1) / nchunks; + int start = chunk * span; + int end = min(start + span, ne00); + + for (int ind = start + get_local_id(0); ind < end; ind += get_local_size(0)) { + dst_row[ind] = src_row[ind]; + } +} + kernel void kernel_get_rows_f16( global void * src0, ulong offset0, diff --git a/ggml/src/ggml-opencl/kernels/im2col_3d_f16.cl b/ggml/src/ggml-opencl/kernels/im2col_3d_f16.cl new file mode 100644 index 000000000..a165e4460 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/im2col_3d_f16.cl @@ -0,0 +1,90 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +kernel void kernel_im2col_3d_f16( + global float * src1, + ulong offset1, + global half * dst, + ulong offsetd, + long IC, + long ID, + long IH, + long IW, + long KD, + long KH, + long KW, + long OD, + long OH, + long OW, + long N, + long stride_x, + long stride_y, + long stride_z, + long stride_q, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2 +) { + const int ic = (int)IC; + const int kd = (int)KD; + const int kh = (int)KH; + const int kw = (int)KW; + const int od = (int)OD; + const int oh = (int)OH; + const int ow = (int)OW; + const int n = (int)N; + + const int KD_KH_KW = kd * kh * kw; + const int KH_KW = kh * kw; + const int IC_KD_KH_KW = ic * KD_KH_KW; + + const int i = (int)get_global_id(0); + if (i >= IC_KD_KH_KW) { + return; + } + + const int iow = (int)get_global_id(1); + if (iow >= ow) { + return; + } + + const int OD_OH = od * oh; + const int iz = (int)get_global_id(2); + if (iz >= n * OD_OH) { + return; + } + + src1 = (global float*)((global char*)src1 + offset1); + dst = (global half*)((global char*)dst + offsetd); + + const int iic = i / KD_KH_KW; + const int ikd = (i - iic * KD_KH_KW) / KH_KW; + const int ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / kw; + const int ikw = i % kw; + + const int in = iz / OD_OH; + const int iod = (iz - in * OD_OH) / oh; + const int ioh = iz % oh; + + const int iiw = iow * s0 + ikw * d0 - p0; + const int iih = ioh * s1 + ikh * d1 - p1; + const int iid = iod * s2 + ikd * d2 - p2; + + const long offset_dst = + (long)(iz * ow + iow) * (long)IC_KD_KH_KW + + (long)(iic * KD_KH_KW + ikd * KH_KW + ikh * kw + ikw); + + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) { + dst[offset_dst] = 0.0f; + } else { + const long offset_src = + (long)(in * ic + iic) * stride_q + (long)iid * stride_z + + (long)iih * stride_y + (long)iiw * stride_x; + dst[offset_dst] = convert_half_rte(src1[offset_src]); + } +} diff --git a/ggml/src/ggml-opencl/kernels/im2col_3d_f32.cl b/ggml/src/ggml-opencl/kernels/im2col_3d_f32.cl new file mode 100644 index 000000000..ea94d5a5d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/im2col_3d_f32.cl @@ -0,0 +1,90 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +kernel void kernel_im2col_3d_f32( + global float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + long IC, + long ID, + long IH, + long IW, + long KD, + long KH, + long KW, + long OD, + long OH, + long OW, + long N, + long stride_x, + long stride_y, + long stride_z, + long stride_q, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2 +) { + const int ic = (int)IC; + const int kd = (int)KD; + const int kh = (int)KH; + const int kw = (int)KW; + const int od = (int)OD; + const int oh = (int)OH; + const int ow = (int)OW; + const int n = (int)N; + + const int KD_KH_KW = kd * kh * kw; + const int KH_KW = kh * kw; + const int IC_KD_KH_KW = ic * KD_KH_KW; + + const int i = (int)get_global_id(0); + if (i >= IC_KD_KH_KW) { + return; + } + + const int iow = (int)get_global_id(1); + if (iow >= ow) { + return; + } + + const int OD_OH = od * oh; + const int iz = (int)get_global_id(2); + if (iz >= n * OD_OH) { + return; + } + + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + const int iic = i / KD_KH_KW; + const int ikd = (i - iic * KD_KH_KW) / KH_KW; + const int ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / kw; + const int ikw = i % kw; + + const int in = iz / OD_OH; + const int iod = (iz - in * OD_OH) / oh; + const int ioh = iz % oh; + + const int iiw = iow * s0 + ikw * d0 - p0; + const int iih = ioh * s1 + ikh * d1 - p1; + const int iid = iod * s2 + ikd * d2 - p2; + + const long offset_dst = + (long)(iz * ow + iow) * (long)IC_KD_KH_KW + + (long)(iic * KD_KH_KW + ikd * KH_KW + ikh * kw + ikw); + + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) { + dst[offset_dst] = 0.0f; + } else { + const long offset_src = + (long)(in * ic + iic) * stride_q + (long)iid * stride_z + + (long)iih * stride_y + (long)iiw * stride_x; + dst[offset_dst] = src1[offset_src]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/iq1_s_grid.cl b/ggml/src/ggml-opencl/kernels/iq1_s_grid.cl new file mode 100644 index 000000000..df28039e0 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq1_s_grid.cl @@ -0,0 +1,259 @@ +// IQ1 codebook: each byte packs grid[j] + 1 and grid[j + 4] + 1. +constant uint iq1_s_grid[2048] = { + 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000, + 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101, + 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200, + 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212, + 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011, + 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111, + 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220, + 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022, + 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220, + 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101, + 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110, + 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111, + 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010, + 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210, + 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221, + 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021, + 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002, + 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101, + 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101, + 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211, + 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110, + 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022, + 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121, + 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220, + 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001, + 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101, + 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102, + 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012, + 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010, + 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111, + 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122, + 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222, + 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001, + 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102, + 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101, + 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000, + 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101, + 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112, + 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110, + 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211, + 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012, + 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111, + 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120, + 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122, + 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121, + 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221, + 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001, + 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101, + 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101, + 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011, + 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111, + 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011, + 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122, + 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121, + 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222, + 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101, + 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000, + 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200, + 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110, + 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112, + 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222, + 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021, + 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121, + 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201, + 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200, + 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101, + 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011, + 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010, + 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211, + 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121, + 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000, + 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202, + 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202, + 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211, + 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112, + 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020, + 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121, + 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222, + 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102, + 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100, + 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110, + 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011, + 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111, + 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110, + 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121, + 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222, + 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201, + 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102, + 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201, + 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012, + 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010, + 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010, + 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110, + 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011, + 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212, + 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021, + 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021, + 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021, + 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101, + 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101, + 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100, + 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010, + 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111, + 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010, + 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111, + 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120, + 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120, + 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101, + 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001, + 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201, + 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210, + 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211, + 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111, + 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112, + 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211, + 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010, + 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021, + 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122, + 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221, + 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102, + 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100, + 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101, + 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101, + 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101, + 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012, + 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110, + 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112, + 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210, + 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210, + 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210, + 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010, + 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110, + 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122, + 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020, + 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021, + 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022, + 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120, + 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222, + 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221, + 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001, + 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102, + 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201, + 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012, + 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111, + 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012, + 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110, + 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110, + 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121, + 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221, + 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220, + 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222, + 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000, + 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201, + 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012, + 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011, + 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212, + 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221, + 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121, + 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202, + 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202, + 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002, + 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101, + 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210, + 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112, + 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011, + 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011, + 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210, + 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020, + 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220, + 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222, + 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222, + 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001, + 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010, + 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111, + 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010, + 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110, + 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221, + 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122, + 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202, + 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100, + 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101, + 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112, + 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111, + 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211, + 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222, + 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221, + 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022, + 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101, + 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211, + 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111, + 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111, + 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010, + 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121, + 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222, + 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000, + 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202, + 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000, + 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202, + 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110, + 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110, + 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222, + 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120, + 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022, + 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101, + 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202, + 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110, + 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110, + 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111, + 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111, + 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120, + 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121, + 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001, + 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202, + 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001, + 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200, + 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011, + 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212, + 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012, + 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110, + 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012, + 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111, + 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020, + 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121, + 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222, + 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102, + 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102, + 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101, + 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212, + 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210, + 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111, + 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212, + 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221, + 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121, + 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002, + 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000, + 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202, + 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112, + 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111, + 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020, + 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221, + 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022, + 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100, + 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201, + 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112, + 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211, + 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012, + 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121, + 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020, + 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120, + 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200, + 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200, + 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110, + 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011, + 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222, + 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020, + 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222 +}; diff --git a/ggml/src/ggml-opencl/kernels/iq2_s_grid.cl b/ggml/src/ggml-opencl/kernels/iq2_s_grid.cl new file mode 100644 index 000000000..e06665746 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq2_s_grid.cl @@ -0,0 +1,259 @@ +// IQ2_S codebook, stored as pairs of four packed magnitudes. +constant uint iq2_s_grid[2048] = { + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x192b192b, 0x08080808, + 0x192b2b19, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, + 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, 0x2b191908, 0x08080808, 0x2b2b0808, 0x08080808, + 0x2b2b1919, 0x08080808, 0x2b2b2b2b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, + 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, 0x0819082b, 0x08080819, + 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, + 0x19080808, 0x08080819, 0x1908082b, 0x08080819, 0x19081919, 0x08080819, 0x19082b08, 0x08080819, + 0x19190819, 0x08080819, 0x19191908, 0x08080819, 0x1919192b, 0x08080819, 0x19192b19, 0x08080819, + 0x192b0808, 0x08080819, 0x192b1919, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, + 0x2b081908, 0x08080819, 0x2b190808, 0x08080819, 0x2b19082b, 0x08080819, 0x2b191919, 0x08080819, + 0x2b2b0819, 0x08080819, 0x2b2b1908, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, + 0x08081919, 0x0808082b, 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, + 0x082b0808, 0x0808082b, 0x082b2b2b, 0x0808082b, 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, + 0x1908192b, 0x0808082b, 0x19082b19, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b081919, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x2b191908, 0x0808082b, + 0x2b2b082b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x0808192b, 0x08081908, + 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, 0x08191919, 0x08081908, + 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, 0x082b192b, 0x08081908, + 0x082b2b19, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, + 0x19082b08, 0x08081908, 0x19082b2b, 0x08081908, 0x19190819, 0x08081908, 0x19191908, 0x08081908, + 0x1919192b, 0x08081908, 0x19192b19, 0x08081908, 0x192b0808, 0x08081908, 0x192b082b, 0x08081908, + 0x192b1919, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b08192b, 0x08081908, + 0x2b082b19, 0x08081908, 0x2b190808, 0x08081908, 0x2b191919, 0x08081908, 0x2b192b08, 0x08081908, + 0x2b2b0819, 0x08081908, 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, + 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08082b2b, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x0819192b, 0x08081919, 0x08192b19, 0x08081919, 0x082b0808, 0x08081919, + 0x082b1919, 0x08081919, 0x082b2b08, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x1908192b, 0x08081919, 0x19082b19, 0x08081919, 0x19190808, 0x08081919, 0x1919082b, 0x08081919, + 0x19191919, 0x08081919, 0x19192b08, 0x08081919, 0x192b0819, 0x08081919, 0x192b1908, 0x08081919, + 0x2b080808, 0x08081919, 0x2b08082b, 0x08081919, 0x2b081919, 0x08081919, 0x2b082b08, 0x08081919, + 0x2b190819, 0x08081919, 0x2b191908, 0x08081919, 0x2b2b0808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x0808192b, 0x0808192b, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, + 0x08191919, 0x0808192b, 0x19080808, 0x0808192b, 0x19081919, 0x0808192b, 0x19082b08, 0x0808192b, + 0x19190819, 0x0808192b, 0x19191908, 0x0808192b, 0x192b0808, 0x0808192b, 0x2b080819, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b190808, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08190819, 0x08082b08, 0x08191908, 0x08082b08, + 0x0819192b, 0x08082b08, 0x08192b19, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, + 0x082b2b2b, 0x08082b08, 0x19080819, 0x08082b08, 0x19081908, 0x08082b08, 0x1908192b, 0x08082b08, + 0x19082b19, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x19191919, 0x08082b08, + 0x19192b08, 0x08082b08, 0x192b0819, 0x08082b08, 0x192b1908, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b081919, 0x08082b08, 0x2b191908, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, + 0x08081908, 0x08082b19, 0x08190808, 0x08082b19, 0x0819082b, 0x08082b19, 0x08191919, 0x08082b19, + 0x08192b08, 0x08082b19, 0x082b0819, 0x08082b19, 0x19080808, 0x08082b19, 0x19081919, 0x08082b19, + 0x19082b08, 0x08082b19, 0x19190819, 0x08082b19, 0x19191908, 0x08082b19, 0x192b0808, 0x08082b19, + 0x2b080819, 0x08082b19, 0x2b190808, 0x08082b19, 0x08080808, 0x08082b2b, 0x08190819, 0x08082b2b, + 0x08191908, 0x08082b2b, 0x082b082b, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x082b2b2b, 0x08082b2b, + 0x19190808, 0x08082b2b, 0x2b192b19, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, + 0x0808192b, 0x08190808, 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, + 0x08191919, 0x08190808, 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, + 0x082b192b, 0x08190808, 0x19080808, 0x08190808, 0x1908082b, 0x08190808, 0x19081919, 0x08190808, + 0x19082b08, 0x08190808, 0x19190819, 0x08190808, 0x19191908, 0x08190808, 0x1919192b, 0x08190808, + 0x19192b19, 0x08190808, 0x192b0808, 0x08190808, 0x192b082b, 0x08190808, 0x192b1919, 0x08190808, + 0x192b2b08, 0x08190808, 0x2b080819, 0x08190808, 0x2b081908, 0x08190808, 0x2b08192b, 0x08190808, + 0x2b190808, 0x08190808, 0x2b191919, 0x08190808, 0x2b192b08, 0x08190808, 0x2b2b0819, 0x08190808, + 0x2b2b1908, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, 0x08081919, 0x08190819, + 0x08082b08, 0x08190819, 0x08082b2b, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x0819192b, 0x08190819, 0x08192b19, 0x08190819, 0x082b0808, 0x08190819, 0x082b082b, 0x08190819, + 0x082b1919, 0x08190819, 0x082b2b08, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, + 0x1908192b, 0x08190819, 0x19082b19, 0x08190819, 0x19190808, 0x08190819, 0x1919082b, 0x08190819, + 0x19191919, 0x08190819, 0x19192b08, 0x08190819, 0x192b0819, 0x08190819, 0x192b1908, 0x08190819, + 0x2b080808, 0x08190819, 0x2b08082b, 0x08190819, 0x2b081919, 0x08190819, 0x2b082b08, 0x08190819, + 0x2b190819, 0x08190819, 0x2b191908, 0x08190819, 0x08080819, 0x0819082b, 0x08081908, 0x0819082b, + 0x08082b19, 0x0819082b, 0x08190808, 0x0819082b, 0x08191919, 0x0819082b, 0x082b0819, 0x0819082b, + 0x082b1908, 0x0819082b, 0x19080808, 0x0819082b, 0x19081919, 0x0819082b, 0x19190819, 0x0819082b, + 0x19191908, 0x0819082b, 0x2b080819, 0x0819082b, 0x2b081908, 0x0819082b, 0x2b190808, 0x0819082b, + 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, 0x08082b08, 0x08191908, + 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x0819192b, 0x08191908, 0x08192b19, 0x08191908, + 0x082b0808, 0x08191908, 0x082b1919, 0x08191908, 0x082b2b08, 0x08191908, 0x19080819, 0x08191908, + 0x19081908, 0x08191908, 0x1908192b, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x1919082b, 0x08191908, 0x19191919, 0x08191908, 0x19192b08, 0x08191908, 0x192b0819, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x2b08082b, 0x08191908, 0x2b081919, 0x08191908, + 0x2b082b08, 0x08191908, 0x2b190819, 0x08191908, 0x2b191908, 0x08191908, 0x2b2b0808, 0x08191908, + 0x08080819, 0x08191919, 0x08081908, 0x08191919, 0x0808192b, 0x08191919, 0x08082b19, 0x08191919, + 0x08190808, 0x08191919, 0x0819082b, 0x08191919, 0x08191919, 0x08191919, 0x08192b08, 0x08191919, + 0x082b0819, 0x08191919, 0x082b1908, 0x08191919, 0x19080808, 0x08191919, 0x1908082b, 0x08191919, + 0x19081919, 0x08191919, 0x19082b08, 0x08191919, 0x19190819, 0x08191919, 0x19191908, 0x08191919, + 0x192b0808, 0x08191919, 0x2b080819, 0x08191919, 0x2b081908, 0x08191919, 0x2b190808, 0x08191919, + 0x08080808, 0x0819192b, 0x08081919, 0x0819192b, 0x08082b08, 0x0819192b, 0x08190819, 0x0819192b, + 0x08191908, 0x0819192b, 0x082b0808, 0x0819192b, 0x19080819, 0x0819192b, 0x19081908, 0x0819192b, + 0x19190808, 0x0819192b, 0x2b080808, 0x0819192b, 0x2b2b2b2b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x0808192b, 0x08192b08, 0x08082b19, 0x08192b08, 0x08190808, 0x08192b08, + 0x08191919, 0x08192b08, 0x08192b08, 0x08192b08, 0x082b0819, 0x08192b08, 0x19080808, 0x08192b08, + 0x1908082b, 0x08192b08, 0x19081919, 0x08192b08, 0x19082b08, 0x08192b08, 0x19190819, 0x08192b08, + 0x19191908, 0x08192b08, 0x192b0808, 0x08192b08, 0x2b080819, 0x08192b08, 0x2b081908, 0x08192b08, + 0x08080808, 0x08192b19, 0x0808082b, 0x08192b19, 0x08081919, 0x08192b19, 0x08082b08, 0x08192b19, + 0x08190819, 0x08192b19, 0x08191908, 0x08192b19, 0x082b0808, 0x08192b19, 0x19080819, 0x08192b19, + 0x19081908, 0x08192b19, 0x19190808, 0x08192b19, 0x192b2b19, 0x08192b19, 0x2b2b082b, 0x08192b19, + 0x08081908, 0x08192b2b, 0x08190808, 0x08192b2b, 0x19080808, 0x08192b2b, 0x1919192b, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, 0x08082b08, 0x082b0808, + 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, 0x0819192b, 0x082b0808, 0x08192b19, 0x082b0808, + 0x082b0808, 0x082b0808, 0x082b1919, 0x082b0808, 0x082b2b2b, 0x082b0808, 0x19080819, 0x082b0808, + 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, 0x1919082b, 0x082b0808, 0x19191919, 0x082b0808, + 0x192b1908, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b082b2b, 0x082b0808, 0x2b191908, 0x082b0808, + 0x2b2b2b2b, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, 0x08190808, 0x082b0819, + 0x0819082b, 0x082b0819, 0x08191919, 0x082b0819, 0x082b0819, 0x082b0819, 0x19080808, 0x082b0819, + 0x1908082b, 0x082b0819, 0x19081919, 0x082b0819, 0x19190819, 0x082b0819, 0x19191908, 0x082b0819, + 0x192b0808, 0x082b0819, 0x2b080819, 0x082b0819, 0x2b081908, 0x082b0819, 0x2b190808, 0x082b0819, + 0x08080808, 0x082b082b, 0x08082b2b, 0x082b082b, 0x082b082b, 0x082b082b, 0x082b2b08, 0x082b082b, + 0x082b2b2b, 0x082b082b, 0x19081908, 0x082b082b, 0x19190808, 0x082b082b, 0x2b082b08, 0x082b082b, + 0x2b082b2b, 0x082b082b, 0x2b2b2b08, 0x082b082b, 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, + 0x0808192b, 0x082b1908, 0x08082b19, 0x082b1908, 0x08190808, 0x082b1908, 0x08191919, 0x082b1908, + 0x08192b08, 0x082b1908, 0x082b0819, 0x082b1908, 0x082b1908, 0x082b1908, 0x19080808, 0x082b1908, + 0x1908082b, 0x082b1908, 0x19081919, 0x082b1908, 0x19082b08, 0x082b1908, 0x19190819, 0x082b1908, + 0x19191908, 0x082b1908, 0x192b0808, 0x082b1908, 0x2b080819, 0x082b1908, 0x2b081908, 0x082b1908, + 0x2b190808, 0x082b1908, 0x08080808, 0x082b1919, 0x08081919, 0x082b1919, 0x08082b08, 0x082b1919, + 0x08190819, 0x082b1919, 0x08191908, 0x082b1919, 0x082b0808, 0x082b1919, 0x19080819, 0x082b1919, + 0x19081908, 0x082b1919, 0x19190808, 0x082b1919, 0x192b192b, 0x082b1919, 0x2b080808, 0x082b1919, + 0x08080819, 0x082b192b, 0x08081908, 0x082b192b, 0x08190808, 0x082b192b, 0x19080808, 0x082b192b, + 0x19192b19, 0x082b192b, 0x08080808, 0x082b2b08, 0x08081919, 0x082b2b08, 0x08190819, 0x082b2b08, + 0x08191908, 0x082b2b08, 0x19080819, 0x082b2b08, 0x19081908, 0x082b2b08, 0x19190808, 0x082b2b08, + 0x2b082b2b, 0x082b2b08, 0x2b2b2b2b, 0x082b2b08, 0x08080819, 0x082b2b19, 0x08081908, 0x082b2b19, + 0x08190808, 0x082b2b19, 0x2b191919, 0x082b2b19, 0x08082b2b, 0x082b2b2b, 0x082b082b, 0x082b2b2b, + 0x192b1908, 0x082b2b2b, 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, + 0x08081908, 0x19080808, 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, + 0x0819082b, 0x19080808, 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x08192b2b, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x082b192b, 0x19080808, 0x19080808, 0x19080808, + 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, 0x19082b2b, 0x19080808, + 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x1919192b, 0x19080808, 0x19192b19, 0x19080808, + 0x192b0808, 0x19080808, 0x192b082b, 0x19080808, 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, + 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, 0x2b191919, 0x19080808, 0x2b192b08, 0x19080808, + 0x2b2b0819, 0x19080808, 0x2b2b1908, 0x19080808, 0x08080808, 0x19080819, 0x0808082b, 0x19080819, + 0x08081919, 0x19080819, 0x08082b08, 0x19080819, 0x08190819, 0x19080819, 0x08191908, 0x19080819, + 0x0819192b, 0x19080819, 0x08192b19, 0x19080819, 0x082b0808, 0x19080819, 0x082b082b, 0x19080819, + 0x082b1919, 0x19080819, 0x19080819, 0x19080819, 0x19081908, 0x19080819, 0x1908192b, 0x19080819, + 0x19082b19, 0x19080819, 0x19190808, 0x19080819, 0x1919082b, 0x19080819, 0x19191919, 0x19080819, + 0x19192b08, 0x19080819, 0x192b0819, 0x19080819, 0x192b1908, 0x19080819, 0x2b080808, 0x19080819, + 0x2b08082b, 0x19080819, 0x2b081919, 0x19080819, 0x2b082b08, 0x19080819, 0x2b190819, 0x19080819, + 0x2b191908, 0x19080819, 0x2b2b0808, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, + 0x08190808, 0x1908082b, 0x0819082b, 0x1908082b, 0x08191919, 0x1908082b, 0x08192b08, 0x1908082b, + 0x082b1908, 0x1908082b, 0x19080808, 0x1908082b, 0x19081919, 0x1908082b, 0x19082b08, 0x1908082b, + 0x19190819, 0x1908082b, 0x19191908, 0x1908082b, 0x192b0808, 0x1908082b, 0x2b080819, 0x1908082b, + 0x2b081908, 0x1908082b, 0x08080808, 0x19081908, 0x0808082b, 0x19081908, 0x08081919, 0x19081908, + 0x08082b08, 0x19081908, 0x08082b2b, 0x19081908, 0x08190819, 0x19081908, 0x08191908, 0x19081908, + 0x0819192b, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x082b082b, 0x19081908, + 0x082b1919, 0x19081908, 0x082b2b08, 0x19081908, 0x19080819, 0x19081908, 0x19081908, 0x19081908, + 0x1908192b, 0x19081908, 0x19082b19, 0x19081908, 0x19190808, 0x19081908, 0x1919082b, 0x19081908, + 0x19191919, 0x19081908, 0x19192b08, 0x19081908, 0x192b0819, 0x19081908, 0x192b1908, 0x19081908, + 0x2b080808, 0x19081908, 0x2b08082b, 0x19081908, 0x2b081919, 0x19081908, 0x2b082b08, 0x19081908, + 0x2b190819, 0x19081908, 0x2b191908, 0x19081908, 0x2b2b0808, 0x19081908, 0x08080819, 0x19081919, + 0x08081908, 0x19081919, 0x0808192b, 0x19081919, 0x08082b19, 0x19081919, 0x08190808, 0x19081919, + 0x0819082b, 0x19081919, 0x08191919, 0x19081919, 0x08192b08, 0x19081919, 0x082b0819, 0x19081919, + 0x082b1908, 0x19081919, 0x19080808, 0x19081919, 0x1908082b, 0x19081919, 0x19081919, 0x19081919, + 0x19082b08, 0x19081919, 0x19190819, 0x19081919, 0x19191908, 0x19081919, 0x192b0808, 0x19081919, + 0x192b2b2b, 0x19081919, 0x2b080819, 0x19081919, 0x2b081908, 0x19081919, 0x2b190808, 0x19081919, + 0x08080808, 0x1908192b, 0x0808082b, 0x1908192b, 0x08081919, 0x1908192b, 0x08082b08, 0x1908192b, + 0x08190819, 0x1908192b, 0x08191908, 0x1908192b, 0x082b0808, 0x1908192b, 0x19080819, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x2b080808, 0x1908192b, 0x2b2b1919, 0x1908192b, + 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, 0x08082b19, 0x19082b08, 0x08190808, 0x19082b08, + 0x0819082b, 0x19082b08, 0x08191919, 0x19082b08, 0x08192b08, 0x19082b08, 0x082b0819, 0x19082b08, + 0x082b1908, 0x19082b08, 0x19080808, 0x19082b08, 0x1908082b, 0x19082b08, 0x19081919, 0x19082b08, + 0x19082b08, 0x19082b08, 0x19190819, 0x19082b08, 0x19191908, 0x19082b08, 0x192b0808, 0x19082b08, + 0x2b081908, 0x19082b08, 0x2b190808, 0x19082b08, 0x08080808, 0x19082b19, 0x0808082b, 0x19082b19, + 0x08081919, 0x19082b19, 0x08082b08, 0x19082b19, 0x08190819, 0x19082b19, 0x08191908, 0x19082b19, + 0x082b0808, 0x19082b19, 0x19080819, 0x19082b19, 0x19081908, 0x19082b19, 0x19190808, 0x19082b19, + 0x2b080808, 0x19082b19, 0x2b19192b, 0x19082b19, 0x08080819, 0x19082b2b, 0x08081908, 0x19082b2b, + 0x08190808, 0x19082b2b, 0x19080808, 0x19082b2b, 0x08080808, 0x19190808, 0x0808082b, 0x19190808, + 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, 0x08191908, 0x19190808, + 0x0819192b, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x082b082b, 0x19190808, + 0x082b1919, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, 0x19081908, 0x19190808, + 0x1908192b, 0x19190808, 0x19082b19, 0x19190808, 0x19190808, 0x19190808, 0x1919082b, 0x19190808, + 0x19191919, 0x19190808, 0x19192b08, 0x19190808, 0x192b0819, 0x19190808, 0x192b1908, 0x19190808, + 0x2b080808, 0x19190808, 0x2b08082b, 0x19190808, 0x2b081919, 0x19190808, 0x2b082b08, 0x19190808, + 0x2b190819, 0x19190808, 0x2b191908, 0x19190808, 0x08080819, 0x19190819, 0x08081908, 0x19190819, + 0x0808192b, 0x19190819, 0x08082b19, 0x19190819, 0x08190808, 0x19190819, 0x0819082b, 0x19190819, + 0x08191919, 0x19190819, 0x08192b08, 0x19190819, 0x082b0819, 0x19190819, 0x082b1908, 0x19190819, + 0x19080808, 0x19190819, 0x1908082b, 0x19190819, 0x19081919, 0x19190819, 0x19082b08, 0x19190819, + 0x19190819, 0x19190819, 0x19191908, 0x19190819, 0x192b0808, 0x19190819, 0x2b080819, 0x19190819, + 0x2b081908, 0x19190819, 0x2b190808, 0x19190819, 0x08080808, 0x1919082b, 0x08081919, 0x1919082b, + 0x08082b08, 0x1919082b, 0x08190819, 0x1919082b, 0x08191908, 0x1919082b, 0x082b0808, 0x1919082b, + 0x19080819, 0x1919082b, 0x19081908, 0x1919082b, 0x19190808, 0x1919082b, 0x192b2b19, 0x1919082b, + 0x2b080808, 0x1919082b, 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x0808192b, 0x19191908, + 0x08082b19, 0x19191908, 0x08190808, 0x19191908, 0x0819082b, 0x19191908, 0x08191919, 0x19191908, + 0x08192b08, 0x19191908, 0x082b0819, 0x19191908, 0x082b1908, 0x19191908, 0x19080808, 0x19191908, + 0x1908082b, 0x19191908, 0x19081919, 0x19191908, 0x19082b08, 0x19191908, 0x19190819, 0x19191908, + 0x19191908, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b081908, 0x19191908, + 0x2b190808, 0x19191908, 0x08080808, 0x19191919, 0x0808082b, 0x19191919, 0x08081919, 0x19191919, + 0x08082b08, 0x19191919, 0x08190819, 0x19191919, 0x08191908, 0x19191919, 0x082b0808, 0x19191919, + 0x19080819, 0x19191919, 0x19081908, 0x19191919, 0x19190808, 0x19191919, 0x2b080808, 0x19191919, + 0x08080819, 0x1919192b, 0x08081908, 0x1919192b, 0x08190808, 0x1919192b, 0x082b192b, 0x1919192b, + 0x19080808, 0x1919192b, 0x08080808, 0x19192b08, 0x0808082b, 0x19192b08, 0x08081919, 0x19192b08, + 0x08082b08, 0x19192b08, 0x08190819, 0x19192b08, 0x08191908, 0x19192b08, 0x082b0808, 0x19192b08, + 0x19080819, 0x19192b08, 0x19081908, 0x19192b08, 0x19190808, 0x19192b08, 0x19192b2b, 0x19192b08, + 0x2b080808, 0x19192b08, 0x08080819, 0x19192b19, 0x08081908, 0x19192b19, 0x08190808, 0x19192b19, + 0x19080808, 0x19192b19, 0x08080808, 0x19192b2b, 0x08192b19, 0x19192b2b, 0x2b081919, 0x19192b2b, + 0x2b2b2b08, 0x19192b2b, 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x0808192b, 0x192b0808, + 0x08190808, 0x192b0808, 0x0819082b, 0x192b0808, 0x08191919, 0x192b0808, 0x08192b08, 0x192b0808, + 0x082b0819, 0x192b0808, 0x082b1908, 0x192b0808, 0x19080808, 0x192b0808, 0x19081919, 0x192b0808, + 0x19082b08, 0x192b0808, 0x19190819, 0x192b0808, 0x19191908, 0x192b0808, 0x192b0808, 0x192b0808, + 0x2b081908, 0x192b0808, 0x2b190808, 0x192b0808, 0x08080808, 0x192b0819, 0x0808082b, 0x192b0819, + 0x08081919, 0x192b0819, 0x08082b08, 0x192b0819, 0x08190819, 0x192b0819, 0x08191908, 0x192b0819, + 0x082b0808, 0x192b0819, 0x19080819, 0x192b0819, 0x19081908, 0x192b0819, 0x19190808, 0x192b0819, + 0x2b080808, 0x192b0819, 0x2b192b19, 0x192b0819, 0x08081908, 0x192b082b, 0x08190808, 0x192b082b, + 0x19080808, 0x192b082b, 0x1919192b, 0x192b082b, 0x2b2b0819, 0x192b082b, 0x08080808, 0x192b1908, + 0x08081919, 0x192b1908, 0x08082b08, 0x192b1908, 0x08190819, 0x192b1908, 0x08191908, 0x192b1908, + 0x082b0808, 0x192b1908, 0x19080819, 0x192b1908, 0x19081908, 0x192b1908, 0x19190808, 0x192b1908, + 0x2b080808, 0x192b1908, 0x08080819, 0x192b1919, 0x08081908, 0x192b1919, 0x08190808, 0x192b1919, + 0x19080808, 0x192b1919, 0x19082b2b, 0x192b1919, 0x192b2b08, 0x192b1919, 0x2b19082b, 0x192b1919, + 0x08080808, 0x192b192b, 0x2b191908, 0x192b192b, 0x08080819, 0x192b2b08, 0x08081908, 0x192b2b08, + 0x08190808, 0x192b2b08, 0x192b1919, 0x192b2b08, 0x2b192b08, 0x192b2b08, 0x08080808, 0x192b2b19, + 0x082b2b2b, 0x192b2b19, 0x1908082b, 0x192b2b2b, 0x2b2b0819, 0x192b2b2b, 0x08080808, 0x2b080808, + 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, 0x08190819, 0x2b080808, + 0x08191908, 0x2b080808, 0x08192b19, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b1919, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x1919082b, 0x2b080808, + 0x19191919, 0x2b080808, 0x19192b08, 0x2b080808, 0x192b0819, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b081919, 0x2b080808, 0x2b190819, 0x2b080808, 0x2b191908, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x08082b19, 0x2b080819, 0x08190808, 0x2b080819, 0x0819082b, 0x2b080819, + 0x08191919, 0x2b080819, 0x08192b08, 0x2b080819, 0x082b0819, 0x2b080819, 0x082b1908, 0x2b080819, + 0x19080808, 0x2b080819, 0x1908082b, 0x2b080819, 0x19081919, 0x2b080819, 0x19082b08, 0x2b080819, + 0x19190819, 0x2b080819, 0x19191908, 0x2b080819, 0x2b080819, 0x2b080819, 0x2b081908, 0x2b080819, + 0x2b190808, 0x2b080819, 0x2b2b2b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x08081919, 0x2b08082b, + 0x08082b2b, 0x2b08082b, 0x08190819, 0x2b08082b, 0x08191908, 0x2b08082b, 0x19080819, 0x2b08082b, + 0x19081908, 0x2b08082b, 0x19190808, 0x2b08082b, 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, + 0x0808192b, 0x2b081908, 0x08082b19, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x08192b08, 0x2b081908, 0x082b0819, 0x2b081908, 0x19080808, 0x2b081908, + 0x1908082b, 0x2b081908, 0x19081919, 0x2b081908, 0x19082b08, 0x2b081908, 0x19190819, 0x2b081908, + 0x19191908, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b080819, 0x2b081908, 0x2b081908, 0x2b081908, + 0x2b190808, 0x2b081908, 0x08080808, 0x2b081919, 0x0808082b, 0x2b081919, 0x08081919, 0x2b081919, + 0x08082b08, 0x2b081919, 0x08190819, 0x2b081919, 0x08191908, 0x2b081919, 0x082b0808, 0x2b081919, + 0x19080819, 0x2b081919, 0x19081908, 0x2b081919, 0x19190808, 0x2b081919, 0x2b080808, 0x2b081919, + 0x2b082b2b, 0x2b081919, 0x08080819, 0x2b08192b, 0x08081908, 0x2b08192b, 0x08190808, 0x2b08192b, + 0x082b2b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08081919, 0x2b082b08, + 0x08190819, 0x2b082b08, 0x08191908, 0x2b082b08, 0x19080819, 0x2b082b08, 0x19081908, 0x2b082b08, + 0x19190808, 0x2b082b08, 0x2b2b082b, 0x2b082b08, 0x08080819, 0x2b082b19, 0x08081908, 0x2b082b19, + 0x19080808, 0x2b082b19, 0x192b1919, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x19192b08, 0x2b082b2b, + 0x19192b2b, 0x2b082b2b, 0x2b08082b, 0x2b082b2b, 0x2b2b082b, 0x2b082b2b, 0x08080819, 0x2b190808, + 0x08081908, 0x2b190808, 0x08082b19, 0x2b190808, 0x08190808, 0x2b190808, 0x0819082b, 0x2b190808, + 0x08191919, 0x2b190808, 0x08192b08, 0x2b190808, 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, + 0x1908082b, 0x2b190808, 0x19081919, 0x2b190808, 0x19082b08, 0x2b190808, 0x19190819, 0x2b190808, + 0x19191908, 0x2b190808, 0x192b0808, 0x2b190808, 0x2b080819, 0x2b190808, 0x2b081908, 0x2b190808, + 0x2b190808, 0x2b190808, 0x08080808, 0x2b190819, 0x08081919, 0x2b190819, 0x08190819, 0x2b190819, + 0x08191908, 0x2b190819, 0x19080819, 0x2b190819, 0x19081908, 0x2b190819, 0x19190808, 0x2b190819, + 0x19192b2b, 0x2b190819, 0x08080819, 0x2b19082b, 0x08081908, 0x2b19082b, 0x08190808, 0x2b19082b, + 0x19080808, 0x2b19082b, 0x2b2b192b, 0x2b19082b, 0x08080808, 0x2b191908, 0x0808082b, 0x2b191908, + 0x08081919, 0x2b191908, 0x08082b08, 0x2b191908, 0x08190819, 0x2b191908, 0x08191908, 0x2b191908, + 0x082b0808, 0x2b191908, 0x19080819, 0x2b191908, 0x19081908, 0x2b191908, 0x19190808, 0x2b191908, + 0x2b080808, 0x2b191908, 0x2b19192b, 0x2b191908, 0x08080819, 0x2b191919, 0x08081908, 0x2b191919, + 0x08190808, 0x2b191919, 0x19080808, 0x2b191919, 0x2b192b08, 0x2b191919, 0x2b2b0819, 0x2b191919, + 0x08080808, 0x2b19192b, 0x1908192b, 0x2b19192b, 0x192b1908, 0x2b19192b, 0x08080819, 0x2b192b08, + 0x08081908, 0x2b192b08, 0x08190808, 0x2b192b08, 0x082b192b, 0x2b192b08, 0x19080808, 0x2b192b08, + 0x2b2b2b19, 0x2b192b08, 0x08080808, 0x2b192b19, 0x19082b19, 0x2b192b19, 0x1919082b, 0x2b192b19, + 0x2b190808, 0x2b192b2b, 0x08080808, 0x2b2b0808, 0x08081919, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, + 0x08191908, 0x2b2b0808, 0x082b082b, 0x2b2b0808, 0x082b2b2b, 0x2b2b0808, 0x19080819, 0x2b2b0808, + 0x19081908, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b2b082b, 0x2b2b0808, 0x2b2b2b2b, 0x2b2b0808, + 0x19080808, 0x2b2b0819, 0x192b1919, 0x2b2b0819, 0x0808082b, 0x2b2b082b, 0x08082b2b, 0x2b2b082b, + 0x082b082b, 0x2b2b082b, 0x082b2b08, 0x2b2b082b, 0x082b2b2b, 0x2b2b082b, 0x2b08082b, 0x2b2b082b, + 0x2b082b08, 0x2b2b082b, 0x2b082b2b, 0x2b2b082b, 0x2b2b2b08, 0x2b2b082b, 0x08080819, 0x2b2b1908, + 0x08081908, 0x2b2b1908, 0x08190808, 0x2b2b1908, 0x19080808, 0x2b2b1908, 0x2b082b19, 0x2b2b1908, + 0x2b2b1908, 0x2b2b1908, 0x08080808, 0x2b2b1919, 0x08192b19, 0x2b2b1919, 0x19190819, 0x2b2b192b, + 0x08082b2b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b082b, 0x2b2b2b08, 0x19191908, 0x2b2b2b19, + 0x2b08192b, 0x2b2b2b19, 0x08082b08, 0x2b2b2b2b, 0x08082b2b, 0x2b2b2b2b, 0x082b0808, 0x2b2b2b2b, + 0x082b082b, 0x2b2b2b2b, 0x082b2b08, 0x2b2b2b2b, 0x2b082b08, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b +}; diff --git a/ggml/src/ggml-opencl/kernels/iq2_xs_grid.cl b/ggml/src/ggml-opencl/kernels/iq2_xs_grid.cl new file mode 100644 index 000000000..0893713a9 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq2_xs_grid.cl @@ -0,0 +1,135 @@ +// IQ2_XS codebook, stored as pairs of four packed magnitudes. +constant uint iq2_xs_grid[1024] = { + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b +}; + +inline uint iq2_xs_signs(uint code) { + return code | ((popcount(code) & 1u) << 7); +} diff --git a/ggml/src/ggml-opencl/kernels/iq2_xxs_grid.cl b/ggml/src/ggml-opencl/kernels/iq2_xxs_grid.cl new file mode 100644 index 000000000..794e097ef --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq2_xxs_grid.cl @@ -0,0 +1,71 @@ +// IQ2_XXS codebook, stored as pairs of four packed magnitudes. +constant uint iq2_xxs_grid[512] = { + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808, + 0x082b082b, 0x08080808, 0x082b2b08, 0x08080808, 0x082b2b2b, 0x08080808, 0x19080819, 0x08080808, + 0x19081908, 0x08080808, 0x19190808, 0x08080808, 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, + 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b082b2b, 0x08080808, + 0x2b2b082b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, 0x08190808, 0x08080819, + 0x08191919, 0x08080819, 0x19080808, 0x08080819, 0x2b081908, 0x08080819, 0x2b192b08, 0x08080819, + 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x082b082b, 0x0808082b, 0x2b08082b, 0x0808082b, + 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x08190808, 0x08081908, 0x082b0819, 0x08081908, + 0x082b1908, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19082b08, 0x08081908, + 0x192b0808, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, + 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, 0x08082b08, 0x08081919, + 0x082b0808, 0x08081919, 0x1908192b, 0x08081919, 0x192b2b19, 0x08081919, 0x2b080808, 0x08081919, + 0x2b190819, 0x08081919, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, 0x19080808, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b2b1908, 0x0808192b, 0x08080808, 0x08082b08, 0x08081919, 0x08082b08, + 0x08082b08, 0x08082b08, 0x08191908, 0x08082b08, 0x082b2b08, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x2b082b08, 0x08082b08, + 0x08081908, 0x08082b19, 0x19080808, 0x08082b19, 0x0808082b, 0x08082b2b, 0x08191908, 0x08082b2b, + 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x08190808, 0x08190808, 0x082b0819, 0x08190808, + 0x19080808, 0x08190808, 0x192b0808, 0x08190808, 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, + 0x2b191919, 0x08190808, 0x08080808, 0x08190819, 0x08082b08, 0x08190819, 0x082b0808, 0x08190819, + 0x19190808, 0x08190819, 0x19192b2b, 0x08190819, 0x2b080808, 0x08190819, 0x082b1908, 0x0819082b, + 0x19081919, 0x0819082b, 0x08080808, 0x08191908, 0x08082b08, 0x08191908, 0x082b0808, 0x08191908, + 0x082b1919, 0x08191908, 0x19082b19, 0x08191908, 0x2b080808, 0x08191908, 0x08192b08, 0x08191919, + 0x192b082b, 0x08191919, 0x08080808, 0x0819192b, 0x0819192b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, 0x19080808, 0x08192b08, 0x2b080819, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x2b2b0808, 0x08192b19, 0x19190819, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08082b2b, 0x082b0808, 0x19081908, 0x082b0808, + 0x192b0819, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b08082b, 0x082b0808, 0x082b2b19, 0x082b0819, + 0x19082b08, 0x082b0819, 0x08080808, 0x082b082b, 0x0808082b, 0x082b082b, 0x08080819, 0x082b1908, + 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x19080808, 0x082b1908, 0x1919192b, 0x082b1908, + 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x192b1908, 0x082b1919, 0x2b190808, 0x082b192b, + 0x08082b08, 0x082b2b08, 0x082b0808, 0x082b2b08, 0x2b191908, 0x082b2b08, 0x19081908, 0x082b2b2b, + 0x08080819, 0x19080808, 0x08081908, 0x19080808, 0x08190808, 0x19080808, 0x08192b08, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x19080808, 0x19080808, 0x19082b08, 0x19080808, + 0x1919192b, 0x19080808, 0x192b0808, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, + 0x2b190808, 0x19080808, 0x08080808, 0x19080819, 0x082b0808, 0x19080819, 0x192b0819, 0x19080819, + 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, 0x08080819, 0x1908082b, 0x08190808, 0x1908082b, + 0x19082b08, 0x1908082b, 0x1919192b, 0x1908082b, 0x192b2b08, 0x1908082b, 0x08080808, 0x19081908, + 0x08082b08, 0x19081908, 0x082b0808, 0x19081908, 0x2b080808, 0x19081908, 0x2b192b19, 0x19081908, + 0x0819082b, 0x19081919, 0x082b1908, 0x19081919, 0x08080808, 0x1908192b, 0x08080819, 0x19082b08, + 0x08081908, 0x19082b08, 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, + 0x08080808, 0x19082b19, 0x19192b08, 0x19082b19, 0x192b0819, 0x19082b19, 0x2b08082b, 0x19082b19, + 0x19081919, 0x19082b2b, 0x2b190808, 0x19082b2b, 0x08080808, 0x19190808, 0x08082b08, 0x19190808, + 0x08190819, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x2b080808, 0x19190808, + 0x2b082b08, 0x19190808, 0x08081908, 0x19190819, 0x1908082b, 0x19190819, 0x2b2b1908, 0x19190819, + 0x2b190819, 0x1919082b, 0x2b190808, 0x19191908, 0x2b19082b, 0x19191908, 0x08082b2b, 0x19191919, + 0x08080819, 0x1919192b, 0x19191908, 0x1919192b, 0x08080808, 0x19192b08, 0x08190819, 0x19192b08, + 0x08192b19, 0x19192b08, 0x192b1908, 0x19192b08, 0x19080808, 0x19192b19, 0x08082b08, 0x19192b2b, + 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, 0x192b2b08, 0x192b0808, + 0x08080808, 0x192b0819, 0x19191919, 0x192b0819, 0x08192b08, 0x192b082b, 0x192b0808, 0x192b082b, + 0x08080808, 0x192b1908, 0x08081919, 0x192b1908, 0x08190808, 0x192b1919, 0x0819082b, 0x192b1919, + 0x2b081908, 0x192b1919, 0x1908082b, 0x192b2b08, 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, + 0x08082b2b, 0x2b080808, 0x19080819, 0x2b080808, 0x2b08082b, 0x2b080808, 0x08081908, 0x2b080819, + 0x08192b08, 0x2b080819, 0x19080808, 0x2b080819, 0x08190819, 0x2b08082b, 0x08080819, 0x2b081908, + 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, + 0x192b0808, 0x2b081908, 0x08080808, 0x2b081919, 0x1908192b, 0x2b081919, 0x2b191908, 0x2b081919, + 0x08082b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x192b0808, 0x2b08192b, 0x0808082b, 0x2b082b08, + 0x08081908, 0x2b082b19, 0x08190819, 0x2b082b2b, 0x08081908, 0x2b190808, 0x08190808, 0x2b190808, + 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, 0x2b2b0819, 0x2b190808, 0x0819192b, 0x2b190819, + 0x2b080808, 0x2b190819, 0x19081919, 0x2b19082b, 0x08080808, 0x2b191908, 0x082b082b, 0x2b191908, + 0x19081908, 0x2b191908, 0x19190819, 0x2b191919, 0x2b080819, 0x2b192b08, 0x082b0808, 0x2b192b19, + 0x0808082b, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b081919, 0x2b2b0808, 0x08082b19, 0x2b2b0819, + 0x08080808, 0x2b2b082b, 0x08192b08, 0x2b2b1908, 0x19190808, 0x2b2b2b08, 0x08081908, 0x2b2b2b19 +}; + +inline uint iq2_xxs_signs(uint code) { + return code | ((popcount(code) & 1u) << 7); +} diff --git a/ggml/src/ggml-opencl/kernels/iq3_s_grid.cl b/ggml/src/ggml-opencl/kernels/iq3_s_grid.cl new file mode 100644 index 000000000..7ffd2c4fe --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq3_s_grid.cl @@ -0,0 +1,66 @@ +constant uint iq3_s_grid[512] = { + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101 +}; diff --git a/ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl b/ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl new file mode 100644 index 000000000..4c79ddf1a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl @@ -0,0 +1,38 @@ +constant uint iq3_xxs_grid[256] = { + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, + 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, + 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, + 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, + 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, + 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, + 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, + 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, + 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, + 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, + 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, + 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, + 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, + 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, + 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, + 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, + 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, + 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, + 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, + 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, + 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, + 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, + 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, + 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, + 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, + 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, + 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, + 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, + 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04 +}; + +inline uint iq3_xxs_signs(uint code) { + return code | ((popcount(code) & 1u) << 7); +} diff --git a/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl b/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl new file mode 100644 index 000000000..26fb42291 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl @@ -0,0 +1,10 @@ +constant char iq4_xs_values[16] = { + -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, +}; + +inline uint iq4_xs_code(uint n) { + const uint lo = (n & 4u) ? 0xF6EADDCFu : 0xBFAD9881u; + const uint hi = (n & 4u) ? 0x71594535u : 0x26190D01u; + const uint t = (n & 8u) ? hi : lo; + return (t >> (8*(n & 3u))) & 255u; +} diff --git a/ggml/src/ggml-opencl/kernels/lightning_indexer.cl b/ggml/src/ggml-opencl/kernels/lightning_indexer.cl new file mode 100644 index 000000000..efd2f1165 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/lightning_indexer.cl @@ -0,0 +1,111 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#endif + +kernel void kernel_lightning_indexer( + global const char * q, + ulong offset_q, + global const char * k, + ulong offset_k, + global const char * w, + ulong offset_w, + global const char * mask, + ulong offset_m, + global float * dst, + ulong offset_d, + int head_size, + int n_heads, + int n_tokens, + int n_keys, + int n_masks, + int type_k, + ulong nbq1, + ulong nbq2, + ulong nbq3, + ulong nbk2, + ulong nbk3, + ulong nbw1, + ulong nbw3, + ulong nbm1, + ulong nbm3, + int use_shuffle +) { + const int tid = get_local_id(0); + const int lane = tid & (use_shuffle ? 3 : 15); + const int key = use_shuffle ? get_group_id(0)*16 + tid/4 : get_group_id(0)*4 + tid/16; + const int token = get_group_id(1); + const int stream = get_group_id(2); + + q += offset_q + token*nbq2 + stream*nbq3; + k += offset_k + (ulong) key*nbk2 + stream*nbk3; + w += offset_w + token*nbw1 + stream*nbw3; + mask += offset_m + token*nbm1 + (stream % n_masks)*nbm3; + dst = (global float *) ((global char *) dst + offset_d); + +#ifdef cl_khr_subgroup_shuffle + if (use_shuffle) { + // Each group of four lanes keeps one key for all query heads. + float4 cached[8]; + #pragma unroll + for (int j = 0; j < 8; ++j) { + const int i = 4*(lane + j*4); + cached[j] = key < n_keys ? (type_k == 0 ? vload4(0, (global const float *) k + i) : convert_float4(vload4(0, (global const half *) k + i))) : (float4) (0.0f); + } + + float score = 0.0f; + for (int h = 0; h < n_heads; ++h) { + global const float * q_head = (global const float *) (q + h*nbq1); + float4 sum = (float4) (0.0f); + #pragma unroll + for (int j = 0; j < 8; ++j) { + sum = mad(vload4(0, q_head + 4*(lane + j*4)), cached[j], sum); + } + float value = sum.s0 + sum.s1 + sum.s2 + sum.s3; + value += sub_group_shuffle_xor(value, 2); + value += sub_group_shuffle_xor(value, 1); + if (lane == 0) { + score = mad(fmax(value, 0.0f), ((global const float *) w)[h], score); + } + } + if (lane == 0 && key < n_keys) { + dst[((ulong) stream*n_tokens + token)*n_keys + key] = score + (float) ((global const half *) mask)[key]; + } + return; + } +#endif + + local float sums[64]; + float score = 0.0f; + for (int h = 0; h < n_heads; ++h) { + global const float * q_head = (global const float *) (q + h*nbq1); + float sum = 0.0f; + if (key < n_keys) { + for (int i = lane; i < head_size; i += 16) { + float kv; + if (type_k == 0) { + kv = ((global const float *) k)[i]; + } else { + kv = (float) ((global const half *) k)[i]; + } + sum = mad(q_head[i], kv, sum); + } + } + sums[tid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + for (int stride = 8; stride > 0; stride /= 2) { + if (lane < stride) { + sums[tid] += sums[tid + stride]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + if (lane == 0) { + score = mad(fmax(sums[tid], 0.0f), ((global const float *) w)[h], score); + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (lane == 0 && key < n_keys) { + dst[((ulong) stream*n_tokens + token)*n_keys + key] = score + (float) ((global const half *) mask)[key]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl b/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl index 6a8e4fb17..df9919ec1 100644 --- a/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +++ b/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl @@ -57,13 +57,17 @@ kernel void kernel_add_id_add_id_swiglu_oai( global const float * g_row = (global const float *)(src_g + i1*nb01_g + i2*nb02_g); global const float * u_row = (global const float *)(src_u + i1*nb01_u + i2*nb02_u); - global const float * gb_row = (global const float *)(src_gb + i11*nb11_g); - global const float * ub_row = (global const float *)(src_ub + i11*nb11_u); global float * d_row = (global float *)(dst + i1*nbd1 + i2*nbd2); for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { - float x0 = g_row[i0] + gb_row[i0]; - float x1 = u_row[i0] + ub_row[i0]; + float x0 = g_row[i0]; + float x1 = u_row[i0]; + if (i11 != -1) { + global const float * gb_row = (global const float *)(src_gb + i11*nb11_g); + global const float * ub_row = (global const float *)(src_ub + i11*nb11_u); + x0 += gb_row[i0]; + x1 += ub_row[i0]; + } x0 = min(x0, limit); x1 = max(min(x1, limit), -limit); diff --git a/ggml/src/ggml-opencl/kernels/moe_combine.cl b/ggml/src/ggml-opencl/kernels/moe_combine.cl index acd08dbe6..cae7a7f2a 100644 --- a/ggml/src/ggml-opencl/kernels/moe_combine.cl +++ b/ggml/src/ggml-opencl/kernels/moe_combine.cl @@ -42,8 +42,11 @@ __kernel void kernel_moe_combine_bias_f32( float4 acc = (float4)(0.0f); for (int e = 0; e < k; ++e) { const int i11 = *((__global const int *)(I + (ulong)e*sizeof(int))); - __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u; - const float4 v = vload4(0, E + (uint)e*e1) + vload4(0, Brow); + float4 v = vload4(0, E + (uint)e*e1); + if (i11 != -1) { + __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u; + v += vload4(0, Brow); + } acc = mad(v, (float4)(W[(uint)e*w1]), acc); } diff --git a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl index d52d11aa5..e8ac31af2 100644 --- a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +++ b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl @@ -5,8 +5,11 @@ __kernel void kernel_moe_histogram( __global int * hist, uint N, uint topK, - uint n_experts + uint n_experts, + ulong offset ) { + input = (__global const int *)((__global const char *)input + offset); + uint n = get_global_id(0); uint k = get_global_id(1); @@ -15,6 +18,9 @@ __kernel void kernel_moe_histogram( } int expert_id = input[n * n_experts + k]; + if (expert_id == -1) { // skipped slot, owned by no expert + return; + } atomic_inc(&hist[expert_id]); } @@ -47,8 +53,11 @@ __kernel void kernel_moe_scatter( __global int * slot_counter, int N, int topK, - uint n_experts + uint n_experts, + ulong offset ) { + input = (__global const int *)((__global const char *)input + offset); + uint n = get_global_id(0); uint k = get_global_id(1); @@ -57,6 +66,9 @@ __kernel void kernel_moe_scatter( } int val = input[n * n_experts + k]; + if (val == -1) { // skipped slot, owned by no expert + return; + } int local_slot = atomic_inc(&slot_counter[val]); @@ -89,8 +101,11 @@ __kernel void kernel_moe_scatter_stable( __global const int * tile_offset, int N, int topK, - uint n_experts + uint n_experts, + ulong offset ) { + input = (__global const int *)((__global const char *)input + offset); + const int e = get_group_id(1); const int lid = get_local_id(0); const int M = N * topK; diff --git a/ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl b/ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl new file mode 100644 index 000000000..79ec21163 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl @@ -0,0 +1,22 @@ +kernel void kernel_moe_zero_dst( + global const char * ids, + ulong offset_ids, + ulong nb_ids1, + global char * dst, + ulong offset_dst, + ulong nb1, + ulong nb2, + int ne0 +) { + const size_t i1 = get_group_id(0); + const size_t i2 = get_group_id(1); + const int expert = *((global const int *)(ids + offset_ids + i1*sizeof(int) + i2*nb_ids1)); + if (expert != -1) { + return; + } + + global float * row = (global float *)(dst + offset_dst + i1*nb1 + i2*nb2); + for (size_t i0 = get_local_id(2); i0 < (size_t)ne0; i0 += get_local_size(2)) { + row[i0] = 0.0f; + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl new file mode 100644 index 000000000..2c0a56c85 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl @@ -0,0 +1,250 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#define LM_FIRST_256B 0 +#define LM_SECOND_256B 64 +#define LM_THIRD_256B 128 +#define LM_FOURTH_256B 192 + + +inline float16 mm_load_a( + image1d_buffer_t matrix_A, + uint subMatrixAStartInElements, + int line_stride_matrix_A_in_bytes +) { + __private float8 regA; + size_t sub_block_id_m = get_local_id(0); + + uint a_texCoord = subMatrixAStartInElements/2 + (sub_block_id_m * line_stride_matrix_A_in_bytes/4); + + regA.s0123 = read_imagef(matrix_A, a_texCoord/4); + regA.s4567 = read_imagef(matrix_A, (a_texCoord+4)/4); + + return convert_float16(as_half16(regA)); +} + +inline float4 alu_32( + float16 regA, + __local float4 * matrix_B_vec +) { + + __private float4 rC = 0; + int i = get_sub_group_id() * 64; + + rC += regA.s0 * matrix_B_vec[i]; + rC += regA.s1 * matrix_B_vec[i + 16]; + rC += regA.s4 * matrix_B_vec[i + 1]; + rC += regA.s5 * matrix_B_vec[i + 17]; + rC += regA.s8 * matrix_B_vec[i + 2]; + rC += regA.s9 * matrix_B_vec[i + 18]; + rC += regA.sc * matrix_B_vec[i + 3]; + rC += regA.sd * matrix_B_vec[i + 19]; + + i += 32; + + rC += regA.s2 * matrix_B_vec[i]; + rC += regA.s3 * matrix_B_vec[i + 16]; + rC += regA.s6 * matrix_B_vec[i + 1]; + rC += regA.s7 * matrix_B_vec[i + 17]; + rC += regA.sa * matrix_B_vec[i + 2]; + rC += regA.sb * matrix_B_vec[i + 18]; + rC += regA.se * matrix_B_vec[i + 3]; + rC += regA.sf * matrix_B_vec[i + 19]; + + return rC; +} + +inline float16 alu_16( + float16 regA, + __local float * matrix_B_local +) { + float16 out; + __local float4 * matrix_B_vec = (__local float4*)matrix_B_local; + + out.s0123 = alu_32(regA, matrix_B_vec); + out.s4567 = alu_32(regA, matrix_B_vec + 4); + out.s89ab = alu_32(regA, matrix_B_vec + 8); + out.scdef = alu_32(regA, matrix_B_vec + 12); + + return out; +} + +inline void mm_mad( + __local float * matrix_B_local, + float16 regA, + float8 regB, + uint b_localOffsetInWords, + float16 * regC0_ptr, + float16 * regC1_ptr +) { + int offset = b_localOffsetInWords + get_sub_group_id() * 256; + + matrix_B_local[offset + LM_FIRST_256B] = regB.s0; + matrix_B_local[offset + LM_SECOND_256B] = regB.s1; + matrix_B_local[offset + LM_THIRD_256B] = regB.s2; + matrix_B_local[offset + LM_FOURTH_256B] = regB.s3; + + float16 add0 = alu_16(regA, matrix_B_local); + *regC0_ptr += add0; + + matrix_B_local[offset + LM_FIRST_256B] = regB.s4; + matrix_B_local[offset + LM_SECOND_256B] = regB.s5; + matrix_B_local[offset + LM_THIRD_256B] = regB.s6; + matrix_B_local[offset + LM_FOURTH_256B] = regB.s7; + + float16 add1 = alu_16(regA, matrix_B_local); + *regC1_ptr += add1; +} + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 +// Fuse KQ and block softmax into U8 P, block maxima and sums. No F32 score matrix is stored. +__kernel void mul_mm_f16_f32_kq_p8( + __read_only image1d_buffer_t matrix_A, + __global float * matrix_B, + __global uchar * qp, // u8 P, [D_B][N][p_pitch] + __global float * bmax, // per 32-row block max, [D_B][N][p_pitch/32] + __global float * bsum, // per block sum of exp(s - bmax), [D_B][N][p_pitch/32] + __global char * mask, // f16 [>=M][N] view (chunk rows), row stride mask_nb1 bytes + ulong offset_mask, + ulong mask_nb1, + float scale, + float max_bias, + float m0, + float m1, + int n_head_log2, + int M, + int K, + int N, + int D_A, + int D_B, + int p_pitch // bytes per P row, >= M, multiple of 32 +) { + + // Match the n-tile-first grid in ggml_cl_fa_kq_p8_dispatch. + const uint m_blocks = (M+TILESIZE_M-1)/TILESIZE_M; + uint block_id_n = get_global_id(1); + uint block_id_m = get_global_id(2) % m_blocks; + uint block_id_d = get_global_id(2) / m_blocks; + + __private float16 regA; + __private float8 regB; + __private float16 regC0 = (float16)(0.0f); + __private float16 regC1 = (float16)(0.0f); + + const uint col = block_id_m * TILESIZE_M; + const uint row = block_id_n * TILESIZE_N; + const uint depth_A = block_id_d / (D_B/D_A); + const uint depth_B = block_id_d; + + int line_stride_matrix_A_in_bytes = K * D_A * 2; + int line_stride_matrix_B_in_bytes = K * D_B * 4; + + const uint strideAinElements = line_stride_matrix_A_in_bytes / 2; + const uint strideBinElements = line_stride_matrix_B_in_bytes / 4; + + size_t sub_block_id_m = get_local_id(0); + + uint b_localOffsetInWords = (sub_block_id_m/16)*16 + + ((((sub_block_id_m)>>0)&1)<<2) + + ((((sub_block_id_m)>>1)&1)<<3) + + ((((sub_block_id_m)>>2)&1)<<0) + + ((((sub_block_id_m)>>3)&1)<<1); + + uint2 b_globalOffsetInWords_xy = {((sub_block_id_m%4)*4), (sub_block_id_m>>2)}; + uint subMatrixAStartInElements = col * strideAinElements + depth_A * K; + uint subMatrixBStartInElements = 0; + + __local float matrix_B_local[1024]; + + // Clamp Q loads in the last tile; columns past N are not stored. + const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N - 1); + const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N - 1); + const uint b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x; + const uint b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x; + + for (uint step=0; step < K; step+=TILESIZE_K) { + regA = mm_load_a(matrix_A, subMatrixAStartInElements, line_stride_matrix_A_in_bytes); + + uint b_coordInWords00 = subMatrixBStartInElements + b_globalOffsetInWords00; + uint b_coordInWords16 = subMatrixBStartInElements + b_globalOffsetInWords16; + + regB.s0123 = vload4(b_coordInWords00/4, matrix_B); + regB.s4567 = vload4(b_coordInWords16/4, matrix_B); + + mm_mad(matrix_B_local, regA, regB, b_localOffsetInWords, ®C0, ®C1); + + subMatrixAStartInElements += TILESIZE_K; + subMatrixBStartInElements += TILESIZE_K; + } + + // Transpose each score tile in local memory so one lane owns a 32-row softmax block. + __local float p8_lds[2048]; + __local float * lds = p8_lds + get_sub_group_id() * 1024; + const uint lane = get_local_id(0); + __global const half * mrow = (__global const half *)(mask + offset_mask); + const uint mstride = (uint)(mask_nb1 / 2); + const int nblk = p_pitch / 32; + + #pragma unroll + for (int half_ = 0; half_ < 2; ++half_) { + barrier(CLK_LOCAL_MEM_FENCE); + const float16 cc = half_ == 0 ? regC0 : regC1; + lds[ 0*64 + lane] = cc.s0; lds[ 1*64 + lane] = cc.s1; lds[ 2*64 + lane] = cc.s2; lds[ 3*64 + lane] = cc.s3; + lds[ 4*64 + lane] = cc.s4; lds[ 5*64 + lane] = cc.s5; lds[ 6*64 + lane] = cc.s6; lds[ 7*64 + lane] = cc.s7; + lds[ 8*64 + lane] = cc.s8; lds[ 9*64 + lane] = cc.s9; lds[10*64 + lane] = cc.sa; lds[11*64 + lane] = cc.sb; + lds[12*64 + lane] = cc.sc; lds[13*64 + lane] = cc.sd; lds[14*64 + lane] = cc.se; lds[15*64 + lane] = cc.sf; + barrier(CLK_LOCAL_MEM_FENCE); + if (lane < 32) { + const int j = lane >> 1; + const int b = lane & 1; + const uint n = row + half_*16 + j; + const uint h = depth_B; + if (n < (uint)N) { + float slope = 1.0f; + if (max_bias > 0.0f) { + const float base = (int)h < n_head_log2 ? m0 : m1; + const int ex = (int)h < n_head_log2 ? (int)h + 1 : 2*((int)h - n_head_log2) + 1; + slope = pow(base, ex); + } + const uint kvb = col + b*32; // first kv row of this block + const __local float * src = lds + j*64 + b*32; + float16 s0 = vload16(0, src); + float16 s1 = vload16(1, src); + const __global half * mp = mrow + (size_t)n*mstride + kvb; + const float16 m0v = convert_float16(vload16(0, mp)); + const float16 m1v = convert_float16(vload16(1, mp)); + s0 = s0*scale + slope*m0v; + s1 = s1*scale + slope*m1v; + const float16 mx16 = fmax(s0, s1); + const float8 mx8 = fmax(mx16.lo, mx16.hi); + const float4 mx4 = fmax(mx8.lo, mx8.hi); + const float2 mx2 = fmax(mx4.lo, mx4.hi); + const float amax = fmax(mx2.x, mx2.y); + const uint prow = h*(uint)N + n; + const uint blk = kvb >> 5; + float sum = 0.0f; + uchar16 q0 = (uchar16)(0); + uchar16 q1 = (uchar16)(0); + // Use a finite sentinel: -cl-finite-math-only can remove infinity checks. + if (amax > -1.0e30f) { + const float16 e0 = exp(s0 - amax); + const float16 e1 = exp(s1 - amax); + const float16 a16 = e0 + e1; + const float8 a8 = a16.lo + a16.hi; + const float4 a4 = a8.lo + a8.hi; + const float2 a2 = a4.lo + a4.hi; + sum = a2.x + a2.y; + q0 = convert_uchar16_sat_rte(e0*255.0f); + q1 = convert_uchar16_sat_rte(e1*255.0f); + } + __global uchar * qdst = qp + (size_t)prow*(size_t)p_pitch + (size_t)kvb; + vstore16(q0, 0, qdst); + vstore16(q1, 1, qdst); + bmax[(size_t)prow*nblk + blk] = amax; + bsum[(size_t)prow*nblk + blk] = sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl index 6982f8f51..eba1778af 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl @@ -4,12 +4,20 @@ #define LOAD_VEC_B 4 #define BM 64 +#ifndef BN #define BN 64 +#endif #define BK 16 #define TM 4 +#ifndef TN #define TN 8 +#endif -kernel void kernel_mul_mm_f16_f32_l4_lm( +#ifndef KERNEL_NAME_LM +#define KERNEL_NAME_LM kernel_mul_mm_f16_f32_l4_lm +#endif + +kernel void KERNEL_NAME_LM( global half4 * src0, ulong offset0, global float4 * src1, @@ -93,7 +101,8 @@ kernel void kernel_mul_mm_f16_f32_l4_lm( } } - for (int l = 0; l < BN; l += loadstride_b) { + // BN=8 uses only the threads that load a column inside the tile. + for (int l = 0; loadc_b + l < BN; l += loadstride_b) { if (ic*BN + loadc_b + l < ne11) { const int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl new file mode 100644 index 000000000..fdcbd6113 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl @@ -0,0 +1,98 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define BM 64 +#define BN 32 +#define BK 16 +#define TM 4 +#define TN 4 + +kernel void kernel_mul_mm_id_f32_f32( + global const char * src0, ulong offset0, + global const char * src1, ulong offset1, + global const int * routes, + global const ushort * experts, + global const int * total_tiles, + global char * dst, ulong offsetd, + int ncols, int nrows, int ne11, int nused, int src0_f16, + ulong nb01, ulong nb02, ulong nb11, ulong nb12, ulong nb1, ulong nb2) { + const int tile = get_group_id(1); + if (tile >= total_tiles[0]) { + return; + } + const int tid = get_local_id(0); + const int row = get_group_id(0)*BM; + const int mr = tid % (BM/TM)*TM; + const int nr = tid / (BM/TM)*TN; + src0 += offset0 + experts[tile]*nb02; + src1 += offset1; + dst += offsetd; + local float a[BM*BK]; + local float b[BN*BK]; + float sums[TM*TN] = {0.0f}; + + for (int base = 0; base < ncols; base += BK) { + for (int i = tid; i < BM*BK/4; i += 128) { + const int r = i/(BK/4); + const int k = base + i%(BK/4)*4; + float4 value = 0.0f; + if (row + r < nrows && k + 3 < ncols) { + global const char * arow = src0 + (row + r)*nb01; + value = src0_f16 ? convert_float4(vload4(0, (global const half *) arow + k)) : vload4(0, (global const float *) arow + k); + } else if (row + r < nrows) { + float tail[4] = {0.0f}; + for (int j = 0; j < 4 && k + j < ncols; ++j) { + global const char * arow = src0 + (row + r)*nb01; + tail[j] = src0_f16 ? (float) ((global const half *) arow)[k + j] : ((global const float *) arow)[k + j]; + } + value = (float4) (tail[0], tail[1], tail[2], tail[3]); + } + a[(i%(BK/4)*4 + 0)*BM + r] = value.s0; + a[(i%(BK/4)*4 + 1)*BM + r] = value.s1; + a[(i%(BK/4)*4 + 2)*BM + r] = value.s2; + a[(i%(BK/4)*4 + 3)*BM + r] = value.s3; + } + for (int i = tid; i < BN*BK/4; i += 128) { + const int route = routes[tile*BN + i/(BK/4)]; + const int k = base + i%(BK/4)*4; + float4 value = 0.0f; + if (route >= 0) { + global const float * brow = (global const float *) (src1 + (route%nused%ne11)*nb11 + (route/nused)*nb12); + if (k + 3 < ncols) { + value = vload4(0, brow + k); + } else { + float tail[4] = {0.0f}; + for (int j = 0; j < 4 && k + j < ncols; ++j) { + tail[j] = brow[k + j]; + } + value = (float4) (tail[0], tail[1], tail[2], tail[3]); + } + } + b[(i%(BK/4)*4 + 0)*BN + i/(BK/4)] = value.s0; + b[(i%(BK/4)*4 + 1)*BN + i/(BK/4)] = value.s1; + b[(i%(BK/4)*4 + 2)*BN + i/(BK/4)] = value.s2; + b[(i%(BK/4)*4 + 3)*BN + i/(BK/4)] = value.s3; + } + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int k = 0; k < BK; ++k) { + #pragma unroll + for (int n = 0; n < TN; ++n) { + const float bv = b[k*BN + nr + n]; + #pragma unroll + for (int m = 0; m < TM; ++m) { + sums[n*TM + m] = mad(a[k*BM + mr + m], bv, sums[n*TM + m]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + for (int n = 0; n < TN; ++n) { + const int route = routes[tile*BN + nr + n]; + if (route >= 0) { + global float * out = (global float *) (dst + (route%nused)*nb1 + (route/nused)*nb2); + for (int m = 0; m < TM && row + mr + m < nrows; ++m) { + out[row + mr + m] = sums[n*TM + m]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq1_m_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq1_m_f32_l4_lm.cl new file mode 100644 index 000000000..192a6c69e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq1_m_f32_l4_lm.cl @@ -0,0 +1,191 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq1_m { + uchar qs[QK_K/8]; + uchar qh[QK_K/16]; + uchar scales[QK_K/32]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq1_m_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_qh, + global ushort * src0_sc, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq1_m super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq1_m); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + global ushort * sc = src0_sc + 4*ib; + ushort u16 = (ushort)((sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) + | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000)); + float dl = (float)as_half(u16) * (float)(2*((sc[ib32/2] >> (6*(ib32%2) + 3*(lg/2))) & 7) + 1); + uchar qhb = src0_qh[16*ib + 2*ib32 + lg/2]; + float dlt = (qhb & (0x08 << (4*(lg%2)))) ? -1.125f : -0.875f; + uint gi = (uint)src0_qs[32*ib + 4*ib32 + lg] | ((((uint)qhb >> (4*(lg%2))) & 7) << 8); + uint g = iq1_s_grid[gi]; + int sh = (j0 >> 2) * 4; + + float4 v1; + v1.s0 = dl * ((float)((g >> ( 0 + sh)) & 0xF) + dlt); + v1.s1 = dl * ((float)((g >> ( 8 + sh)) & 0xF) + dlt); + v1.s2 = dl * ((float)((g >> (16 + sh)) & 0xF) + dlt); + v1.s3 = dl * ((float)((g >> (24 + sh)) & 0xF) + dlt); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq1_s_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq1_s_f32_l4_lm.cl new file mode 100644 index 000000000..fca4eeace --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq1_s_f32_l4_lm.cl @@ -0,0 +1,188 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq1_s { + half d; + uchar qs[QK_K/8]; + ushort qh[QK_K/32]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq1_s_f32_l4_lm( + global uchar * src0_qs, + global ushort * src0_qh, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq1_s super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq1_s); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + ushort qhb = src0_qh[8*ib + ib32]; + float dl = (float)src0_d[ib] * (float)(2*((qhb >> 12) & 7) + 1); + float dlt = (qhb & 0x8000) ? -1.125f : -0.875f; + uint gi = (uint)src0_qs[32*ib + 4*ib32 + lg] | ((((uint)qhb >> (3*lg)) & 7) << 8); + uint g = iq1_s_grid[gi]; + int sh = (j0 >> 2) * 4; + + float4 v1; + v1.s0 = dl * ((float)((g >> ( 0 + sh)) & 0xF) + dlt); + v1.s1 = dl * ((float)((g >> ( 8 + sh)) & 0xF) + dlt); + v1.s2 = dl * ((float)((g >> (16 + sh)) & 0xF) + dlt); + v1.s3 = dl * ((float)((g >> (24 + sh)) & 0xF) + dlt); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq2_s_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_s_f32_l4_lm.cl new file mode 100644 index 000000000..f33f500a4 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_s_f32_l4_lm.cl @@ -0,0 +1,191 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq2_s { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar scales[QK_K/32]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq2_s_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_sg, + global uchar * src0_qh, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq2_s super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq2_s); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + uchar scb = src0_sc[8*ib + ib32]; + float db = (float)src0_d[ib] * (0.5f + (float)((lg < 2) ? (scb & 0xf) : (scb >> 4))) * 0.25f; + + uint gi = (uint)src0_qs[32*ib + 4*ib32 + lg] | (((uint)src0_qh[8*ib + ib32] << (8-2*lg)) & 0x300); + uchar sg = src0_sg[32*ib + 4*ib32 + lg]; + uint g = iq2_s_grid[2*gi + (j0 >> 2)]; + + float4 v1; + v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f); + v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f); + v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f); + v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xs_f32_l4_lm.cl new file mode 100644 index 000000000..cdff38bc4 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xs_f32_l4_lm.cl @@ -0,0 +1,189 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq2_xs { + half d; + ushort qs[QK_K/8]; + uchar scales[QK_K/32]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq2_xs_f32_l4_lm( + global ushort * src0_qs, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq2_xs super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq2_xs); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + uchar scb = src0_sc[8*ib + ib32]; + float db = (float)src0_d[ib] * (0.5f + (float)((lg < 2) ? (scb & 0xf) : (scb >> 4))) * 0.25f; + + uint q = src0_qs[32*ib + 4*ib32 + lg]; + uint gi = q & 511u; + uint sg = iq2_xs_signs(q >> 9); + uint g = iq2_xs_grid[2*gi + (j0 >> 2)]; + + float4 v1; + v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f); + v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f); + v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f); + v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xxs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xxs_f32_l4_lm.cl new file mode 100644 index 000000000..7a44038b8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xxs_f32_l4_lm.cl @@ -0,0 +1,187 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq2_xxs { + half d; + ushort qs[QK_K/8]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq2_xxs_f32_l4_lm( + global uchar * src0_qs, + global uint * src0_sas, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq2_xxs super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq2_xxs); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + uint aux = src0_sas[8*ib + ib32]; + float db = (float)src0_d[ib] * (0.5f + (float)(aux >> 28)) * 0.25f; + + uint gi = src0_qs[32*ib + 4*ib32 + lg]; + uint sg = iq2_xxs_signs((aux >> (7*lg)) & 127u); + uint g = iq2_xxs_grid[2*gi + (j0 >> 2)]; + + float4 v1; + v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f); + v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f); + v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f); + v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl new file mode 100644 index 000000000..e75c296ac --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl @@ -0,0 +1,193 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq3_s { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar signs[QK_K/8]; + uchar scales[QK_K/64]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq3_s_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_sg, + global uchar * src0_qh, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq3_s super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq3_s); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + uchar scb = src0_sc[4*ib + ib32/2]; + float db = (float)src0_d[ib] * (float)(1 + 2*((scb >> (4*(ib32 & 1))) & 15)); + int which = j0 >> 2; + uint gi = (uint)src0_qs[64*ib + 8*ib32 + 2*lg + which] + | (((uint)src0_qh[8*ib + ib32] << (8 - 2*lg - which)) & 256); + uchar sg = src0_sg[32*ib + 4*ib32 + lg]; + uint g = iq3_s_grid[gi]; + + float4 v1; + v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f); + v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f); + v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f); + v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl new file mode 100644 index 000000000..3c50d6230 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl @@ -0,0 +1,187 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq3_xxs { + half d; + uchar qs[3*QK_K/8]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq3_xxs_f32_l4_lm( + global uchar * src0_qs, + global uint * src0_sas, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq3_xxs super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq3_xxs); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + uint aux = src0_sas[8*ib + ib32]; + float db = (float)src0_d[ib] * (0.5f + (float)(aux >> 28)) * 0.5f; + + uint gi = src0_qs[64*ib + 8*ib32 + 2*lg + (j0 >> 2)]; + uint sg = iq3_xxs_signs((aux >> (7*lg)) & 127u); + uint g = iq3_xxs_grid[gi]; + + float4 v1; + v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f); + v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f); + v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f); + v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl new file mode 100644 index 000000000..f127bd9f8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl @@ -0,0 +1,186 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq4_xs { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq4_xs_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_sl, + global ushort * src0_sh, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq4_xs super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq4_xs); + int e = idx % QK_K; + + int ib32 = e >> 5; + int j = e & 15; + int shift = (e & 16) >> 2; + int ls = ((src0_sl[4*ib + ib32/2] >> (4*(ib32 & 1))) & 15) | (((src0_sh[ib] >> (2*ib32)) & 3) << 4); + float db = (float)src0_d[ib] * (float)(ls - 32); + uchar4 q = vload4(0, src0_qs + 128*ib + 16*ib32 + j); + q = (q >> (uchar4)(shift)) & (uchar4)(15); + + float4 v1; + v1.s0 = db * (float)iq4_xs_values[q.s0]; + v1.s1 = db * (float)iq4_xs_values[q.s1]; + v1.s2 = db * (float)iq4_xs_values[q.s2]; + v1.s3 = db * (float)iq4_xs_values[q.s3]; + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q2_0_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q2_0_f32_l4_lm.cl new file mode 100644 index 000000000..673b70655 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q2_0_f32_l4_lm.cl @@ -0,0 +1,171 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK2_0 64 + +struct block_q2_0 { + half d; + uchar qs[16]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_q2_0_f32_l4_lm( + global uchar * src0_qs, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK2_0; + ib += offset0/sizeof(struct block_q2_0); + int e = idx % QK2_0; + + const uint q = src0_qs[16*ib + e/4]; + const int4 v = convert_int4(((uint4)(q) >> (uint4)(0, 2, 4, 6)) & (uint4)(3)) - (int4)(1); + const float4 v1 = (float)src0_d[ib] * convert_float4(v); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl new file mode 100644 index 000000000..e192bdefb --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl @@ -0,0 +1,203 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_q2_K { + uchar scales[QK_K/16]; + uchar qs[QK_K/4]; + half d; + half dmin; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#ifndef BK +#define BK 32 +#endif +#ifdef INTEL_GPU +#define TM 8 +#else +#define TM 4 +#endif +#define TN 8 + +#ifndef LM_HALF +#define LM_HALF 0 +#endif +#if LM_HALF +typedef half lm_st; +#define LM_LD4(p, i) convert_float4(vload4((i), (p))) +#else +typedef float lm_st; +#define LM_LD4(p, i) vload4((i), (p)) +#endif + +kernel void kernel_mul_mm_q2_k_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_sc, + global half * src0_dm, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local lm_st buf_a[BM * BK]; + local lm_st buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one q2_K super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float4 sums4[(TM/4) * TN]; + float4 cache_a4[TM/4]; + + for (int i = 0; i < (TM/4) * TN; i++) { + sums4[i] = (float4)(0.0f); + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_q2_K); + int e = idx % QK_K; + + // dequantize_row_q2_K walks 2 halves of 128, each as 4 shifts of + // 2 groups of 16. Recover that position from the element index. + int n = e >> 7; + int rem = e & 127; + int j = rem >> 5; + int s = (rem >> 4) & 1; + int l16 = rem & 15; + int isc = 8*n + 2*j + s; + int shift = 2*j; + + uchar sc = src0_sc[16*ib + isc]; + float d = (float)src0_dm[2*ib] * (float)(sc & 0xF); + float m = -(float)src0_dm[2*ib + 1] * (float)(sc >> 4); + + uchar4 q = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16); + float4 v1 = convert_float4((uchar4)((q.s0 >> shift) & 3, + (q.s1 >> shift) & 3, + (q.s2 >> shift) & 3, + (q.s3 >> shift) & 3)) * d + m; + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int a = 0; a < TM/4; a++) { + cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a); + } + + for (int cc = 0; cc < TN; cc++) { + const float cache_b = buf_b[(i) * BN + th_c * TN + cc]; + for (int a = 0; a < TM/4; a++) { + const int sums_idx = cc*(TM/4) + a; + sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int a = 0; a < TM/4; a++) { + const float4 v = sums4[cc * (TM/4) + a]; + const float vs[4] = { v.s0, v.s1, v.s2, v.s3 }; + for (int k = 0; k < 4; k++) { + if (dr + 4*a + k < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k]; + } + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl new file mode 100644 index 000000000..da3997417 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl @@ -0,0 +1,219 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_q3_K { + uchar hmask[QK_K/8]; + uchar qs[QK_K/4]; + uchar scales[12]; + half d; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#ifndef BK +#define BK 32 +#endif +#ifdef INTEL_GPU +#define TM 8 +#else +#define TM 4 +#endif +#define TN 8 + +#ifndef LM_HALF +#define LM_HALF 0 +#endif +#if LM_HALF +typedef half lm_st; +#define LM_LD4(p, i) convert_float4(vload4((i), (p))) +#else +typedef float lm_st; +#define LM_LD4(p, i) vload4((i), (p)) +#endif + +kernel void kernel_mul_mm_q3_k_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_hm, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local lm_st buf_a[BM * BK]; + local lm_st buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one q3_K super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float4 sums4[(TM/4) * TN]; + float4 cache_a4[TM/4]; + + for (int i = 0; i < (TM/4) * TN; i++) { + sums4[i] = (float4)(0.0f); + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_q3_K); + int e = idx % QK_K; + + // Only qs advances in the second half of a super-block. + int n = e >> 7; + int rem = e & 127; + int j = rem >> 5; + int s = (rem >> 4) & 1; + int l16 = rem & 15; + int is = 8*n + 2*j + s; + int shift = 2*j; + uchar m = (uchar)(1u << (4*n + j)); + + global ushort * a = (global ushort *)(src0_sc + 12*ib); + uint a0 = (uint)a[0] | ((uint)a[1] << 16); + uint a1 = (uint)a[2] | ((uint)a[3] << 16); + uint tmp = (uint)a[4] | ((uint)a[5] << 16); + + uint g = is >> 2; + uint w = (g & 1u) ? a1 : a0; + w = ((w >> ((g >> 1)*4)) & 0x0f0f0f0fu) | (((tmp >> (2*g)) & 0x03030303u) << 4); + + uint sc6 = (w >> (8*(is & 3))) & 0xFFu; + float dl = (float)src0_d[ib] * ((float)sc6 - 32.0f); + + uchar4 q = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16); + uchar4 hb = vload4(0, src0_hm + 32*ib + 16*s + l16); + + float4 qv = convert_float4((uchar4)((q.s0 >> shift) & 3, + (q.s1 >> shift) & 3, + (q.s2 >> shift) & 3, + (q.s3 >> shift) & 3)); + float4 hv = (float4)((hb.s0 & m) ? 0.f : 4.f, + (hb.s1 & m) ? 0.f : 4.f, + (hb.s2 & m) ? 0.f : 4.f, + (hb.s3 & m) ? 0.f : 4.f); + float4 v1 = (qv - hv) * dl; + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int a = 0; a < TM/4; a++) { + cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a); + } + + for (int cc = 0; cc < TN; cc++) { + const float cache_b = buf_b[(i) * BN + th_c * TN + cc]; + for (int a = 0; a < TM/4; a++) { + const int sums_idx = cc*(TM/4) + a; + sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int a = 0; a < TM/4; a++) { + const float4 v = sums4[cc * (TM/4) + a]; + const float vs[4] = { v.s0, v.s1, v.s2, v.s3 }; + for (int k = 0; k < 4; k++) { + if (dr + 4*a + k < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k]; + } + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q8_f32_kqv.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q8_f32_kqv.cl new file mode 100644 index 000000000..08d80961c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q8_f32_kqv.cl @@ -0,0 +1,131 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// Multiply U8 P by S8 V^T, with one scale per 32 KV entries. +// Group query heads by KV head to share V^T in L2. +// The packed dot takes the V word (registers) first: swapped operands miscompile on Adreno. + +#define KQV_TM 64 // d rows per workgroup, one per lane +#ifndef KQV_TN +#define KQV_TN 32 // queries per workgroup, all owned by every lane +#endif +#ifndef KQV_NB +#define KQV_NB 8 // 32-blocks of P staged per barrier +#endif +#define KQV_WG 64 + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_mul_mm_q8_f32_kqv( + global const uint * vq, // V^T int8, [head_kv][dv][kv_pitch] as packed uints + ulong off_vq, + global const half * vd, // V^T scales, [head_kv][dv][kv_pitch/32] + ulong off_vd, + global const uint * pq, // P u8, [head][n_q][kv_pitch] as packed uints + ulong off_pq, + global const float * pd, // P scales f32, [head][n_q][kv_pitch/32] + ulong off_pd, + global float * dst, // [head][n_q][dv] + ulong off_dst, + int dv, + int n_kv, + int n_q, + int n_head, + int n_head_kv, + int kv_pitch // bytes per V^T row and per P row, >= n_kv, multiple of 32 +) { + vq = (global const uint *)((global const char *)vq + off_vq); + vd = (global const half *)((global const char *)vd + off_vd); + pq = (global const uint *)((global const char *)pq + off_pq); + pd = (global const float *)((global const char *)pd + off_pd); + dst = (global float *)((global const char *)dst + off_dst); + + const int lid = get_local_id(0); + const int gsz = n_head / n_head_kv; // query heads per KV head + const int g1 = get_group_id(1); + const int head_kv = get_group_id(2); + const int head = head_kv*gsz + (g1 % gsz); + const int qn0 = (g1 / gsz) * KQV_TN; // first query of this tile + const int d = get_group_id(0)*KQV_TM + lid; // this lane's d row + + const int nblk = n_kv / 32; // dispatch guarantees n_kv % 32 == 0 + const int nu = kv_pitch / 4; // uints per row + const int nbp = kv_pitch / 32; // scales per row + + __local uint sh_pq[KQV_TN][KQV_NB][8]; + __local float sh_pd[KQV_TN][KQV_NB]; + + float acc[KQV_TN]; + #pragma unroll + for (int t = 0; t < KQV_TN; ++t) { + acc[t] = 0.0f; + } + + const int dl = min(d, dv - 1); // clamp so a tail lane loads in bounds + const size_t vbase = ((size_t)head_kv*dv + dl)*nu; + const size_t vdbas = ((size_t)head_kv*dv + dl)*nbp; + const size_t pbase = (size_t)head*n_q; + + for (int bg = 0; bg < nblk; bg += KQV_NB) { + const int nb_here = min(KQV_NB, nblk - bg); + + // stage P: each (query, block) is 8 uints = two 16-byte loads + for (int i = lid; i < KQV_TN*KQV_NB*2; i += KQV_WG) { + const int t = i / (KQV_NB*2); + const int bb = (i / 2) % KQV_NB; + const int h = (i & 1) * 4; + const int qi = qn0 + t; + uint4 v = (uint4)(0u); + if (qi < n_q && bb < nb_here) { + v = vload4(0, &pq[(pbase + qi)*nu + (size_t)(bg + bb)*8 + h]); + } + vstore4(v, 0, &sh_pq[t][bb][h]); + } + for (int i = lid; i < KQV_TN*KQV_NB; i += KQV_WG) { + const int t = i / KQV_NB; + const int bb = i % KQV_NB; + const int qi = qn0 + t; + sh_pd[t][bb] = (qi < n_q && bb < nb_here) + ? pd[(pbase + qi)*nbp + bg + bb] : 0.0f; + } + barrier(CLK_LOCAL_MEM_FENCE); + + for (int bb = 0; bb < nb_here; ++bb) { + const uint4 w0 = vload4(0, &vq[vbase + (size_t)(bg + bb)*8]); + const uint4 w1 = vload4(0, &vq[vbase + (size_t)(bg + bb)*8 + 4]); + const float dvs = (float)vd[vdbas + bg + bb]; + + #pragma unroll + for (int t = 0; t < KQV_TN; ++t) { + const uint4 a0 = vload4(0, &sh_pq[t][bb][0]); + const uint4 a1 = vload4(0, &sh_pq[t][bb][4]); + int raw = 0; + raw = dot_acc_sat_4x8packed_su_int(w0.s0, a0.s0, raw); + raw = dot_acc_sat_4x8packed_su_int(w0.s1, a0.s1, raw); + raw = dot_acc_sat_4x8packed_su_int(w0.s2, a0.s2, raw); + raw = dot_acc_sat_4x8packed_su_int(w0.s3, a0.s3, raw); + raw = dot_acc_sat_4x8packed_su_int(w1.s0, a1.s0, raw); + raw = dot_acc_sat_4x8packed_su_int(w1.s1, a1.s1, raw); + raw = dot_acc_sat_4x8packed_su_int(w1.s2, a1.s2, raw); + raw = dot_acc_sat_4x8packed_su_int(w1.s3, a1.s3, raw); + acc[t] += dvs * sh_pd[t][bb] * (float)raw; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (d >= dv) { + return; + } + + // dst is [head][n_q][dv] with dv contiguous, so the 64 lanes of a d-block write contiguously + #pragma unroll + for (int t = 0; t < KQV_TN; ++t) { + const int qi = qn0 + t; + if (qi < n_q) { + dst[(pbase + qi)*dv + d] = acc[t]; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl index ec71b8756..1ac250a84 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl @@ -116,3 +116,70 @@ kernel void kernel_mul_mat_f32_f32( } } } + +// Each group computes two rows and four columns, with 32 lanes per row. +kernel void kernel_mul_mv_f32_f32_r2( + global const char * src0, + ulong offset0, + global const char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 += offset0; + src1 += offset1; + dst = (global float *)((global char *)dst + offsetd); + + const int lid = get_local_id(0); + const int lane = lid % 32; + const int row = get_group_id(0)*2 + lid/32; + const int col = get_group_id(1)*4; + const int im = get_group_id(2); + const int i12 = im % ne12; + const int i13 = im / ne12; + global const float4 * x = (global const float4 *)(src0 + min(row, ne01 - 1)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); + global const char * y = src1 + col*nb11 + i12*nb12 + i13*nb13; + + float4 sum = 0.0f; + for (int i = lane; i < ne00/4; i += 32) { + const float4 xv = x[i]; + if (col + 0 < ne11) sum.s0 += dot(xv, ((global const float4 *)(y + 0*nb11))[i]); + if (col + 1 < ne11) sum.s1 += dot(xv, ((global const float4 *)(y + 1*nb11))[i]); + if (col + 2 < ne11) sum.s2 += dot(xv, ((global const float4 *)(y + 2*nb11))[i]); + if (col + 3 < ne11) sum.s3 += dot(xv, ((global const float4 *)(y + 3*nb11))[i]); + } + + local float4 part[64]; + part[lid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + for (int s = 16; s > 0; s /= 2) { + if (lane < s) { + part[lid] += part[lid + s]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + if (lane == 0 && row < ne01) { + if (col + 0 < ne11) dst[im*ne1*ne0 + (col + 0)*ne0 + row] = part[lid].s0; + if (col + 1 < ne11) dst[im*ne1*ne0 + (col + 1)*ne0 + row] = part[lid].s1; + if (col + 2 < ne11) dst[im*ne1*ne0 + (col + 2)*ne0 + row] = part[lid].s2; + if (col + 3 < ne11) dst[im*ne1*ne0 + (col + 3)*ne0 + row] = part[lid].s3; + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl new file mode 100644 index 000000000..b82ca90c8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl @@ -0,0 +1,89 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("half"))) +#endif +kernel void kernel_mul_mv_id_f32_f32( + global const char * src0, + ulong offset0, + global const char * src1, + ulong offset1, + global const char * ids, + ulong offset2, + global char * dst, + ulong offsetd, + int ncols, + int ne11, + int src0_f16, + ulong nb01, + ulong nb02, + ulong nb11, + ulong nb12, + ulong nb20, + ulong nb21, + ulong nb1, + ulong nb2, + int nrows) { + const int row = get_group_id(0)*4; + const int slot = get_group_id(1); + const int token = get_group_id(2); + const int tid = get_local_id(0); + const int expert = *(global const int *) (ids + offset2 + slot*nb20 + token*nb21); + dst += offsetd + slot*nb1 + token*nb2; + if (expert == -1) { + if (tid == 0) { + for (int r = 0; r < 4 && row + r < nrows; ++r) { + ((global float *) dst)[row + r] = 0.0f; + } + } + return; + } + src0 += offset0 + row*nb01 + expert*nb02; + src1 += offset1 + (slot % ne11)*nb11 + token*nb12; + float sum[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + for (int i = tid*4; i + 3 < ncols; i += 64*4) { + const float4 b = vload4(0, (global const float *) src1 + i); + for (int r = 0; r < 4 && row + r < nrows; ++r) { + global const char * arow = src0 + r*nb01; + const float4 a = src0_f16 ? convert_float4(vload4(0, (global const half *) arow + i)) : vload4(0, (global const float *) arow + i); + sum[r] += dot(a, b); + } + } + for (int i = ncols/4*4 + tid; i < ncols; i += 64) { + const float b = ((global const float *) src1)[i]; + for (int r = 0; r < 4 && row + r < nrows; ++r) { + global const char * arow = src0 + r*nb01; + const float a = src0_f16 ? (float) ((global const half *) arow)[i] : ((global const float *) arow)[i]; + sum[r] = mad(a, b, sum[r]); + } + } +#ifdef cl_qcom_reqd_sub_group_size + for (int r = 0; r < 4; ++r) { + sum[r] = sub_group_reduce_add(sum[r]); + } +#else + local float sums[4][64]; + for (int r = 0; r < 4; ++r) { + sums[r][tid] = sum[r]; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int stride = 32; stride > 0; stride /= 2) { + for (int r = 0; r < 4; ++r) { + if (tid < stride) { + sums[r][tid] += sums[r][tid + stride]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + for (int r = 0; r < 4; ++r) { + sum[r] = sums[r][0]; + } +#endif + if (tid == 0) { + for (int r = 0; r < 4 && row + r < nrows; ++r) { + ((global float *) dst)[row + r] = sum[r]; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl index d50bd1fc4..4c0755e14 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl @@ -172,6 +172,9 @@ kernel void kernel_mul_mv_id_mxfp4_f32( const int idx = get_group_id(2)%ne20; int i02 = ((global int *) (src2 + iid1*nb21))[idx]; + if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int i11 = idx % ne11; int i12 = iid1; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl index f65e86ed6..962ddd76d 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl @@ -101,6 +101,9 @@ kernel void kernel_mul_mv_id_mxfp4_f32_flat( const int idx = get_group_id(2) % ne20; uint i02 = ((global uint *) (src2 + offset2 + iid1 * nb21))[idx]; + if ((int)i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int i11 = idx % ne11; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl index 7ccf41efb..13314689f 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl @@ -267,6 +267,9 @@ kernel void kernel_mul_mv_id_q4_0_f32_8x_flat( const int idx = get_group_id(2)%ne20; const int i02 = ((global int *)(src2 + iid1*nb21))[idx]; + if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } const int i11 = idx%ne11; const int i12 = iid1; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl index f37e83ee8..2c40a43f1 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl @@ -73,6 +73,9 @@ kernel void kernel_mul_mv_id_q8_0_f32( int idx = get_group_id(2)%ne20; int i02 = ((global int *) (src2 + iid1*nb21))[idx]; + if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int i11_ = idx % ne11; int i12_ = iid1; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl index fd3a0710f..3b1b39b62 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl @@ -72,6 +72,9 @@ kernel void kernel_mul_mv_id_q8_0_f32_flat( int idx = (int)get_group_id(2)%ne20; int i02 = ((global int *) (src2 + iid1*nb21))[idx]; + if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int i11_ = idx % ne11; int i12_ = iid1; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32.cl new file mode 100644 index 000000000..9e1554dc7 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32.cl @@ -0,0 +1,149 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 +#define IQ1_M_DELTA 0.125f + +// no d field: the super block scale is 4 nibbles spread over scales[] +typedef struct { + uchar qs[QK_K/8]; + uchar qh[QK_K/16]; + uchar scales[QK_K/32]; +} block_iq1_m; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq1_m_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq1_m * x = (global block_iq1_m *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq1_m * xb = (global block_iq1_m *)(xrow + row*nb01); + + global ushort * sc = (global ushort *)xb->scales; + ushort u16 = (ushort)(((uint)sc[0] >> 12) | (((uint)sc[1] >> 8) & 0x00f0) + | (((uint)sc[2] >> 4) & 0x0f00) | ((uint)sc[3] & 0xf000)); + float dsuper = (float)as_half(u16); + + float acc = 0.f; + for (int il = 0; il < 4; ++il) { + int ib16 = 2*it + il/2; + float dl = dsuper * (float)(2*(((uint)sc[ib16/4] >> (3*(ib16%4))) & 7) + 1); + + uchar qhb = xb->qh[2*it + il/2]; + float dlt = (qhb & (0x08 << (4*(il%2)))) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA); + + uint gi = (uint)xb->qs[4*it+il] | ((((uint)qhb >> (4*(il%2))) & 7) << 8); + uint g = iq1_s_grid[gi]; + + float a = 0.f; + for (int j = 0; j < 4; ++j) { + a += yl[8*il+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt); + a += yl[8*il+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt); + } + acc += dl * a; + } + + sumf[row] += acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32_flat.cl new file mode 100644 index 000000000..3b5e18934 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32_flat.cl @@ -0,0 +1,148 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 +#define IQ1_M_DELTA 0.125f + +// no d field: the super block scale is 4 nibbles spread over scales[] +typedef struct { + uchar qs[QK_K/8]; + uchar qh[QK_K/16]; + uchar scales[QK_K/32]; +} block_iq1_m; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq1_m_f32_flat( + global uchar * src0_qs, + global uchar * src0_qh, + global ushort * src0_sc, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq1_m); + ulong row_stride = nb01/sizeof(block_iq1_m); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + global ushort * sc = src0_sc + 4*b; + ushort u16 = (ushort)(((uint)sc[0] >> 12) | (((uint)sc[1] >> 8) & 0x00f0) + | (((uint)sc[2] >> 4) & 0x0f00) | ((uint)sc[3] & 0xf000)); + float dsuper = (float)as_half(u16); + + float acc = 0.f; + for (int il = 0; il < 4; ++il) { + int ib16 = 2*it + il/2; + float dl = dsuper * (float)(2*(((uint)sc[ib16/4] >> (3*(ib16%4))) & 7) + 1); + + uchar qhb = src0_qh[16*b + 2*it + il/2]; + float dlt = (qhb & (0x08 << (4*(il%2)))) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA); + + uint gi = (uint)src0_qs[32*b + 4*it+il] | ((((uint)qhb >> (4*(il%2))) & 7) << 8); + uint g = iq1_s_grid[gi]; + + float a = 0.f; + for (int j = 0; j < 4; ++j) { + a += yl[8*il+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt); + a += yl[8*il+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt); + } + acc += dl * a; + } + + sumf[row] += acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32.cl new file mode 100644 index 000000000..46a673072 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32.cl @@ -0,0 +1,140 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 +#define IQ1_S_DELTA 0.125f + +typedef struct { + half d; + uchar qs[QK_K/8]; + ushort qh[QK_K/32]; +} block_iq1_s; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq1_s_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq1_s * x = (global block_iq1_s *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq1_s * xb = (global block_iq1_s *)(xrow + row*nb01); + + ushort qhb = xb->qh[it]; + float dl = (float)xb->d * (float)(2*((qhb >> 12) & 7) + 1); + float dlt = (qhb & 0x8000) ? (-1.f - IQ1_S_DELTA) : (-1.f + IQ1_S_DELTA); + + global uchar * qsb = xb->qs + 4*it; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uint gi = (uint)qsb[l] | ((((uint)qhb >> (3*l)) & 7) << 8); + uint g = iq1_s_grid[gi]; + for (int j = 0; j < 4; ++j) { + acc += yl[8*l+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt); + acc += yl[8*l+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt); + } + } + + sumf[row] += dl * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32_flat.cl new file mode 100644 index 000000000..9aba05b9e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32_flat.cl @@ -0,0 +1,139 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 +#define IQ1_S_DELTA 0.125f + +typedef struct { + half d; + uchar qs[QK_K/8]; + ushort qh[QK_K/32]; +} block_iq1_s; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq1_s_f32_flat( + global uchar * src0_qs, + global ushort * src0_qh, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq1_s); + ulong row_stride = nb01/sizeof(block_iq1_s); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + ushort qhb = src0_qh[8*b + it]; + float dl = (float)src0_d[b] * (float)(2*((qhb >> 12) & 7) + 1); + float dlt = (qhb & 0x8000) ? (-1.f - IQ1_S_DELTA) : (-1.f + IQ1_S_DELTA); + + global uchar * qsb = src0_qs + 32*b + 4*it; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uint gi = (uint)qsb[l] | ((((uint)qhb >> (3*l)) & 7) << 8); + uint g = iq1_s_grid[gi]; + for (int j = 0; j < 4; ++j) { + acc += yl[8*l+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt); + acc += yl[8*l+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt); + } + } + + sumf[row] += dl * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32.cl new file mode 100644 index 000000000..43650972b --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32.cl @@ -0,0 +1,147 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +// qs holds 32 grid low bytes then 32 sign bytes +typedef struct { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar scales[QK_K/32]; +} block_iq2_s; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq2_s_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq2_s * x = (global block_iq2_s *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq2_s * xb = (global block_iq2_s *)(xrow + row*nb01); + + uchar scb = xb->scales[it]; + float dlo = (float)xb->d * (0.5f + (float)(scb & 0xf)) * 0.25f; + float dhi = (float)xb->d * (0.5f + (float)(scb >> 4)) * 0.25f; + + uchar qhb = xb->qh[it]; + global uchar * qsb = xb->qs + 4*it; + global uchar * sgb = xb->qs + 32 + 4*it; + + float alo = 0.f, ahi = 0.f; + for (int l = 0; l < 4; ++l) { + uint gi = (uint)qsb[l] | (((uint)qhb << (8-2*l)) & 0x300); + uchar sg = sgb[l]; + uint lo = iq2_s_grid[2*gi+0]; + uint hi = iq2_s_grid[2*gi+1]; + float a = 0.f; + for (int j = 0; j < 4; ++j) { + a += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + a += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + if (l < 2) alo += a; else ahi += a; + } + + sumf[row] += dlo * alo + dhi * ahi; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32_flat.cl new file mode 100644 index 000000000..94d3ffd6f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32_flat.cl @@ -0,0 +1,148 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +// qs holds 32 grid low bytes then 32 sign bytes +typedef struct { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar scales[QK_K/32]; +} block_iq2_s; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq2_s_f32_flat( + global uchar * src0_qs, + global uchar * src0_sg, + global uchar * src0_qh, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq2_s); + ulong row_stride = nb01/sizeof(block_iq2_s); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + uchar scb = src0_sc[8*b + it]; + float dlo = (float)src0_d[b] * (0.5f + (float)(scb & 0xf)) * 0.25f; + float dhi = (float)src0_d[b] * (0.5f + (float)(scb >> 4)) * 0.25f; + + uchar qhb = src0_qh[8*b + it]; + global uchar * qsb = src0_qs + 32*b + 4*it; + global uchar * sgb = src0_sg + 32*b + 4*it; + + float alo = 0.f, ahi = 0.f; + for (int l = 0; l < 4; ++l) { + uint gi = (uint)qsb[l] | (((uint)qhb << (8-2*l)) & 0x300); + uchar sg = sgb[l]; + uint lo = iq2_s_grid[2*gi+0]; + uint hi = iq2_s_grid[2*gi+1]; + float a = 0.f; + for (int j = 0; j < 4; ++j) { + a += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + a += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + if (l < 2) alo += a; else ahi += a; + } + + sumf[row] += dlo * alo + dhi * ahi; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32.cl new file mode 100644 index 000000000..264e8e8e2 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32.cl @@ -0,0 +1,143 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort qs[QK_K/8]; + uchar scales[QK_K/32]; +} block_iq2_xs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq2_xs_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq2_xs * x = (global block_iq2_xs *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq2_xs * xb = (global block_iq2_xs *)(xrow + row*nb01); + + uchar scb = xb->scales[it]; + float dlo = (float)xb->d * (0.5f + (float)(scb & 0xf)) * 0.25f; + float dhi = (float)xb->d * (0.5f + (float)(scb >> 4)) * 0.25f; + + global ushort * q16 = xb->qs + 4*it; + + float alo = 0.f, ahi = 0.f; + for (int l = 0; l < 4; ++l) { + ushort q = q16[l]; + uchar sg = iq2_xs_signs(q >> 9); + uint lo = iq2_xs_grid[2*(q & 511)+0]; + uint hi = iq2_xs_grid[2*(q & 511)+1]; + float a = 0.f; + for (int j = 0; j < 4; ++j) { + a += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + a += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + if (l < 2) alo += a; else ahi += a; + } + + sumf[row] += dlo * alo + dhi * ahi; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32_flat.cl new file mode 100644 index 000000000..a699a0e9f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32_flat.cl @@ -0,0 +1,142 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort qs[QK_K/8]; + uchar scales[QK_K/32]; +} block_iq2_xs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq2_xs_f32_flat( + global ushort * src0_qs, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq2_xs); + ulong row_stride = nb01/sizeof(block_iq2_xs); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + uchar scb = src0_sc[8*b + it]; + float dlo = (float)src0_d[b] * (0.5f + (float)(scb & 0xf)) * 0.25f; + float dhi = (float)src0_d[b] * (0.5f + (float)(scb >> 4)) * 0.25f; + + global ushort * q16 = src0_qs + 32*b + 4*it; + + float alo = 0.f, ahi = 0.f; + for (int l = 0; l < 4; ++l) { + ushort q = q16[l]; + uchar sg = iq2_xs_signs(q >> 9); + uint lo = iq2_xs_grid[2*(q & 511)+0]; + uint hi = iq2_xs_grid[2*(q & 511)+1]; + float a = 0.f; + for (int j = 0; j < 4; ++j) { + a += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + a += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + if (l < 2) alo += a; else ahi += a; + } + + sumf[row] += dlo * alo + dhi * ahi; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32.cl new file mode 100644 index 000000000..7d4595086 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32.cl @@ -0,0 +1,140 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort qs[QK_K/8]; +} block_iq2_xxs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq2_xxs_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq2_xxs * x = (global block_iq2_xxs *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq2_xxs * xb = (global block_iq2_xxs *)(xrow + row*nb01); + + global ushort * q16 = xb->qs + 4*it; + uint a0 = (uint)q16[0] | ((uint)q16[1] << 16); + uint a1 = (uint)q16[2] | ((uint)q16[3] << 16); + + float db = (float)xb->d * (0.5f + (float)(a1 >> 28)) * 0.25f; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uint gi = (a0 >> (8*l)) & 0xFF; + uchar sg = iq2_xxs_signs((a1 >> (7*l)) & 127); + uint lo = iq2_xxs_grid[2*gi+0]; + uint hi = iq2_xxs_grid[2*gi+1]; + for (int j = 0; j < 4; ++j) { + acc += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + acc += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32_flat.cl new file mode 100644 index 000000000..4d9b98416 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32_flat.cl @@ -0,0 +1,137 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort qs[QK_K/8]; +} block_iq2_xxs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq2_xxs_f32_flat( + global uchar * src0_qs, + global uint * src0_sas, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq2_xxs); + ulong row_stride = nb01/sizeof(block_iq2_xxs); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + uint a0 = as_uint(vload4(8*b + it, src0_qs)); + uint a1 = src0_sas[8*b + it]; + + float db = (float)src0_d[b] * (0.5f + (float)(a1 >> 28)) * 0.25f; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uint gi = (a0 >> (8*l)) & 0xFF; + uchar sg = iq2_xxs_signs((a1 >> (7*l)) & 127); + uint lo = iq2_xxs_grid[2*gi+0]; + uint hi = iq2_xxs_grid[2*gi+1]; + for (int j = 0; j < 4; ++j) { + acc += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + acc += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl new file mode 100644 index 000000000..0d9b67b4d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl @@ -0,0 +1,146 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 +#define IQ3_S_N_SCALE (QK_K/64) + +typedef struct { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar signs[QK_K/8]; + uchar scales[IQ3_S_N_SCALE]; +} block_iq3_s; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq3_s_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq3_s * x = (global block_iq3_s *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq3_s * xb = (global block_iq3_s *)(xrow + row*nb01); + + uchar nib = (it & 1) ? (xb->scales[it >> 1] >> 4) : (xb->scales[it >> 1] & 0xf); + float db = (float)xb->d * (float)(1 + 2*nib); + + uchar qhb = xb->qh[it]; + global uchar * qsb = xb->qs + 8*it; + global uchar * sgb = xb->signs + 4*it; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uint i1 = (uint)qsb[2*l+0] | (((uint)qhb << (8-2*l)) & 256); + uint i2 = (uint)qsb[2*l+1] | (((uint)qhb << (7-2*l)) & 256); + uint g1 = iq3_s_grid[i1]; + uint g2 = iq3_s_grid[i2]; + uchar sg = sgb[l]; + for (int j = 0; j < 4; ++j) { + acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl new file mode 100644 index 000000000..0208a53c3 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl @@ -0,0 +1,147 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 +#define IQ3_S_N_SCALE (QK_K/64) + +typedef struct { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar signs[QK_K/8]; + uchar scales[IQ3_S_N_SCALE]; +} block_iq3_s; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq3_s_f32_flat( + global uchar * src0_qs, + global uchar * src0_sg, + global uchar * src0_qh, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq3_s); + ulong row_stride = nb01/sizeof(block_iq3_s); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + uchar nib = (it & 1) ? (src0_sc[4*b + (it >> 1)] >> 4) : (src0_sc[4*b + (it >> 1)] & 0xf); + float db = (float)src0_d[b] * (float)(1 + 2*nib); + + uchar qhb = src0_qh[8*b + it]; + global uchar * qsb = src0_qs + 64*b + 8*it; + global uchar * sgb = src0_sg + 32*b + 4*it; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uint i1 = (uint)qsb[2*l+0] | (((uint)qhb << (8-2*l)) & 256); + uint i2 = (uint)qsb[2*l+1] | (((uint)qhb << (7-2*l)) & 256); + uint g1 = iq3_s_grid[i1]; + uint g2 = iq3_s_grid[i2]; + uchar sg = sgb[l]; + for (int j = 0; j < 4; ++j) { + acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl new file mode 100644 index 000000000..8355621d1 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl @@ -0,0 +1,145 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +// d, then 64 grid index bytes, then 8 uint32 of packed scale+signs +typedef struct { + half d; + uchar qs[3*QK_K/8]; +} block_iq3_xxs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq3_xxs_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq3_xxs * x = (global block_iq3_xxs *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq3_xxs * xb = (global block_iq3_xxs *)(xrow + row*nb01); + + global uchar * qs = xb->qs + 8*it; + + // Blocks are 98 bytes, so packed signs are only 2-byte aligned. + global ushort * sw = (global ushort *)(xb->qs + 64 + 4*it); + uint aux32 = (uint)sw[0] | ((uint)sw[1] << 16); + + float db = (float)xb->d * (0.5f + (float)(aux32 >> 28)) * 0.5f; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uchar signs = iq3_xxs_signs((aux32 >> (7*l)) & 127u); + uint g1 = iq3_xxs_grid[qs[2*l+0]]; + uint g2 = iq3_xxs_grid[qs[2*l+1]]; + + for (int j = 0; j < 4; ++j) { + float s1 = (signs & (1 << j)) ? -1.f : 1.f; + float s2 = (signs & (1 << (j+4))) ? -1.f : 1.f; + acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * s1; + acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * s2; + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl new file mode 100644 index 000000000..974daf859 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl @@ -0,0 +1,142 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +// d, then 64 grid index bytes, then 8 uint32 of packed scale+signs +typedef struct { + half d; + uchar qs[3*QK_K/8]; +} block_iq3_xxs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq3_xxs_f32_flat( + global uchar * src0_qs, + global uint * src0_sas, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq3_xxs); + ulong row_stride = nb01/sizeof(block_iq3_xxs); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + global uchar * qs = src0_qs + 64*b + 8*it; + + uint aux32 = src0_sas[8*b + it]; + + float db = (float)src0_d[b] * (0.5f + (float)(aux32 >> 28)) * 0.5f; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uchar signs = iq3_xxs_signs((aux32 >> (7*l)) & 127u); + uint g1 = iq3_xxs_grid[qs[2*l+0]]; + uint g2 = iq3_xxs_grid[qs[2*l+1]]; + + for (int j = 0; j < 4; ++j) { + float s1 = (signs & (1 << j)) ? -1.f : 1.f; + float s2 = (signs & (1 << (j+4))) ? -1.f : 1.f; + acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * s1; + acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * s2; + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl new file mode 100644 index 000000000..2fe0044eb --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +} block_iq4_xs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq4_xs_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq4_xs * x = (global block_iq4_xs *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq4_xs * xb = (global block_iq4_xs *)(xrow + row*nb01); + + int ls = ((xb->scales_l[it/2] >> (4*(it%2))) & 15) | (((xb->scales_h >> (2*it)) & 3) << 4); + float db = (float)xb->d * (float)(ls - 32); + global uchar * qs = xb->qs + 16*it; + + float acc = 0.f; + for (int j = 0; j < 16; ++j) { + acc += yl[j] * (float)iq4_xs_values[qs[j] & 15]; + acc += yl[j+16] * (float)iq4_xs_values[qs[j] >> 4]; + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl new file mode 100644 index 000000000..4426137f0 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +} block_iq4_xs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq4_xs_f32_flat( + global uchar * src0_qs, + global uchar * src0_sl, + global ushort * src0_sh, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq4_xs); + ulong row_stride = nb01/sizeof(block_iq4_xs); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + int ls = ((src0_sl[4*b + it/2] >> (4*(it%2))) & 15) | (((src0_sh[b] >> (2*it)) & 3) << 4); + float db = (float)src0_d[b] * (float)(ls - 32); + global uchar * qs = src0_qs + 128*b + 16*it; + + float acc = 0.f; + for (int j = 0; j < 16; ++j) { + acc += yl[j] * (float)iq4_xs_values[qs[j] & 15]; + acc += yl[j+16] * (float)iq4_xs_values[qs[j] >> 4]; + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl index 3d5a923ee..2e31831ca 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl @@ -132,7 +132,7 @@ kernel void kernel_mul_mv_mxfp4_f32_flat( global float4 * y4 = (global float4 *)yb; #pragma unroll - for (short row = 0; row < N_R0_MXFP4; row++) { + for (short row = 0; row < N_R0_MXFP4 && first_row + row < ne0; row++) { uchar xb_e = x_e[row * nb + ib]; #ifdef SRC0Q_IMG ushort4 xb_q = as_ushort4(read_imageui(src0_q, (offset_q + row * nb + ib) * 2 + it).xy); diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q2_0_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q2_0_f32.cl new file mode 100644 index 000000000..1be4f5430 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q2_0_f32.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK2_0 64 + +// 00=-1, 01=0, 10=+1, 11=+2 +typedef struct { + half d; + uchar qs[QK2_0/4]; +} block_q2_0; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 4 threads cover one block +#define BLOCK_STRIDE (N_SIMDWIDTH/4) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q2_0_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/4; // block index + int it = get_sub_group_local_id()%4; // part of the block + + int nb = ne00/QK2_0; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_q2_0 * x = (global block_q2_0 *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[16]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK2_0 + 16 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 16; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_q2_0 * xb = (global block_q2_0 *)(xrow + row*nb01); + + global uchar * q = xb->qs + 4*it; + + float acc = 0.f; + for (int i = 0; i < 4; ++i) { + uchar b = q[i]; + acc += yl[4*i+0] * (float)(((b >> 0) & 3) - 1); + acc += yl[4*i+1] * (float)(((b >> 2) & 3) - 1); + acc += yl[4*i+2] * (float)(((b >> 4) & 3) - 1); + acc += yl[4*i+3] * (float)(((b >> 6) & 3) - 1); + } + + sumf[row] += (float)xb->d * acc; + } + + y4 += BLOCK_STRIDE * QK2_0; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q2_0_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q2_0_f32_flat.cl new file mode 100644 index 000000000..7fd8899d2 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q2_0_f32_flat.cl @@ -0,0 +1,132 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK2_0 64 + +// 00=-1, 01=0, 10=+1, 11=+2 +typedef struct { + half d; + uchar qs[QK2_0/4]; +} block_q2_0; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 4 threads cover one block +#define BLOCK_STRIDE (N_SIMDWIDTH/4) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q2_0_f32_flat( + global uchar * src0_qs, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/4; // block index + int it = get_sub_group_local_id()%4; // part of the block + + int nb = ne00/QK2_0; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_q2_0); + ulong row_stride = nb01/sizeof(block_q2_0); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[16]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK2_0 + 16 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 16; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + global uchar * q = src0_qs + 16*b + 4*it; + + float acc = 0.f; + for (int i = 0; i < 4; ++i) { + uchar b = q[i]; + acc += yl[4*i+0] * (float)(((b >> 0) & 3) - 1); + acc += yl[4*i+1] * (float)(((b >> 2) & 3) - 1); + acc += yl[4*i+2] * (float)(((b >> 4) & 3) - 1); + acc += yl[4*i+3] * (float)(((b >> 6) & 3) - 1); + } + + sumf[row] += (float)src0_d[b] * acc; + } + + y4 += BLOCK_STRIDE * QK2_0; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl new file mode 100644 index 000000000..7f5328136 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl @@ -0,0 +1,161 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// block_q2_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q + b +typedef struct { + uchar scales[QK_K/16]; // scales and mins, quantized with 4 bits + uchar qs[QK_K/4]; // 2-bit quants + half d; // super-block scale for quantized scales + half dmin; // super-block scale for quantized mins +} block_q2_K; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// number of (super) blocks each subgroup processes +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q2_K_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // block index (inside super block) + int iq = it/4; // 0 or 1 - first or second half of the super block + int ir = it%4; // 0...3 - block index in the half super block + int is = (8*ir)/16; // 0 or 1 - scale pair + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_q2_K * x = (global block_q2_K *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 128 * iq + 8 * ir; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; ++i) { + yl[i+ 0] = y4[i+ 0]; sumy.s0 += yl[i+ 0]; + yl[i+ 8] = y4[i+32]; sumy.s1 += yl[i+ 8]; + yl[i+16] = y4[i+64]; sumy.s2 += yl[i+16]; + yl[i+24] = y4[i+96]; sumy.s3 += yl[i+24]; + } + + global uchar * sc = (global uchar *)x[ib].scales + 8*iq + is; + global ushort * qs = (global ushort *)x[ib].qs + 16 * iq + 4 * ir; + global half * dh = &x[ib].d; + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + float4 acc1 = {0.f, 0.f, 0.f, 0.f}; + float4 acc2 = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; i += 2) { + acc1.s0 += yl[i+ 0] * (qs[i/2] & 0x0003); + acc2.s0 += yl[i+ 1] * (qs[i/2] & 0x0300); + acc1.s1 += yl[i+ 8] * (qs[i/2] & 0x000c); + acc2.s1 += yl[i+ 9] * (qs[i/2] & 0x0c00); + acc1.s2 += yl[i+16] * (qs[i/2] & 0x0030); + acc2.s2 += yl[i+17] * (qs[i/2] & 0x3000); + acc1.s3 += yl[i+24] * (qs[i/2] & 0x00c0); + acc2.s3 += yl[i+25] * (qs[i/2] & 0xc000); + } + + float dall = dh[0]; + float dmin = dh[1] * 1.f/16.f; + sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc2.s0) * (sc[0] & 0xF) * 1.f/ 1.f + + (acc1.s1 + 1.f/256.f * acc2.s1) * (sc[2] & 0xF) * 1.f/ 4.f + + (acc1.s2 + 1.f/256.f * acc2.s2) * (sc[4] & 0xF) * 1.f/16.f + + (acc1.s3 + 1.f/256.f * acc2.s3) * (sc[6] & 0xF) * 1.f/64.f) - + dmin * (sumy.s0 * (sc[0] & 0xF0) + sumy.s1 * (sc[2] & 0xF0) + + sumy.s2 * (sc[4] & 0xF0) + sumy.s3 * (sc[6] & 0xF0)); + + qs += nb01/2; + sc += nb01; + dh += nb01/2; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl new file mode 100644 index 000000000..c993abebe --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl @@ -0,0 +1,162 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// struct block_q2_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q + b +struct block_q2_K { + uchar scales[QK_K/16]; // scales and mins, quantized with 4 bits + uchar qs[QK_K/4]; // 2-bit quants + half d; // super-block scale for quantized scales + half dmin; // super-block scale for quantized mins +}; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// number of (super) blocks each subgroup processes +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q2_K_f32_flat( + global uchar * src0_qs, + global uchar * src0_sc, + global half * src0_dm, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // block index (inside super block) + int iq = it/4; // 0 or 1 - first or second half of the super block + int ir = it%4; // 0...3 - block index in the half super block + int is = (8*ir)/16; // 0 or 1 - scale pair + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(struct block_q2_K); + ulong row_stride = nb01/sizeof(struct block_q2_K); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 128 * iq + 8 * ir; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; ++i) { + yl[i+ 0] = y4[i+ 0]; sumy.s0 += yl[i+ 0]; + yl[i+ 8] = y4[i+32]; sumy.s1 += yl[i+ 8]; + yl[i+16] = y4[i+64]; sumy.s2 += yl[i+16]; + yl[i+24] = y4[i+96]; sumy.s3 += yl[i+24]; + } + + global uchar * sc = (src0_sc + 16*(offset_src0 + ib)) + 8*iq + is; + global ushort * qs = (global ushort *)(src0_qs + 64*(offset_src0 + ib)) + 16 * iq + 4 * ir; + global half * dh = src0_dm + 2*(offset_src0 + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + float4 acc1 = {0.f, 0.f, 0.f, 0.f}; + float4 acc2 = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; i += 2) { + acc1.s0 += yl[i+ 0] * (qs[i/2] & 0x0003); + acc2.s0 += yl[i+ 1] * (qs[i/2] & 0x0300); + acc1.s1 += yl[i+ 8] * (qs[i/2] & 0x000c); + acc2.s1 += yl[i+ 9] * (qs[i/2] & 0x0c00); + acc1.s2 += yl[i+16] * (qs[i/2] & 0x0030); + acc2.s2 += yl[i+17] * (qs[i/2] & 0x3000); + acc1.s3 += yl[i+24] * (qs[i/2] & 0x00c0); + acc2.s3 += yl[i+25] * (qs[i/2] & 0xc000); + } + + float dall = dh[0]; + float dmin = dh[1] * 1.f/16.f; + sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc2.s0) * (sc[0] & 0xF) * 1.f/ 1.f + + (acc1.s1 + 1.f/256.f * acc2.s1) * (sc[2] & 0xF) * 1.f/ 4.f + + (acc1.s2 + 1.f/256.f * acc2.s2) * (sc[4] & 0xF) * 1.f/16.f + + (acc1.s3 + 1.f/256.f * acc2.s3) * (sc[6] & 0xF) * 1.f/64.f) - + dmin * (sumy.s0 * (sc[0] & 0xF0) + sumy.s1 * (sc[2] & 0xF0) + + sumy.s2 * (sc[4] & 0xF0) + sumy.s3 * (sc[6] & 0xF0)); + + qs += row_stride*32; + sc += row_stride*16; + dh += row_stride*2; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl new file mode 100644 index 000000000..77c70f4ac --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl @@ -0,0 +1,197 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// block_q3_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q +typedef struct { + uchar hmask[QK_K/8]; // quants - high bit + uchar qs[QK_K/4]; // quants - low 2 bits + uchar scales[12]; // scales, quantized with 6 bits + half d; // super-block scale +} block_q3_K; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 2 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 2 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q3_K_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_q3_K * x = (global block_q3_K *) (src0 + offset_src0); + global float * yy = (global float *) (src1 + offset_src1); + + float yl[32]; + + // Eight lanes cover one super-block. + int ix = get_sub_group_local_id() % BLOCK_STRIDE; + int tid = get_sub_group_local_id() / BLOCK_STRIDE; + + int ip = tid/4; // 0 or 1 + int il = 2*((tid%4)/2); // 0 or 2 + int ir = tid%2; + int l0 = 8*ir; + + // Shift the masks instead of indexing private arrays to avoid spills. + ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2)); + int4 qm = (int4) (0x0003, 0x0300, 0x000c, 0x0c00) << (int4) (4*(il/2)); + + int shift = 2*il; + + float v1 = il == 0 ? 4.f : 64.f; + float v2 = 4.f * v1; + + int s_shift1 = 4*ip; + int s_shift2 = s_shift1 + il; + + int q_offset = 32*ip + l0; + int y_offset = 128*ip + 32*il + l0; + + global float * y1 = yy + ix*QK_K + y_offset; + + float sumf1[N_DST] = {0.f}; + float sumf2[N_DST] = {0.f}; + float all_sum; + + for (int i = ix; i < nb; i += BLOCK_STRIDE) { + for (int l = 0; l < 8; ++l) { + yl[l+ 0] = y1[l+ 0]; + yl[l+ 8] = y1[l+16]; + yl[l+16] = y1[l+32]; + yl[l+24] = y1[l+48]; + } + + global ushort * q = (global ushort *)(x[i].qs + q_offset); + global ushort * h = (global ushort *)(x[i].hmask + l0); + global ushort * a = (global ushort *)(x[i].scales); + global half * dh = &x[i].d; + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; ++row) { + float d_all = dh[0]; + + uint s32 = (uint)a[4] | ((uint)a[5] << 16); + uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u; + s32 = (uint)a[il+0] | ((uint)a[il+1] << 16); + s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32; + char4 sc = as_char4(s32); + + float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2]; + s1 += yl[l+0] * (qs & qm.s0); + s2 += yl[l+1] * (qs & qm.s1); + s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]); + s4 += yl[l+16] * (qs & qm.s2); + s5 += yl[l+17] * (qs & qm.s3); + s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]); + } + float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s0 - 32); + sumf2[row] += d2 * (sc.s2 - 32); + + s1 = s2 = s3 = s4 = s5 = s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2+8]; + s1 += yl[l+8] * (qs & qm.s0); + s2 += yl[l+9] * (qs & qm.s1); + s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]); + s4 += yl[l+24] * (qs & qm.s2); + s5 += yl[l+25] * (qs & qm.s3); + s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]); + } + d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s1 - 32); + sumf2[row] += d2 * (sc.s3 - 32); + + q += nb01/2; + h += nb01/2; + a += nb01/2; + dh += nb01/2; + } + + y1 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift)); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl new file mode 100644 index 000000000..c2c2219d8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl @@ -0,0 +1,199 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// block_q3_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q +typedef struct { + uchar hmask[QK_K/8]; // quants - high bit + uchar qs[QK_K/4]; // quants - low 2 bits + uchar scales[12]; // scales, quantized with 6 bits + half d; // super-block scale +} block_q3_K; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 2 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 2 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q3_K_f32_flat( + global uchar * src0_qs, + global uchar * src0_hm, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_q3_K); + ulong row_stride = nb01/sizeof(block_q3_K); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * yy = (global float *) (src1 + offset_src1); + + float yl[32]; + + // Eight lanes cover one super-block. + int ix = get_sub_group_local_id() % BLOCK_STRIDE; + int tid = get_sub_group_local_id() / BLOCK_STRIDE; + + int ip = tid/4; // 0 or 1 + int il = 2*((tid%4)/2); // 0 or 2 + int ir = tid%2; + int l0 = 8*ir; + + // Shift the masks instead of indexing private arrays to avoid spills. + ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2)); + int4 qm = (int4) (0x0003, 0x0300, 0x000c, 0x0c00) << (int4) (4*(il/2)); + + int shift = 2*il; + + float v1 = il == 0 ? 4.f : 64.f; + float v2 = 4.f * v1; + + int s_shift1 = 4*ip; + int s_shift2 = s_shift1 + il; + + int q_offset = 32*ip + l0; + int y_offset = 128*ip + 32*il + l0; + + global float * y1 = yy + ix*QK_K + y_offset; + + float sumf1[N_DST] = {0.f}; + float sumf2[N_DST] = {0.f}; + float all_sum; + + for (int i = ix; i < nb; i += BLOCK_STRIDE) { + for (int l = 0; l < 8; ++l) { + yl[l+ 0] = y1[l+ 0]; + yl[l+ 8] = y1[l+16]; + yl[l+16] = y1[l+32]; + yl[l+24] = y1[l+48]; + } + + global ushort * q = (global ushort *)(src0_qs + 64*(offset_src0 + i) + q_offset); + global ushort * h = (global ushort *)(src0_hm + 32*(offset_src0 + i) + l0); + global ushort * a = (global ushort *)(src0_sc + 12*(offset_src0 + i)); + global half * dh = src0_d + offset_src0 + i; + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; ++row) { + float d_all = dh[0]; + + uint s32 = (uint)a[4] | ((uint)a[5] << 16); + uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u; + s32 = (uint)a[il+0] | ((uint)a[il+1] << 16); + s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32; + char4 sc = as_char4(s32); + + float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2]; + s1 += yl[l+0] * (qs & qm.s0); + s2 += yl[l+1] * (qs & qm.s1); + s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]); + s4 += yl[l+16] * (qs & qm.s2); + s5 += yl[l+17] * (qs & qm.s3); + s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]); + } + float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s0 - 32); + sumf2[row] += d2 * (sc.s2 - 32); + + s1 = s2 = s3 = s4 = s5 = s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2+8]; + s1 += yl[l+8] * (qs & qm.s0); + s2 += yl[l+9] * (qs & qm.s1); + s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]); + s4 += yl[l+24] * (qs & qm.s2); + s5 += yl[l+25] * (qs & qm.s3); + s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]); + } + d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s1 - 32); + sumf2[row] += d2 * (sc.s3 - 32); + + q += row_stride*32; + h += row_stride*16; + a += row_stride*6; + dh += row_stride*1; + } + + y1 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift)); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl index 3ce06305a..5b6dea773 100644 --- a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +++ b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl @@ -1,9 +1,7 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable -// Quantize a contiguous [N, K] f32 activation buffer (token-major, K contiguous -// per token) into q8_1 blocks of 32: int8 quants + per-block scale d + per-block -// sum s (= d * Sum(qs)). Consumed by kernel_gemm_noshuffle_q4_k_q8_1_dp4a for the -// dp4a (int8) dense q4_K prefill GEMM. One work-item per 32-element block. +// Quantize a contiguous K-major [N, K] f32 activation buffer (K contiguous) into 32-value Q8_1 SoA blocks. +// Store int8 quants, per-block scale d, and scaled sum s = d * sum(qs); one work-item handles each block. __kernel void kernel_quant_a_q8_1( __global const float * src, // [N * K] __global char * qa, // [N * K] diff --git a/ggml/src/ggml-opencl/kernels/repeat.cl b/ggml/src/ggml-opencl/kernels/repeat.cl index 53951a554..88dee31cc 100644 --- a/ggml/src/ggml-opencl/kernels/repeat.cl +++ b/ggml/src/ggml-opencl/kernels/repeat.cl @@ -1,38 +1,43 @@ -kernel void kernel_repeat_f32( - global const char * src0, - ulong offset0, - global char * dst, - ulong offsetd, - int ne00, - int ne01, - int ne02, - int ne03, - ulong nb00, - ulong nb01, - ulong nb02, - ulong nb03, - int ne0, - ulong nb0, - ulong nb1, - ulong nb2, - ulong nb3 -) { - src0 = src0 + offset0; - dst = dst + offsetd; - - const int i3 = get_group_id(2); - const int i2 = get_group_id(1); - const int i1 = get_group_id(0); - - const int i03 = i3%ne03; - const int i02 = i2%ne02; - const int i01 = i1%ne01; - - global const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01; - global char * dst_ptr = dst + i3*nb3 + i2*nb2 + i1*nb1; - - for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { - const int i00 = i0%ne00; - *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i00*nb00)); - } +#define KERNEL_REPEAT(SUFFIX, T) \ +kernel void kernel_repeat_##SUFFIX( \ + global const char * src0, \ + ulong offset0, \ + global char * dst, \ + ulong offsetd, \ + int ne00, \ + int ne01, \ + int ne02, \ + int ne03, \ + ulong nb00, \ + ulong nb01, \ + ulong nb02, \ + ulong nb03, \ + int ne0, \ + ulong nb0, \ + ulong nb1, \ + ulong nb2, \ + ulong nb3 \ +) { \ + src0 = src0 + offset0; \ + dst = dst + offsetd; \ + \ + const int i3 = get_group_id(2); \ + const int i2 = get_group_id(1); \ + const int i1 = get_group_id(0); \ + \ + const int i03 = i3%ne03; \ + const int i02 = i2%ne02; \ + const int i01 = i1%ne01; \ + \ + global const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01; \ + global char * dst_ptr = dst + i3*nb3 + i2*nb2 + i1*nb1; \ + \ + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { \ + const int i00 = i0%ne00; \ + *((global T *)(dst_ptr + i0*nb0)) = *((global T *)(src0_ptr + i00*nb00)); \ + } \ } + +KERNEL_REPEAT(f32, float) +KERNEL_REPEAT(f16, ushort) +KERNEL_REPEAT(i32, int) diff --git a/ggml/src/ggml-opencl/kernels/rope.cl b/ggml/src/ggml-opencl/kernels/rope.cl index 27fdbbbc4..5b99d42de 100644 --- a/ggml/src/ggml-opencl/kernels/rope.cl +++ b/ggml/src/ggml-opencl/kernels/rope.cl @@ -520,7 +520,7 @@ kernel void kernel_rope_multi_f16( src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); src2 = (global float*)((global char*)src2 + offset2); - dst = (global float*)((global char*)dst + offsetd); + dst = (global half*)((global char*)dst + offsetd); int i3 = get_group_id(2); int i2 = get_group_id(1); @@ -714,7 +714,7 @@ kernel void kernel_rope_vision_f16( src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); src2 = (global float*)((global char*)src2 + offset2); - dst = (global float*)((global char*)dst + offsetd); + dst = (global half*)((global char*)dst + offsetd); int i3 = get_group_id(2); int i2 = get_group_id(1); diff --git a/ggml/src/ggml-opencl/kernels/sigmoid.cl b/ggml/src/ggml-opencl/kernels/sigmoid.cl index e3f669dde..cfe9f7295 100644 --- a/ggml/src/ggml-opencl/kernels/sigmoid.cl +++ b/ggml/src/ggml-opencl/kernels/sigmoid.cl @@ -27,3 +27,63 @@ kernel void kernel_sigmoid_f16( dst[get_global_id(0)] = 1.0f / (1.0f + exp(-src0[get_global_id(0)])); } + +kernel void kernel_sigmoid_f32_nc( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + int ne00, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + dst = dst + offsetd; + + const int i3 = get_group_id(2); + const int i2 = get_group_id(1); + const int i1 = get_group_id(0); + + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { + global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + *y = 1.0f / (1.0f + exp(-*x)); + } +} + +kernel void kernel_sigmoid_f16_nc( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + int ne00, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + dst = dst + offsetd; + + const int i3 = get_group_id(2); + const int i2 = get_group_id(1); + const int i1 = get_group_id(0); + + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { + global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + *y = 1.0f / (1.0f + exp(-(float)*x)); + } +} diff --git a/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl b/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl index 571d16507..8dfc5519d 100644 --- a/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +++ b/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl @@ -106,3 +106,176 @@ kernel void kernel_soft_max_4_f16( pdst4[i00] /= sum; } } + +// Same as kernel_soft_max_4_f16, but leaves exp(x - max) in dst and writes the row sum to psums. +// The decomposed flash attention applies 1/sum to the KQV result, which is much smaller than P. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_fa_decompose_soft_max_4_f16( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * src2, + ulong offset2, + global char * dst, + ulong offsetd, + global char * psums, + ulong offset_sums, + int ne00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + int ne13, + ulong nb11, + ulong nb12, + ulong nb13, + ulong nb1, + ulong nb2, + ulong nb3, + float scale, + float max_bias, + float m0, + float m1, + int n_head_log2 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + src2 = src2 + offset2; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0); + + int i13 = i03%ne13; + int i12 = i02%ne12; + int i11 = i01; + + global float4 * psrc4 = (global float4 *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + global half4 * pmask = src1 != src0 ? (global half4 *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0; + global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0; + global float4 * pdst4 = (global float4 *)(dst + i01*nb1 + i02*nb2 + i03*nb3); + + float slope = 1.0f; + + // ALiBi + if (max_bias > 0.0f) { + int h = i02; + + float base = h < n_head_log2 ? m0 : m1; + int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slope = pow(base, exp); + } + + // parallel max + float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY; + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + lmax4 = fmax(lmax4, psrc4[i00]*scale + slope*(pmask ? convert_float4(pmask[i00]) : 0.0f)); + } + float lmax = fmax(fmax(lmax4.s0, lmax4.s1), fmax(lmax4.s2, lmax4.s3)); + + float max = sub_group_reduce_max(lmax); + // Avoid -inf - (-inf) for fully masked rows. + if (max == -INFINITY) { + max = 0.0f; + } + + // parallel sum + float4 lsum4 = 0.0f; + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + const float4 exp_psrc4 = exp((psrc4[i00]*scale + slope*(pmask ? convert_float4(pmask[i00]) : 0.0f)) - max); + lsum4 += exp_psrc4; + pdst4[i00] = exp_psrc4; + } + float lsum = lsum4.s0 + lsum4.s1 + lsum4.s2 + lsum4.s3; + + float sum = sub_group_reduce_add(lsum); + + if (psrc2) { + sum += exp(psrc2[i02] - max); + } + + if (get_local_id(0) == 0) { + global float * sums = (global float *)(psums + offset_sums); + sums[(i03*get_num_groups(1) + i02)*get_num_groups(0) + i01] = sum; + } +} + +// out[:, n, h] *= 1/sums[h*ne1 + n], out is [dv, ne1, n_head] contiguous. +kernel void kernel_fa_decompose_scale_rows_f32( + global char * dst, + ulong offsetd, + global char * psums, + ulong offset_sums, + int dv, + int ne1 +) { + global float * out = (global float *)(dst + offsetd); + global float * sums = (global float *)(psums + offset_sums); + + const int n = get_group_id(0); + const int h = get_group_id(1); + + const float s = sums[h*ne1 + n]; + const float r = s > 0.0f ? 1.0f/s : 0.0f; + + global float * row = out + ((size_t)h*ne1 + n)*dv; + for (int i = get_local_id(0); i < dv; i += get_local_size(0)) { + row[i] *= r; + } +} + +// Merge the P8 block maxima and sums into F32 block scales and row sums. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_fa_p8_fixup( + global float * bmax, + global float * bsum, + global float * dp, + global float * psums, + global char * sinks, + ulong offset_sinks, + int has_sinks, + int nblk, + int row_blk, // blocks per row of bmax/bsum/dp (the padded pitch), >= nblk + int N +) { + const int n = get_group_id(0); + const int head = get_group_id(1); + const uint row = (uint)head*(uint)N + (uint)n; + global float * bm = bmax + (size_t)row*row_blk; + global float * bs = bsum + (size_t)row*row_blk; + global float * dr = dp + (size_t)row*row_blk; + + float lmax = -INFINITY; + for (int b = get_local_id(0); b < nblk; b += get_local_size(0)) { + lmax = fmax(lmax, bm[b]); + } + float rowmax = sub_group_reduce_max(lmax); + float sink = 0.0f; + if (has_sinks) { + sink = ((global float *)(sinks + offset_sinks))[head]; + rowmax = fmax(rowmax, sink); + } + + float lsum = 0.0f; + for (int b = get_local_id(0); b < nblk; b += get_local_size(0)) { + const float m = bm[b]; + // Use a finite sentinel: -cl-finite-math-only can remove infinity checks. + const float f = (m > -1.0e30f && rowmax > -1.0e30f) ? exp(m - rowmax) : 0.0f; + dr[b] = f / 255.0f; + lsum += bs[b]*f; + } + float sum = sub_group_reduce_add(lsum); + if (has_sinks && rowmax > -1.0e30f) { + sum += exp(sink - rowmax); + } + if (get_local_id(0) == 0) { + psums[row] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/top_k.cl b/ggml/src/ggml-opencl/kernels/top_k.cl new file mode 100644 index 000000000..6b5126e0d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/top_k.cl @@ -0,0 +1,207 @@ +#ifdef cl_khr_subgroups +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +uint top_k_key(float value) { + uint bits = as_uint(value); + if ((bits & 0x7fffffffu) == 0) { + bits = 0; + } + return bits & 0x80000000u ? ~bits : bits ^ 0x80000000u; +} + +uint4 top_k_keys(global const char * src, int i, int n, ulong nb0) { + if (nb0 == sizeof(float) && i + 3 < n) { + const float4 v = vload4(0, (global const float *) src + i); + return (uint4) (top_k_key(v.s0), top_k_key(v.s1), top_k_key(v.s2), top_k_key(v.s3)); + } + return (uint4) ( + i < n ? top_k_key(*(global const float *) (src + (ulong) i *nb0)) : 0, + i + 1 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 1)*nb0)) : 0, + i + 2 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 2)*nb0)) : 0, + i + 3 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 3)*nb0)) : 0); +} + +void top_k_store(global int * dst, global uint2 * candidates, global const char * src, + ulong nb0, int indexed, int base, int i, uint pos) { + const int index = indexed ? *(global const int *) (src + (ulong) i*nb0 + sizeof(float)) : base + i; + if (candidates) { + candidates[pos] = (uint2) (as_uint(*(global const float *) (src + (ulong) i*nb0)), index); + } else { + dst[pos] = index; + } +} + +void top_k_select( + global const char * src, + global int * dst, + global uint2 * candidates, + ulong nb00, + int ne00, + int k, + int base, + int indexed, + local uint * hist +) { + const int tid = get_local_id(0); + const int nth = get_local_size(0); + if (k == ne00) { + for (int i = tid; i < ne00; i += nth) { + top_k_store(dst, candidates, src, nb00, indexed, base, i, i); + } + return; + } + + uint prefix = 0; + uint mask = 0; + uint rank = k; + for (int shift = 24; shift >= 0; shift -= 8) { + for (int i = tid; i < 256; i += nth) { + hist[i] = 0; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int i = 4*tid; i < ne00; i += 4*nth) { + const uint4 v = top_k_keys(src, i, ne00, nb00); + const uint keys[4] = {v.s0, v.s1, v.s2, v.s3}; + #pragma unroll + for (int j = 0; j < 4; ++j) { + const uint key = keys[j]; + if (i + j < ne00 && (key & mask) == prefix) { + atomic_inc(hist + ((key >> shift) & 255)); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if (tid == 0) { + uint above = 0; + for (int b = 255; b >= 0; --b) { + uint count = hist[b]; + if (above + count >= rank) { + hist[256] = b; + hist[257] = rank - above; + hist[258] = count; + break; + } + above += count; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + prefix |= hist[256] << shift; + rank = hist[257]; + const bool done = hist[258] == rank; + barrier(CLK_LOCAL_MEM_FENCE); + if (done) { + // The whole bucket is selected, so its low bits do not matter. + break; + } + mask |= 255u << shift; + } + + uint greater = 0; + uint equal = 0; + for (int i = 4*tid; i < ne00; i += 4*nth) { + const uint4 v = top_k_keys(src, i, ne00, nb00); + const uint keys[4] = {v.s0, v.s1, v.s2, v.s3}; + #pragma unroll + for (int j = 0; j < 4; ++j) { + greater += i + j < ne00 && keys[j] > prefix; + equal += i + j < ne00 && keys[j] == prefix; + } + } + uint pos_greater; + uint pos_equal; +#ifdef cl_khr_subgroups + const uint sg = get_sub_group_id(); + const uint nsg = get_num_sub_groups(); + const uint total_greater = sub_group_reduce_add(greater); + const uint total_equal = sub_group_reduce_add(equal); + pos_greater = sub_group_scan_exclusive_add(greater); + pos_equal = sub_group_scan_exclusive_add(equal); + if (get_sub_group_local_id() == 0) { + hist[sg] = total_greater; + hist[nsg + sg] = total_equal; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (uint i = 0; i < nsg; ++i) { + pos_equal += hist[i]; + if (i < sg) { + pos_greater += hist[i]; + pos_equal += hist[nsg + i]; + } + } +#else + hist[tid] = greater; + hist[nth + tid] = equal; + barrier(CLK_LOCAL_MEM_FENCE); + pos_greater = 0; + pos_equal = 0; + for (int i = 0; i < nth; ++i) { + pos_equal += hist[i]; + if (i < tid) { + pos_greater += hist[i]; + pos_equal += hist[nth + i]; + } + } +#endif + for (int i = 4*tid; i < ne00; i += 4*nth) { + const uint4 v = top_k_keys(src, i, ne00, nb00); + const uint keys[4] = {v.s0, v.s1, v.s2, v.s3}; + #pragma unroll + for (int j = 0; j < 4; ++j) { + if (i + j < ne00) { + if (keys[j] > prefix) { + top_k_store(dst, candidates, src, nb00, indexed, base, i + j, pos_greater++); + } else if (keys[j] == prefix && pos_equal < k) { + top_k_store(dst, candidates, src, nb00, indexed, base, i + j, pos_equal++); + } + } + } + } +} + +kernel void kernel_top_k_f32( + global const char * src, + ulong offset0, + global int * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int k_max, + local uint * hist, + global uint2 * candidates, + int tile_width +) { + const ulong row = get_group_id(1); + const int part = get_group_id(0); + const int parts = get_num_groups(0); + const int base = part*tile_width; + const int n = min(ne00 - base, tile_width); + const int k = min(k_max, n); + const int out_width = (parts - 1)*k_max + min(k_max, ne00 - (parts - 1)*tile_width); + src += offset0 + (row % ne01)*nb01 + (row / ne01 % ne02)*nb02 + (row / ne01 / ne02)*nb03 + (ulong) base*nb00; + if (candidates) { + candidates += row*out_width + part*k_max; + } else { + dst = (global int *) ((global char *) dst + offsetd) + row*out_width + part*k_max; + } + top_k_select(src, dst, candidates, nb00, n, k, base, 0, hist); +} + +kernel void kernel_top_k_merge_f32( + global const uint2 * candidates, + global int * dst, + ulong offsetd, + int n_candidates, + int k, + local uint * hist +) { + const ulong row = get_group_id(1); + candidates += row*n_candidates; + dst = (global int *) ((global char *) dst + offsetd) + row*k; + top_k_select((global const char *) candidates, dst, 0, sizeof(uint2), n_candidates, k, 0, 1, hist); +} diff --git a/ggml/src/ggml-opencl/kernels/top_k_mask.cl b/ggml/src/ggml-opencl/kernels/top_k_mask.cl new file mode 100644 index 000000000..00fc0fb13 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/top_k_mask.cl @@ -0,0 +1,129 @@ +// Integer storage preserves the mask bits without FP16 conversion. +#if MASK_F16 +#define MASK_T ushort +#define MASK_V ushort8 +#define MASK_NV 8 +#define MASK_NINF 0xfc00 +#define MASK_STORE vstore8 +#else +#define MASK_T uint +#define MASK_V uint4 +#define MASK_NV 4 +#define MASK_NINF 0xff800000u +#define MASK_STORE vstore4 +#endif + +kernel void kernel_top_k_mask_fill( + global char * dst, + ulong offsetd, + ulong n +) { + global MASK_T * out = (global MASK_T *) (dst + offsetd); + const ulong c = get_global_id(0)*MASK_NV; + if (c + MASK_NV <= n) { + MASK_STORE((MASK_V) (MASK_NINF), 0, out + c); + } else { + for (ulong j = c; j < n; ++j) { + out[j] = MASK_NINF; + } + } +} + +kernel void kernel_top_k_mask( + global const char * mask, + ulong offsetm, + global const char * top_k, + ulong offsetk, + global const char * score, + ulong offsets, + global const char * blocks, + ulong offsetb, + global const char * extra, + ulong offsete, + global char * dst, + ulong offsetd, + ulong nem0, + ulong nem1, + ulong nem2, + ulong nem3, + ulong nbm1, + ulong nbm2, + ulong nbm3, + ulong nek0, + ulong nbk0, + ulong nbk1, + ulong nbk2, + ulong nbk3, + ulong nes0, + ulong nbs0, + ulong nbs1, + ulong nbs2, + ulong nbs3, + ulong neb0, + ulong neb1, + ulong nbb0, + ulong nbb1, + ulong nee0, + ulong nbe0, + ulong nbe1, + ulong nbe2, + ulong nbe3, + ulong ned1, + ulong ned2, + ulong nbd1, + ulong nbd2, + ulong nbd3, + int fill +) { + const ulong row = get_group_id(1); + const ulong i1 = row % ned1; + const ulong i2 = (row / ned1) % ned2; + const ulong i3 = row / (ned1*ned2); + const ulong tid = get_local_id(0); + const ulong nth = get_local_size(0); + const ulong n_pick = blocks ? neb1 : score ? nes0 : nem0; + + global const MASK_T * mask_row = (global const MASK_T *) (mask + offsetm + + (i1 % nem1)*nbm1 + (i2 % nem2)*nbm2 + (i3 % nem3)*nbm3); + global MASK_T * dst_row = (global MASK_T *) (dst + offsetd + i1*nbd1 + i2*nbd2 + i3*nbd3); + global const char * top_k_row = top_k + offsetk + i1*nbk1 + i2*nbk2 + i3*nbk3; + + if (fill) { + for (ulong c = tid; c < nem0/MASK_NV; c += nth) { + MASK_STORE((MASK_V) (MASK_NINF), c, dst_row); + } + for (ulong c = nem0/MASK_NV*MASK_NV + tid; c < nem0; c += nth) { + dst_row[c] = MASK_NINF; + } + barrier(CLK_GLOBAL_MEM_FENCE); + } + + // Repeated cells copy the same mask value, as in the CUDA and Metal kernels. + for (ulong j = tid; j < nek0 + nee0; j += nth) { + if (j >= nek0) { + const int c = *(global const int *) (extra + offsete + (j - nek0)*nbe0 + i1*nbe1 + i2*nbe2 + i3*nbe3); + if (c >= 0 && (ulong) c < nem0) { + dst_row[c] = mask_row[c]; + } + continue; + } + + const int p = *(global const int *) (top_k_row + j*nbk0); + if (p < 0 || (ulong) p >= n_pick) { + continue; + } + if (score && !(*(global const float *) (score + offsets + p*nbs0 + i1*nbs1 + i2*nbs2 + i3*nbs3) > -INFINITY)) { + continue; + } + if (blocks) { + for (ulong k = 0; k < neb0; ++k) { + const int c = *(global const int *) (blocks + offsetb + k*nbb0 + p*nbb1); + if (c >= 0 && (ulong) c < nem0) { + dst_row[c] = mask_row[c]; + } + } + } else if ((ulong) p < nem0) { + dst_row[p] = mask_row[p]; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl b/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl new file mode 100644 index 000000000..34d54b57e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl @@ -0,0 +1,35 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#define tile_size 32 + +kernel void kernel_transpose_activation_q8( + __global uint * src1_qa, + __global uint * src1_qa_trans, + __global half * src1_sa, + __global float * src1_sa_trans, + __global const int * total_tiles, + uint K +) { + uint row_in_tile = get_global_id(0); + uint blk_k = get_group_id(1); + uint col_in_tile = get_local_id(1); + uint tile_n = get_global_id(2); + + if (blk_k >= (K / tile_size) || tile_n >= (uint)total_tiles[0]) { + return; + } + + uint src_idx = (tile_n * tile_size + row_in_tile) * K / 4 + blk_k * 8 + col_in_tile; + uint dst_idx = tile_n * tile_size * K / 4 + (blk_k * 8 + col_in_tile) * tile_size + row_in_tile; + + src1_qa_trans[dst_idx] = src1_qa[src_idx]; + + if (col_in_tile == 0) { + src_idx = (tile_n * tile_size + row_in_tile) * K / 32 + blk_k; + dst_idx = tile_n * tile_size * K / 32 + blk_k * tile_size + row_in_tile; + + src1_sa_trans[dst_idx] = (float)(src1_sa[src_idx]); + } + +} diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index ae36a0959..10e82f3e6 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -336,7 +336,9 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { } if (op_case == 1 && m_is_stateful) { // Recurrent convolution and GDN gates retain their rank-4 layout. - bool recurrent = src->op == GGML_OP_GET_ROWS && is_recurrent_cache(src->src[0]); + // The gathered states may come through a view of the one gather of all states (see build_rs). + const auto * gather = src->op == GGML_OP_VIEW ? src->src[0] : src; + bool recurrent = gather->op == GGML_OP_GET_ROWS && is_recurrent_cache(gather->src[0]); for (int i = 0; i < m_cgraph->n_nodes && !recurrent; ++i) { const auto * consumer = m_cgraph->nodes[i]; if (consumer->op == GGML_OP_GATED_DELTA_NET) { @@ -411,15 +413,19 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { break; } case GGML_OP_GET_ROWS: { - if (node->src[1]->op == GGML_OP_VIEW) { - // GET_ROWS gathering recurrent state cache rows via the inp->s_copy index list: - // src[0] is a reshape of cache_r/cache_s, src[1] is a view of the s_copy leaf. - // op_case 1/2: active/extra rows of a multi-slot cache - // op_case 3/4: active/extra rows of a single-slot cache - if (node->src[0]->op == GGML_OP_RESHAPE && node->src[0]->src[0] != nullptr && - is_recurrent_cache(node->src[0]->src[0])) { - const bool single_slot = node->src[0]->src[0]->ne[1] == 1; + // GET_ROWS gathering recurrent state cache rows via the inp->s_copy index list: + // src[0] is a reshape of cache_r/cache_s, src[1] is a view of the s_copy leaf, or the leaf itself + // when one gather covers all states (see build_rs). + // op_case 1/2: active/extra rows of a multi-slot cache + // op_case 3/4: active/extra rows of a single-slot cache + if (node->src[0]->op == GGML_OP_RESHAPE && node->src[0]->src[0] != nullptr && + is_recurrent_cache(node->src[0]->src[0])) { + const bool single_slot = node->src[0]->src[0]->ne[1] == 1; + if (node->src[1]->op == GGML_OP_VIEW) { op_case = (node->src[1]->view_offs == 0 ? 1 : 2) + (single_slot ? 2 : 0); + } else if (single_slot) { + // a single-slot cache holds exactly the one state the gather selects + op_case = 3; } } break; @@ -579,6 +585,12 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) { op_case = 4; break; + } else if (node->src[0]->src[0]->op == GGML_OP_GET_ROWS && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr && + is_recurrent_cache(node->src[1]->view_src) && node->src[1]->view_src->ne[1] == 1) { + // defrag remainder writeback of a single-slot cache, taken from a view of the one gather of + // all states (see build_rs) + op_case = 9; } } else if (node->src[0]->op == GGML_OP_GET_ROWS && node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr && @@ -1097,6 +1109,9 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, input_shape = m_is_static ? ov::PartialShape{1, 1, input->ne[1], m_prefill_chunk_size} : ov::PartialShape{1, 1, -1, -1}; + } else if (is_inp_scale_rows(input, op)) { + input_shape = ov::PartialShape{1, 1, m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1, 1}; + } else if (is_inp_mask(input, op)) { // mask if (m_is_static) { @@ -2101,6 +2116,10 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { m_node_dynamic_dims[src] = 0; continue; } + if (is_inp_scale_rows(src, node)) { + m_node_dynamic_dims[src] = 1; + continue; + } if (node->op == GGML_OP_VIEW && src->op == GGML_OP_NONE && !is_stateful() && !m_model_is_splitted) { m_node_dynamic_dims[src] = 1; continue; @@ -2181,8 +2200,11 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { } if (m_node_dynamic_dims[node] != -1 && dynamic_dim_value != node->ne[m_node_dynamic_dims[node]]) { m_node_dynamic_dims[node] = -1; - GGML_LOG_WARN("ggml-openvino: dynamic dim value mismatch for VIEW node '%s', src[0]: '%s'\n", - node->name, node->src[0]->name); + // an empty view, e.g. the extra states of a single-slot recurrent cache, always mismatches + if (ggml_nelements(node) > 0) { + GGML_LOG_WARN("ggml-openvino: dynamic dim value mismatch for VIEW node '%s', src[0]: '%s'\n", + node->name, node->src[0]->name); + } } } break; @@ -2307,6 +2329,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { case GGML_OP_DIAG: case GGML_OP_TRI: case GGML_OP_REPEAT: + case GGML_OP_DUP: // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0]. // DIV/CLAMP are used in the MoE routing-weight normalization // (sum_rows -> clamp -> div). If they are left untracked here the dynamic diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 33b95340a..a614d291a 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -393,6 +393,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE; } + // per-token embedding scale [1, n_tokens] (inp->scale_rows in llama-graph.cpp) + static bool is_inp_scale_rows(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_MUL && tensor == op->src[1] && strcmp(tensor->name, "inp_scale_rows") == 0; + } + static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[2]; } diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 0257e23db..14a0b0382 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -131,6 +131,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_DISABLE_KV_SLICE", "GGML_OPENVINO_ENABLE_FALLBACK", "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN", "GGML_OPENVINO_MOE_OP", "GGML_OPENVINO_MEMORY_OPTIMIZE", "GGML_OPENVINO_RELEASE_WEIGHTS", diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 87ee096fa..3aff66724 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -638,6 +638,9 @@ static const ggml_backend_buffer_i ggml_backend_openvino_buffer_interface = { /* .cpy_tensor = */ ggml_backend_openvino_buffer_cpy_tensor, /* .clear = */ ggml_backend_openvino_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; // Buffer type interface functions @@ -1369,6 +1372,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { if (op->type == GGML_TYPE_I64) { return {false, "CONCAT with I64 type is not supported"}; } + // quantized inputs are dequantized, so the output cannot be written in the quantized type + if (ggml_is_quantized(op->type)) { + return {false, "CONCAT with quantized type is not supported"}; + } if (ggml_openvino_is_gpu() && op->type == GGML_TYPE_BF16 && has_view_op_input(op)) { return {false, "CONCAT with BF16 type and VIEW input is not supported on GPU"}; } @@ -1442,6 +1449,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { if (op->type == GGML_TYPE_BF16) { return {false, "SCALE with BF16 type is not supported"}; } + // The result is copied back densely, so an in-place SCALE on a strided view would be written wrongly. + if (op->view_src != nullptr && !ggml_is_contiguous(op)) { + return {false, "SCALE in place on a non-contiguous view is not supported"}; + } break; } case GGML_OP_ADD_ID: { @@ -1509,6 +1520,11 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { return {false, "FLASH_ATTN_EXT gemma3n pattern on GPU is not supported"}; } + // K/V row lists (ggml_flash_attn_ext_rows) are not supported + if (op->src[5] != nullptr) { + return {false, "FLASH_ATTN_EXT with K/V row lists is not supported"}; + } + if (op->src[4] != nullptr) { return {false, "FLASH_ATTN_EXT with sinks is not supported"}; } @@ -1529,6 +1545,13 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { } break; } + case GGML_OP_DUP: { + // translated as CONT, so only a plain copy + if (op->type != op->src[0]->type || !ggml_are_same_shape(op, op->src[0]) || !ggml_is_contiguous(op->src[0])) { + return {false, "DUP with type conversion or non-contiguous src is not supported"}; + } + break; + } case GGML_OP_CPY: { if (op->src[0]->type != GGML_TYPE_BF16 && op->src[1]->type == GGML_TYPE_BF16) { return {false, "CPY with BF16 src[1] type is not supported"}; @@ -1558,6 +1581,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { break; } case GGML_OP_MUL_MAT: { + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return {false, "MUL_MAT with a BF16 or F32 src1 precision is not supported"}; + } if (ggml_openvino_is_gpu() && op->src[0] != nullptr && op->src[1] != nullptr && ggml_is_quantized(op->src[0]->type) && strcmp(op->src[0]->name, "a") == 0 && strcmp(op->src[1]->name, "b") == 0 && op->src[0]->ne[1] == 1 && op->src[1]->ne[1] == 64 && @@ -1568,6 +1594,14 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { (op->src[0]->buffer == nullptr || op->src[0]->buffer->usage != GGML_BACKEND_BUFFER_USAGE_WEIGHTS)) { return {false, "MUL_MAT scalar dot product with non-weight src[0] on GPU is not supported"}; } + // The GPU plugin fails to compile u4 weights with an f16 zero point for some row counts + // (clFinish CL_OUT_OF_RESOURCES). Op tests build Q4_1/Q4_K weights in that form; model weights use a + // u4 zero point. Op tests check support before allocating, while model loading checks with a dummy + // buffer, so only unbound weights are excluded. Remove once the GPU plugin is fixed. + if (ggml_openvino_is_gpu() && (op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q4_K) && + op->src[0]->buffer == nullptr) { + return {false, "MUL_MAT with unbound Q4_1/Q4_K src[0] on GPU is not supported"}; + } if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) + ", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])}; diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp index 79bdbe877..04b1d3ff2 100644 --- a/ggml/src/ggml-openvino/openvino/op/add_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/add_id.cpp @@ -9,8 +9,12 @@ #include <openvino/op/constant.hpp> #include <openvino/op/convert.hpp> #include <openvino/op/gather.hpp> +#include <openvino/op/greater_eq.hpp> +#include <openvino/op/maximum.hpp> +#include <openvino/op/multiply.hpp> #include <openvino/op/reshape.hpp> #include <openvino/op/shape_of.hpp> +#include <openvino/op/unsqueeze.hpp> namespace ov { namespace frontend { @@ -52,8 +56,16 @@ OutputVector translate_add_id(const NodeContext & context) { ids = std::make_shared<ov::op::v0::Convert>(ids, ov::element::i32); } + // an id of -1 marks a skipped slot: gather row 0 for it and zero the bias with a mask + auto zero = ov::op::v0::Constant::create(ids.get_element_type(), ov::Shape{}, {0}); + ov::Output<ov::Node> keep = std::make_shared<ov::op::v1::GreaterEqual>(ids, zero); + keep = std::make_shared<ov::op::v0::Convert>(keep, bias.get_element_type()); + keep = std::make_shared<ov::op::v0::Unsqueeze>(keep, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1})); + ids = std::make_shared<ov::op::v1::Maximum>(ids, zero); + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); ov::Output<ov::Node> selected_bias = std::make_shared<ov::op::v8::Gather>(bias, ids, gather_axis); + selected_bias = std::make_shared<ov::op::v1::Multiply>(selected_bias, keep); selected_bias = std::make_shared<ov::op::v1::Reshape>( selected_bias, std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64), false); diff --git a/ggml/src/ggml-openvino/openvino/op/fill.cpp b/ggml/src/ggml-openvino/openvino/op/fill.cpp index db2fecb53..87358e12e 100644 --- a/ggml/src/ggml-openvino/openvino/op/fill.cpp +++ b/ggml/src/ggml-openvino/openvino/op/fill.cpp @@ -20,7 +20,7 @@ OutputVector translate_fill(const NodeContext & context) { auto shape = context.get_input_shape(0).to_shape(); - auto val = ov::op::v0::Constant::create(ov::element::f32, {}, {c}); + auto val = ov::op::v0::Constant::create(context.get_output_type(), {}, {c}); auto target_shape = ov::op::v0::Constant::create(ov::element::i64, {shape.size()}, std::vector<int64_t>(shape.begin(), shape.end())); auto res = std::make_shared<ov::op::v3::Broadcast>(val, target_shape); diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp index 7eea81d96..2c206a597 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -75,6 +75,15 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) { const float alpha = reinterpret_cast<const float *>(params)[2]; const float limit = reinterpret_cast<const float *>(params)[3]; + // Compute in f32: alpha_const/one are f32, and Swish/Add need matching input types. + auto output_type = context.get_output_type(); + if (src0.get_element_type() != ov::element::f32) { + src0 = std::make_shared<ov::op::v0::Convert>(src0, ov::element::f32); + } + if (src1.get_element_type() != ov::element::f32) { + src1 = std::make_shared<ov::op::v0::Convert>(src1, ov::element::f32); + } + auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit); auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha}); auto out_glu = std::make_shared<ov::op::v4::Swish>(gate, alpha_const); @@ -82,7 +91,10 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) { auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit); auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f}); auto up_plus_one = std::make_shared<ov::op::v1::Add>(up, one); - auto res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one); + ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one); + if (res.get_element_type() != output_type) { + res = std::make_shared<ov::op::v0::Convert>(res, output_type); + } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/scale.cpp b/ggml/src/ggml-openvino/openvino/op/scale.cpp index 2947ee03c..4be2e700c 100644 --- a/ggml/src/ggml-openvino/openvino/op/scale.cpp +++ b/ggml/src/ggml-openvino/openvino/op/scale.cpp @@ -70,7 +70,7 @@ OutputVector translate_scale(const NodeContext & context) { return rename_outputs_with_suffix({cleared_cache_rs}, context.get_name()); } - auto scaled = std::make_shared<ov::op::v1::Multiply>(context.get_input(0), scale_node); + auto scaled = std::make_shared<ov::op::v1::Multiply>(process_view_input_new(context, 0), scale_node); std::shared_ptr<ov::Node> res; if (bias != 0.0f) { diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index 12a0953e3..857fbf175 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -33,10 +33,10 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() { using namespace ov::op; return { {"GGML_OP_ADD", op::translate_add }, - {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs<v1::Add> }, {"GGML_OP_ADD_ID", op::translate_add_id }, {"GGML_OP_CONCAT", op::translate_concat }, {"GGML_OP_CONT", op::translate_cont }, + {"GGML_OP_DUP", op::translate_cont }, {"GGML_OP_DIV", op::translate_div }, {"GGML_OP_FILL", op::translate_fill }, {"GGML_OP_GET_ROWS", op::translate_get_rows }, diff --git a/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.cpp b/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.cpp new file mode 100644 index 000000000..eae5b2c4e --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.cpp @@ -0,0 +1,96 @@ +#include "align_eltwise_ranks.h" + +#include <numeric> +#include <openvino/op/add.hpp> +#include <openvino/op/constant.hpp> +#include <openvino/op/divide.hpp> +#include <openvino/op/multiply.hpp> +#include <openvino/op/sqrt.hpp> +#include <openvino/op/subtract.hpp> +#include <openvino/op/unsqueeze.hpp> +#include <openvino/pass/pattern/op/wrap_type.hpp> + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +// True for an RMS-norm output, x * (1 / sqrt(mean(x^2) + eps)) as translate_rms_norm builds it, optionally +// scaled by the norm weight. +bool is_rms_norm_output(const ov::Output<ov::Node> & value, int depth = 1) { + const auto * node = value.get_node(); + if (!ov::is_type<ov::op::v1::Multiply>(node)) { + return false; + } + for (const auto & input : node->input_values()) { + const auto * src = input.get_node(); + if (ov::is_type<ov::op::v1::Divide>(src) && ov::is_type<ov::op::v0::Sqrt>(src->get_input_node_ptr(1))) { + return true; + } + if (depth > 0 && is_rms_norm_output(input, depth - 1)) { + return true; + } + } + return false; +} + +} // namespace + +AlignEltwiseOperandRanks::AlignEltwiseOperandRanks() { + auto eltwise_m = ov::pass::pattern::wrap_type<ov::op::v1::Add, ov::op::v1::Multiply, ov::op::v1::Subtract>(); + + const auto callback = [this](ov::pass::pattern::Matcher & m) { + auto node = m.get_match_root(); + if (node->get_input_size() != 2) { + return false; + } + + auto lhs = node->input_value(0); + auto rhs = node->input_value(1); + + // A tensor-vs-Constant mismatch is the norm's own eps / 1-over-sqrt arithmetic. That + // folds into the `rms` primitive itself instead of becoming a fused eltwise post-op, + // so it is not affected and is left alone. + if (ov::is_type<ov::op::v0::Constant>(lhs.get_node()) || + ov::is_type<ov::op::v0::Constant>(rhs.get_node())) { + return false; + } + + const auto lhs_rank = lhs.get_partial_shape().rank(); + const auto rhs_rank = rhs.get_partial_shape().rank(); + if (lhs_rank.is_dynamic() || rhs_rank.is_dynamic() || lhs_rank == rhs_rank) { + return false; + } + + const size_t shorter_idx = lhs_rank.get_length() < rhs_rank.get_length() ? 0 : 1; + const auto & shorter = shorter_idx == 0 ? lhs : rhs; + + // The defect is with the norm output as the higher-rank operand. When the norm output is the + // lower-rank one (gemma-3 adds it to a rank-4 residual), unsqueezing it puts the Unsqueeze between + // `rms` and its post-op, and the GPU plugin then computes the layer wrongly. + if (is_rms_norm_output(shorter)) { + return false; + } + const int64_t diff = std::abs(lhs_rank.get_length() - rhs_rank.get_length()); + + std::vector<int64_t> axes(static_cast<size_t>(diff)); + std::iota(axes.begin(), axes.end(), 0); + auto unsqueeze = std::make_shared<ov::op::v0::Unsqueeze>( + shorter, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ axes.size() }, axes)); + + node->input(shorter_idx).replace_source_output(unsqueeze->output(0)); + register_new_node(unsqueeze); + return true; + }; + + register_matcher( + std::make_shared<ov::pass::pattern::Matcher>(eltwise_m, "ov::frontend::ggml::pass::AlignEltwiseOperandRanks"), + callback); +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.h b/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.h new file mode 100644 index 000000000..04f59cce6 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.h @@ -0,0 +1,39 @@ +#pragma once + +#include <openvino/pass/matcher_pass.hpp> + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +// Give a binary eltwise op's two operands the same rank, by unsqueezing leading axes onto the +// shorter one. +// +// This is semantically a no-op -- NUMPY broadcasting already left-pads the lower-rank operand +// with 1s, and the result rank is max(rank_a, rank_b) either way. It exists purely to work +// around an OpenVINO GPU-plugin defect: an eltwise op whose operands differ in rank is computed +// incorrectly once the plugin fuses it as a post-op into an `rms` primitive. gemma-4 dense hits +// this under stateful execution, where the layer tail adds a rank-3 residual to the rank-4 +// RMS-norm output; the model then degenerates into repeated tokens on GPU while CPU is correct. +// Equalising the ranks keeps the fusion and makes it compute the right answer. +// +// Deliberately a graph pass rather than something the op translators do: rank is load-bearing +// during translation (several translators and later passes read operand ranks), and rewriting +// operands mid-translate breaks the attention path. Running after the graph is complete avoids +// that entirely. +// +// Only applies when both operands are real tensors -- a tensor-vs-Constant mismatch is the +// RMS norm's own eps/rsqrt arithmetic, which folds into the `rms` primitive rather than +// becoming a fused post-op, and is not affected by the defect. The norm output itself is never +// unsqueezed: when it is the lower-rank operand (gemma-3), that breaks the fused path instead. +class AlignEltwiseOperandRanks : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::AlignEltwiseOperandRanks") + AlignEltwiseOperandRanks(); +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp index db8fc6133..3041a6ca7 100644 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp @@ -321,22 +321,39 @@ FuseMoeCompressedFusedGateUp::FuseMoeCompressedFusedGateUp() { auto gate_up_w_m = any_input(); auto ids_gate_up_m = any_input(); auto bgm_fused_m = wrap_type<ov::op::internal::GatherMatmul>({ a_m, gate_up_w_m, ids_gate_up_m, any_input() }); - auto gu_u_m = optional<ov::op::v0::Convert>({ wrap_type<ov::op::v0::Unsqueeze>( - { wrap_type<ov::op::v1::Transpose>({ bgm_fused_m, any_input() }), any_input() }) }); + // Two graph shapes reach here. The rank-4 (stateless) graph restores the batch dim with an + // Unsqueeze after the Transpose and may Convert afterwards: + // GatherMatmul -> Transpose -> Unsqueeze -> [Convert] -> Slice + // The rank-3 (stateful) graph never drops to a batch dim at all, so there is no Unsqueeze + // here - it appears later, between the GEGLU and the down Reshape - and the Convert sits on + // the other side of the Transpose: + // GatherMatmul -> Convert -> Transpose -> Slice + // optional<T>({a, b}) matches T(a, b) or bare a, so one pattern covers both. + auto gu_t_m = + wrap_type<ov::op::v1::Transpose>({ optional<ov::op::v0::Convert>({ bgm_fused_m }), any_input() }); + auto gu_u_m = + optional<ov::op::v0::Convert>({ optional<ov::op::v0::Unsqueeze>({ gu_t_m, any_input() }) }); auto gate_slice_m = wrap_type<ov::op::v8::Slice>({ gu_u_m, any_input(), any_input(), any_input(), any_input() }); auto up_slice_m = wrap_type<ov::op::v8::Slice>({ gu_u_m, any_input(), any_input(), any_input(), any_input() }); auto gelu_m = wrap_type<ov::op::v7::Gelu>({ gate_slice_m }); auto geglu_m = wrap_type<ov::op::v1::Multiply>({ gelu_m, up_slice_m }); + // The rank-3 graph inserts the Unsqueeze the gate/up branch lacked right here, before the + // Reshape that feeds the down projection; the rank-4 graph goes straight from the GEGLU + // into the Reshape. auto d_t_m = wrap_type<ov::op::v1::Transpose>( - { optional<ov::op::v0::Convert>({ wrap_type<ov::op::v1::Reshape>({ geglu_m, any_input() }) }), + { optional<ov::op::v0::Convert>({ wrap_type<ov::op::v1::Reshape>( + { optional<ov::op::v0::Unsqueeze>({ geglu_m, any_input() }), any_input() }) }), any_input() }); auto down_w_m = any_input(); auto ids_down_m = any_input(); auto bgm_down_m = wrap_type<ov::op::internal::GatherMatmul>({ d_t_m, down_w_m, ids_down_m, any_input() }); - auto down_u_m = optional<ov::op::v0::Convert>({ wrap_type<ov::op::v0::Unsqueeze>( - { wrap_type<ov::op::v1::Transpose>({ bgm_down_m, any_input() }), any_input() }) }); + // Same two shapes as the gate/up branch above. + auto down_t_m = + wrap_type<ov::op::v1::Transpose>({ optional<ov::op::v0::Convert>({ bgm_down_m }), any_input() }); + auto down_u_m = + optional<ov::op::v0::Convert>({ optional<ov::op::v0::Unsqueeze>({ down_t_m, any_input() }) }); // gemma-4 applies an extra per-expert output scale to the down projection before the // router-weight multiply (llama-graph.cpp's ffn_down_exps.scale); FuseMoeCompressed's @@ -421,13 +438,20 @@ FuseMoeCompressedFusedGateUp::FuseMoeCompressedFusedGateUp() { } const size_t top_k = ids_pshape[ids_pshape.rank().get_length() - 1].get_length(); - // routing weights arrive as [1, n_tokens, top_k, 1]; the op wants [..., top_k] - auto routing = pm.at(routing_m); - const auto routing_pshape = routing.get_partial_shape(); - if (routing_pshape.rank().is_dynamic() || routing_pshape.rank().get_length() != 4 || - routing_pshape[3] != 1) { + // Routing weights arrive as [1, n_tokens, top_k, 1] on the rank-4 (stateless) graph and + // as [n_tokens, top_k, 1] on the rank-3 (stateful) one, which is the same thing without + // the leading batch dim. Normalise the rank-3 form up to the rank-4 one so everything + // below - and the op's own config - stays in the shape that is already validated on the + // stateless path; the batch dim is taken back off the result at the end. + const auto routing_in = pm.at(routing_m); + const auto routing_pshape = routing_in.get_partial_shape(); + const auto routing_rank = routing_pshape.rank(); + if (routing_rank.is_dynamic() || (routing_rank.get_length() != 4 && routing_rank.get_length() != 3) || + routing_pshape[routing_rank.get_length() - 1] != 1) { return false; } + const bool batchless = routing_rank.get_length() == 3; + // Fold gemma-4's per-expert output scale into the routing weights: the reduction is // sum_e(routing[e] * scale[e] * down_out[e]), and MOECompressed only takes one // per-expert weight, so pre-multiply it into routing here (same [.., top_k, 1] shape). @@ -435,7 +459,11 @@ FuseMoeCompressedFusedGateUp::FuseMoeCompressedFusedGateUp() { if (down_scale.get_partial_shape() != routing_pshape) { return false; } - routing = std::make_shared<ov::op::v1::Multiply>(routing, down_scale); + ov::Output<ov::Node> routing = std::make_shared<ov::op::v1::Multiply>(routing_in, down_scale); + if (batchless) { + routing = std::make_shared<ov::op::v0::Unsqueeze>( + routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); + } routing = std::make_shared<ov::op::v0::Squeeze>( routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 3 })); if (ids_pshape.rank().get_length() == 2) { @@ -494,10 +522,19 @@ FuseMoeCompressedFusedGateUp::FuseMoeCompressedFusedGateUp() { auto moe = std::make_shared<ov::op::internal::MOECompressed>(args, config); ov::Output<ov::Node> result = moe->output(0); + // The op was fed the batched form, so it produces [1, n_tokens, hidden]. On the rank-3 + // graph the ReduceSum being replaced is [n_tokens, hidden], so drop the batch dim again. + if (batchless) { + result = std::make_shared<ov::op::v0::Squeeze>( + result, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); + } const auto root_type = m.get_match_root()->get_output_element_type(0); if (result.get_element_type() != root_type) { result = std::make_shared<ov::op::v0::Convert>(result, root_type); } + if (result.get_partial_shape() != m.get_match_root()->get_output_partial_shape(0)) { + return false; + } result.get_node_shared_ptr()->set_friendly_name(m.get_match_root()->get_friendly_name()); ov::copy_runtime_info(m.get_matched_nodes(), result.get_node_shared_ptr()); diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index 5a3d11f2d..ec3596de2 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -7,6 +7,7 @@ #include "input_model.h" #include "pass/fuse_argsort_topk.h" #include "pass/fuse_moe_router.h" +#include "pass/align_eltwise_ranks.h" #include "pass/fuse_moe_compressed.h" #include "pass/fuse_to_conv.h" #include "pass/kv_state_seq_axis.h" @@ -498,6 +499,17 @@ std::shared_ptr<Model> TranslateSession::apply_transformations(std::shared_ptr<M manager.register_pass<pass::FuseMoeCompressedFusedGateUp>(); } + // Workaround for an OpenVINO GPU-plugin defect: an eltwise op whose operands differ in + // rank is computed wrongly once the plugin fuses it as a post-op into `rms`. gemma-4 + // dense under stateful execution adds a rank-3 residual to the rank-4 norm output and + // decodes garbage on GPU while CPU is correct. Equalising the ranks is a no-op for + // NUMPY broadcasting and makes the fused path correct. + // Remove once the plugin guards that fusion. Opt out with + // GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN=1. + if (ggml_openvino_is_gpu() && !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN")) { + manager.register_pass<pass::AlignEltwiseOperandRanks>(); + } + if (ggml_model_decoder->is_stateful()) { const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names(); const auto kv_param_res_pairs = get_kv_param_res_pairs(model, kv_param_res_names); diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 40ba938a6..e47f97695 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -1,5 +1,7 @@ #include "utils.h" +#include "ggml-impl.h" + #include <cmath> #include <cstddef> #include <ctime> @@ -90,24 +92,6 @@ ov::Output<ov::Node> rope_yarn_ramp_mix(int num_elements, const float corr_dims[ return ramp_mix; } -float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) { -#ifndef M_PI -# define M_PI 3.14159265358979323846 -#endif - return n_dims * logf(n_ctx_orig / (n_rot * 2 * (float) M_PI)) / (2 * logf(base)); -} - -void ggml_rope_yarn_corr_dims(int n_dims, - int n_ctx_orig, - float freq_base, - float beta_fast, - float beta_slow, - float dims[2]) { - float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base)); - float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base)); - dims[0] = std::max(0.0f, start); - dims[1] = std::min(static_cast<float>(n_dims - 1), end); -} } // namespace std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params, @@ -154,6 +138,7 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params float beta_slow; const int n_dims = rope_params[1]; const size_t n_dims_half = n_dims >> 1; + const bool truncate = rope_params[3] == 0; const int n_ctx_orig = rope_params[4]; memcpy(&freq_base, rope_params + 5, sizeof(float)); memcpy(&freq_scale, rope_params + 6, sizeof(float)); @@ -248,7 +233,7 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params theta = theta_interp; } else { float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, truncate, corr_dims); auto ramp_mix = rope_yarn_ramp_mix(num_pairs, corr_dims, ext_factor); Output<Node> one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); auto one_minus_ramp = std::make_shared<ov::op::v1::Subtract>(one, ramp_mix); @@ -261,7 +246,7 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params std::vector<float> factor(n_dims_half); Output<Node> freq_factors; float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, truncate, corr_dims); factor[0] = 1.0f; for (size_t i = 1; i < factor.size(); i++) { factor[i] = theta_scale * factor[i - 1]; diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 67a4123cb..f07e180a4 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -676,6 +676,15 @@ ov::Tensor get_ov_input_tensor_static_prefill(const std::shared_ptr<GgmlOvDecode return input_tensor; } + if (GgmlOvDecoder::is_inp_scale_rows(ggml_tensor, op)) { + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, 1}); + auto * dst = input_tensor.data<float>(); + const auto * src = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size; + std::copy(src, src + chunk_valid_size, dst); + std::fill(dst + chunk_valid_size, dst + chunk_size, 1.0f); + return input_tensor; + } + if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) { const size_t n_seqs = ggml_tensor->ne[1]; const size_t src_stride = ggml_tensor->ne[0]; @@ -1726,6 +1735,25 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared } } // namespace +// Nodes on the unselected branches of ggml_build_forward_select() stay in the graph but must not be +// computed. Keep them out of the OV model, or their inputs become parameters with fixed shapes. +static ggml_cgraph * get_compute_graph(ggml_cgraph * cgraph, ov_runtime_context & r_ctx) { + auto is_skipped = [](const ggml_tensor * node) { + return node->op != GGML_OP_NONE && !(node->flags & GGML_TENSOR_FLAG_COMPUTE); + }; + if (std::none_of(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, is_skipped)) { + return cgraph; + } + auto & compute = r_ctx.compute_graphs[cgraph]; + compute.nodes.clear(); + std::copy_if(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, std::back_inserter(compute.nodes), + [&](const ggml_tensor * node) { return !is_skipped(node); }); + compute.graph = *cgraph; + compute.graph.nodes = compute.nodes.data(); + compute.graph.n_nodes = (int) compute.nodes.size(); + return &compute.graph; +} + // Both execution paths use two cache levels: // 1. Reuse this backend's decoder/request via graph_key and compatibility checks. // 2. On a local miss, look up compiled_graph_key in the shared compilation cache, @@ -1744,6 +1772,7 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) GGML_ASSERT(ctx->runtime_context != nullptr); std::shared_ptr<ov_runtime_context> r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context); std::lock_guard<std::mutex> execution_lock(r_ctx->execution_mutex); + cgraph = get_compute_graph(cgraph, *r_ctx); return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); } catch (const ov::Exception & e) { diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 491dbadc2..c34441f5e 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -119,6 +119,12 @@ struct ov_runtime_context { std::unordered_map<graph_key, std::vector<std::string>, graph_key_hash> ov_output_names_cache; size_t stateful_kv_size; std::map<std::string, std::string> kv_state_input_name_map; + // compute-only copies of graphs that carry unselected ggml_build_forward_select() branches + struct compute_graph { + ggml_cgraph graph; + std::vector<ggml_tensor *> nodes; + }; + std::unordered_map<const ggml_cgraph *, compute_graph> compute_graphs; ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0) {} diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 7750a72ce..3586f67ae 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -25,15 +25,12 @@ #define UNUSED GGML_UNUSED -static inline int best_index_int8(int n, const int8_t * val, float x) { - if (x <= val[0]) return 0; - if (x >= val[n-1]) return n-1; - int ml = 0, mu = n-1; - while (mu-ml > 1) { - int mav = (ml+mu)/2; - if (x < val[mav]) mu = mav; else ml = mav; - } - return x - val[mu-1] < val[mu] - x ? mu-1 : mu; +// index of the nearest kvalues_iq4nl value, with mid[k] = (values[k] + values[k+1])/2 +// for this table it matches a binary search with the tie rule x - values[k] < values[k+1] - x for every float x +static inline int best_index_iq4nl(const float * GGML_RESTRICT mid, float x) { + int l = 15; + for (int k = 0; k < 15; ++k) l -= x < mid[k]; + return l; } // reference implementation for deterministic creation of model files @@ -416,6 +413,12 @@ void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RE } } +void quantize_row_f8_e4m3_ref(const float * GGML_RESTRICT x, ggml_fp8_e4m3_t * GGML_RESTRICT y, int64_t k) { + for (int64_t i = 0; i < k; ++i) { + y[i].bits = ggml_fp32_to_f8_e4m3(x[i]); + } +} + void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { static const int qk = QK1_0; @@ -611,6 +614,12 @@ void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_REST } } +void dequantize_row_f8_e4m3(const ggml_fp8_e4m3_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + for (int64_t i = 0; i < k; ++i) { + y[i] = ggml_f8_e4m3_to_fp32(x[i].bits); + } +} + // // 2-6 bit quantization in super-blocks // @@ -2312,6 +2321,12 @@ size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * ggml_row_size(GGML_TYPE_NVFP4, n_per_row); } +size_t quantize_f8_e4m3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + GGML_UNUSED(quant_weights); + quantize_row_f8_e4m3_ref(src, dst, (int64_t)nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_F8_E4M3, n_per_row); +} + // ====================== Ternary (de)-quantization (BitNet b1.58 and TriLMs) void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k) { @@ -4949,6 +4964,11 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block memset(q4, 0, super_block_size/2); dh[0] = GGML_FP32_TO_FP16(0.f); + float mid[15]; + float fval[16]; + for (int k = 0; k < 15; ++k) mid[k] = 0.5f*(values[k] + values[k+1]); + for (int k = 0; k < 16; ++k) fval[k] = values[k]; + float max_scale = 0, amax_scale = 0; for (int ib = 0; ib < super_block_size/block_size; ++ib) { const float * xb = x + ib*block_size; @@ -4971,32 +4991,28 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block continue; } float d = ntry > 0 ? -max/values[0] : max/values[0]; - float id = 1/d; - float sumqx = 0, sumq2 = 0; + // lane 0 is the initial guess, lanes 1..nt-1 are itry = -ntry..ntry; each lane sums over j in order + const int nt = ntry >= 0 ? 2*ntry + 2 : 1; + float idt[16], sqx[16], sq2[16]; + idt[0] = 1/d; + for (int t = 1; t < nt; ++t) idt[t] = (t - 1 - ntry + values[0])/max; + for (int t = 0; t < nt; ++t) sqx[t] = sq2[t] = 0; for (int j = 0; j < block_size; ++j) { - float al = id*xb[j]; - int l = best_index_int8(16, values, al); - Lb[j] = l; - float q = values[l]; - float w = weight[j]; - sumqx += w*q*xb[j]; - sumq2 += w*q*q; + Lb[j] = best_index_iq4nl(mid, idt[0]*xb[j]); } - d = sumq2 > 0 ? sumqx/sumq2 : 0.f; - float best = d*sumqx; - for (int itry = -ntry; itry <= ntry; ++itry) { - id = (itry + values[0])/max; - sumqx = sumq2 = 0; - for (int j = 0; j < block_size; ++j) { - float al = id*xb[j]; - int l = best_index_int8(16, values, al); - float q = values[l]; - float w = weight[j]; - sumqx += w*q*xb[j]; - sumq2 += w*q*q; + for (int j = 0; j < block_size; ++j) { + float w = weight[j]; + for (int t = 0; t < nt; ++t) { + float q = fval[best_index_iq4nl(mid, idt[t]*xb[j])]; + sqx[t] += w*q*xb[j]; + sq2[t] += w*q*q; } - if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { - d = sumqx/sumq2; best = d * sumqx; + } + d = sq2[0] > 0 ? sqx[0]/sq2[0] : 0.f; + float best = d*sqx[0]; + for (int t = 1; t < nt; ++t) { + if (sq2[t] > 0 && sqx[t]*sqx[t] > best*sq2[t]) { + d = sqx[t]/sq2[t]; best = d * sqx[t]; } } scales[ib] = d; @@ -5020,7 +5036,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block uint8_t * Lb = L + ib*block_size; const float * xb = x + ib*block_size; for (int j = 0; j < block_size; ++j) { - Lb[j] = best_index_int8(16, values, idl*xb[j]); + Lb[j] = best_index_iq4nl(mid, idl*xb[j]); } l += 32; uint8_t l_l = l & 0xf; @@ -5034,7 +5050,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block if (ntry > 0) { float id = scales[0] ? 1/scales[0] : 0; for (int j = 0; j < super_block_size; ++j) { - L[j] = best_index_int8(16, values, id*x[j]); + L[j] = best_index_iq4nl(mid, id*x[j]); } } } @@ -5535,10 +5551,20 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte } break; case GGML_TYPE_NVFP4: { - // UE4M3 scales are uint8_t — all byte values are valid + // TODO: validate each unsigned E4M3 scale; NaN encodings and scales with the sign bit set are invalid. GGML_UNUSED(data); GGML_UNUSED(nb); } break; + case GGML_TYPE_F8_E4M3: + { + const ggml_fp8_e4m3_t * q = (const ggml_fp8_e4m3_t *) data; + for (size_t i = 0; i < nb; ++i) { + if ((q[i].bits & 0x7F) == 0x7F) { + fprintf(stderr, "%s: found NaN at index %zu in row of %zu F8_E4M3 values\n", __func__, i, nb); + return false; + } + } + } break; case GGML_TYPE_Q2_K: { VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_K, data, nb, d, dmin); diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index 75188f1af..ca36d1369 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -25,6 +25,7 @@ GGML_API void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_f8_e4m3_ref(const float * GGML_RESTRICT x, ggml_fp8_e4m3_t * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k); @@ -54,6 +55,7 @@ GGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GG GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_f8_e4m3(const ggml_fp8_e4m3_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -104,6 +106,7 @@ GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTR GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_f8_e4m3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API void iq2xs_init_impl(enum ggml_type type); GGML_API void iq2xs_free_impl(enum ggml_type type); diff --git a/ggml/src/ggml-rpc/CMakeLists.txt b/ggml/src/ggml-rpc/CMakeLists.txt index e3d0c9b4c..897665123 100644 --- a/ggml/src/ggml-rpc/CMakeLists.txt +++ b/ggml/src/ggml-rpc/CMakeLists.txt @@ -2,6 +2,7 @@ message(STATUS "Using RPC backend") ggml_add_backend_library(ggml-rpc ggml-rpc.cpp + log.cpp transport.cpp ) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 158a15bb8..e4a84cb0c 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -3,11 +3,14 @@ #include "ggml-backend-impl.h" #include "ggml-cpp.h" #include "transport.h" +#include "log.h" #include <array> +#include <chrono> #include <cinttypes> #include <optional> #include <string> +#include <thread> #include <vector> #include <queue> #include <condition_variable> @@ -21,13 +24,6 @@ #include <filesystem> #include <algorithm> #include <atomic> -#include <thread> - -static const char * RPC_DEBUG = std::getenv("GGML_RPC_DEBUG"); - -#define LOG_DBG(...) \ - do { if (RPC_DEBUG) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) - namespace fs = std::filesystem; @@ -77,24 +73,68 @@ enum rpc_cmd { RPC_CMD_DEVICE_COUNT, RPC_CMD_GRAPH_RECOMPUTE, RPC_CMD_MEMSET_TENSOR, + RPC_CMD_SET_TENSOR_2D, + RPC_CMD_GET_TENSOR_2D, + RPC_CMD_COMM_INIT, + RPC_CMD_COMM_ALLREDUCE, + RPC_CMD_COMM_FREE, RPC_CMD_NONE, RPC_CMD_COUNT, }; static_assert(RPC_CMD_HELLO == 14, "RPC_CMD_HELLO must be always 14"); +static const char * rpc_cmd_name(enum rpc_cmd cmd) { + switch (cmd) { + case RPC_CMD_ALLOC_BUFFER: return "ALLOC_BUFFER"; + case RPC_CMD_GET_ALIGNMENT: return "GET_ALIGNMENT"; + case RPC_CMD_GET_MAX_SIZE: return "GET_MAX_SIZE"; + case RPC_CMD_BUFFER_GET_BASE: return "BUFFER_GET_BASE"; + case RPC_CMD_FREE_BUFFER: return "FREE_BUFFER"; + case RPC_CMD_BUFFER_CLEAR: return "BUFFER_CLEAR"; + case RPC_CMD_SET_TENSOR: return "SET_TENSOR"; + case RPC_CMD_SET_TENSOR_HASH: return "SET_TENSOR_HASH"; + case RPC_CMD_GET_TENSOR: return "GET_TENSOR"; + case RPC_CMD_COPY_TENSOR: return "COPY_TENSOR"; + case RPC_CMD_GRAPH_COMPUTE: return "GRAPH_COMPUTE"; + case RPC_CMD_GET_DEVICE_MEMORY: return "GET_DEVICE_MEMORY"; + case RPC_CMD_INIT_TENSOR: return "INIT_TENSOR"; + case RPC_CMD_GET_ALLOC_SIZE: return "GET_ALLOC_SIZE"; + case RPC_CMD_HELLO: return "HELLO"; + case RPC_CMD_DEVICE_COUNT: return "DEVICE_COUNT"; + case RPC_CMD_GRAPH_RECOMPUTE: return "GRAPH_RECOMPUTE"; + case RPC_CMD_MEMSET_TENSOR: return "MEMSET_TENSOR"; + case RPC_CMD_SET_TENSOR_2D: return "SET_TENSOR_2D"; + case RPC_CMD_GET_TENSOR_2D: return "GET_TENSOR_2D"; + case RPC_CMD_COMM_INIT: return "COMM_INIT"; + case RPC_CMD_COMM_ALLREDUCE: return "COMM_ALLREDUCE"; + case RPC_CMD_COMM_FREE: return "COMM_FREE"; + case RPC_CMD_NONE: return "NONE"; + default: return "UNKNOWN"; + } +} + // Try RPC_CMD_SET_TENSOR_HASH first when data size is larger than this threshold const size_t HASH_THRESHOLD = 10 * 1024 * 1024; +// Maximum number of graphs cached per device; client and server must use the same value +// so that both sides clear their caches at the same point in the message stream +const size_t GRAPH_CACHE_MAX = 1024; + struct rpc_msg_hello_req { uint8_t conn_caps[RPC_CONN_CAPS_SIZE]; }; +// server feature flags in rpc_msg_hello_rsp::flags +enum rpc_dispatcher_flags : uint8_t { + RPC_DISPATCHER_FLAGS_SEND_TENSOR_HASH = 1 << 0, // client should try SET_TENSOR_HASH before sending tensor data +}; + struct rpc_msg_hello_rsp { uint8_t major; uint8_t minor; uint8_t patch; - uint8_t padding; + uint8_t flags; uint8_t conn_caps[RPC_CONN_CAPS_SIZE]; }; @@ -202,6 +242,36 @@ struct rpc_msg_get_device_memory_rsp { struct rpc_msg_graph_recompute_req { uint32_t device; + uint64_t uid; +}; + +struct rpc_msg_get_tensor_2d_req { + rpc_tensor tensor; + uint64_t offset; + uint64_t size; + uint64_t n_copies; + uint64_t stride; +}; + +struct rpc_msg_comm_init_req { + uint32_t device; + uint32_t rank; + uint32_t world; + uint32_t port; // rank 0: port to listen on; rank > 0: rank 0's comm port + char host[64]; // rank > 0: rank 0's host +}; + +struct rpc_msg_comm_init_rsp { + uint8_t ok; +}; + +struct rpc_msg_comm_allreduce_req { + uint32_t device; + rpc_tensor tensor; +}; + +struct rpc_msg_comm_free_req { + uint32_t device; }; #pragma pack(pop) @@ -218,7 +288,6 @@ struct ggml_backend_rpc_device_context { uint32_t device; std::string name; std::string description; - uint64_t last_graph_uid; }; struct ggml_backend_rpc_buffer_type_context { @@ -232,6 +301,7 @@ struct ggml_backend_rpc_buffer_type_context { class rpc_dispatcher; struct ggml_backend_rpc_context { std::shared_ptr<rpc_dispatcher> dispatcher; + std::string endpoint; uint32_t device; std::string name; }; @@ -283,8 +353,11 @@ static bool recv_msg(socket_ptr sock, std::vector<uint8_t> & input) { } try { input.resize(size); - } catch (const std::bad_alloc & e) { - GGML_LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size); + } catch (const std::length_error &) { + LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size); + return false; + } catch (const std::bad_alloc &) { + LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size); return false; } return sock->recv_data(input.data(), size); @@ -305,21 +378,30 @@ static bool parse_endpoint(const std::string & endpoint, std::string & host, int } // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) | +// request_data is input followed by an optional payload, so large data does not need to be copied into input // No response -static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) { +static bool send_rpc_cmd_payload(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, const void * payload, size_t payload_size) { uint8_t cmd_byte = cmd; if (!sock->send_data(&cmd_byte, sizeof(cmd_byte))) { return false; } - if (!sock->send_data(&input_size, sizeof(input_size))) { + const size_t request_size = input_size + payload_size; + if (!sock->send_data(&request_size, sizeof(request_size))) { return false; } if (!sock->send_data(input, input_size)) { return false; } + if (payload_size > 0 && !sock->send_data(payload, payload_size)) { + return false; + } return sock->flush(); } +static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) { + return send_rpc_cmd_payload(sock, cmd, input, input_size, nullptr, 0); +} + // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) | // RPC response: | response_size (8 bytes) | response_data (response_size bytes) | static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, void * output, size_t output_size) { @@ -341,25 +423,42 @@ static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, // RPC client-side implementation +// with busy spinning on, the dispatcher still blocks on its queue after this long without commands +static constexpr auto RPC_BUSY_SPIN_IDLE_TIME = std::chrono::milliseconds(100); + +static inline void rpc_cpu_relax() { +#if defined(__aarch64__) && (defined(__clang__) || defined(__GNUC__)) + __asm__ volatile("yield" ::: "memory"); +#else + std::this_thread::yield(); +#endif +} + // Performs HELLO handshake with transport auto-negotiation. // Advertises local capabilities via conn_caps; if the server responds with // matching capabilities, the socket is upgraded transparently. -static bool negotiate_hello(const std::shared_ptr<socket_t> & sock) { +static bool negotiate_hello(const std::shared_ptr<socket_t> & sock, rpc_dispatcher_flags & flags) { rpc_msg_hello_req request = {}; rpc_msg_hello_rsp response = {}; sock->get_caps(request.conn_caps); bool status = send_rpc_cmd(sock, RPC_CMD_HELLO, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + if (!status) { + LOG_ERROR("RPC hello exchange with the server failed\n"); + return false; + } if (response.major != RPC_PROTO_MAJOR_VERSION || response.minor > RPC_PROTO_MINOR_VERSION) { - GGML_LOG_ERROR("RPC server version mismatch: %d.%d.%d\n", + LOG_ERROR("RPC server version mismatch: %d.%d.%d\n", response.major, response.minor, response.patch); return false; } sock->update_caps(response.conn_caps); + flags = (rpc_dispatcher_flags) response.flags; + LOG_DBG("[%s] handshake ok: server version %d.%d.%d, transport: %s\n", + __func__, response.major, response.minor, response.patch, sock->transport_name()); return true; } @@ -389,6 +488,16 @@ class message_queue { return true; } + bool try_pop(T* out) { + std::unique_lock<std::mutex> lock(mutex); + if (interrupted || queue.empty()) { + return false; + } + *out = queue.front(); + queue.pop(); + return true; + } + void interrupt() { std::unique_lock<std::mutex> lock(mutex); interrupted = true; @@ -408,20 +517,23 @@ class rpc_dispatcher { rpc_dispatcher() { } - void send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size); - void send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size); - void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size); - void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size); + void send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, const void * payload, size_t payload_size, void * output, size_t output_size); + void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output = nullptr, size_t output_size = 0); ggml_backend_event_t event_new(ggml_backend_dev_t dev); void event_free(ggml_backend_event_t event); void event_synchronize(ggml_backend_event_t event); void event_record(ggml_backend_event_t event); void synchronize(); + void busy_spin_acquire(); + void busy_spin_release(); + void graph_compute(uint32_t device, const ggml_cgraph * cgraph); - void start(const std::string & endpoint); + bool start(const std::string & endpoint); void work(); + bool get_send_tensor_hash() const { return flags & RPC_DISPATCHER_FLAGS_SEND_TENSOR_HASH; } + ~rpc_dispatcher(); private: @@ -431,16 +543,27 @@ class rpc_dispatcher { size_t input_size; void * output; size_t output_size; + const void * payload = nullptr; + size_t payload_size = 0; std::promise<void> completion; }; using rpc_msg_ptr = std::shared_ptr<rpc_msg>; using rpc_msg_queue = message_queue<rpc_msg_ptr>; + struct rpc_event { rpc_msg_ptr msg; std::shared_future<void> sf; }; - rpc_msg_queue queue; - socket_ptr sock; + + std::mutex graph_mutex; + std::unordered_map<uint32_t, std::unordered_set<uint64_t>> graph_uids; + + rpc_msg_queue queue; + socket_ptr sock; + + rpc_dispatcher_flags flags = {}; + + std::atomic_uint busy_spin_users = 0; std::atomic_bool running; std::thread thread; }; @@ -450,35 +573,15 @@ static void rpc_dispatcher_trampoline(rpc_dispatcher * dispatcher) dispatcher->work(); } -void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size) { +void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, const void * payload, size_t payload_size, void * output, size_t output_size) { auto msg = std::make_shared<rpc_msg>(); msg->cmd = cmd; - msg->input = input; - msg->input_size = input_size; - msg->output = nullptr; - msg->output_size = 0; - GGML_ASSERT(queue.push(msg)); - auto future = msg->completion.get_future(); - future.wait(); -} - -void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size) { - auto msg = std::make_shared<rpc_msg>(); - msg->cmd = cmd; - msg->input = input; - msg->input_size = input_size; - msg->output = nullptr; - msg->output_size = 0; - GGML_ASSERT(queue.push(msg)); -} - -void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size) { - auto msg = std::make_shared<rpc_msg>(); - msg->cmd = cmd; - msg->input = input; + msg->input = std::move(input); msg->input_size = input_size; msg->output = output; msg->output_size = output_size; + msg->payload = payload; + msg->payload_size = payload_size; GGML_ASSERT(queue.push(msg)); auto future = msg->completion.get_future(); future.wait(); @@ -487,7 +590,7 @@ void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr<const void> input, s void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size) { auto msg = std::make_shared<rpc_msg>(); msg->cmd = cmd; - msg->input = input; + msg->input = std::move(input); msg->input_size = input_size; msg->output = output; msg->output_size = output_size; @@ -532,44 +635,78 @@ void rpc_dispatcher::synchronize() { msg->completion.get_future().wait(); } -void rpc_dispatcher::start(const std::string & endpoint) { +void rpc_dispatcher::busy_spin_acquire() { + busy_spin_users.fetch_add(1, std::memory_order_relaxed); +} + +void rpc_dispatcher::busy_spin_release() { + const unsigned previous = busy_spin_users.fetch_sub(1, std::memory_order_relaxed); + GGML_ASSERT(previous > 0); +} + +bool rpc_dispatcher::start(const std::string & endpoint) { std::string host; int port; if (!parse_endpoint(endpoint, host, port)) { - GGML_ABORT("Failed to parse endpoint: %s\n", endpoint.c_str()); + LOG_ERROR("Failed to parse endpoint: %s\n", endpoint.c_str()); + return false; } if (!rpc_transport_init()) { - GGML_ABORT("RPC transport initialization failed\n"); + LOG_ERROR("RPC transport initialization failed\n"); + return false; } sock = socket_t::connect(host.c_str(), port); if (sock == nullptr) { - GGML_ABORT("Failed to connect to %s\n", endpoint.c_str()); + LOG_ERROR("Failed to connect to %s\n", endpoint.c_str()); + return false; } - if (!negotiate_hello(sock)) { - GGML_ABORT("RPC handshake failed for %s\n", endpoint.c_str()); + if (!negotiate_hello(sock, flags)) { + LOG_ERROR("RPC handshake failed for %s\n", endpoint.c_str()); + return false; } LOG_DBG("[%s] connected to %s\n", __func__, endpoint.c_str()); running = true; thread = std::thread(rpc_dispatcher_trampoline, this); + return true; } void rpc_dispatcher::work() { + auto last_cmd = std::chrono::steady_clock::now(); while (running) { rpc_msg_ptr msg_ptr; - if (!queue.pop(&msg_ptr)) { + // spin only while commands keep coming, so an idle dispatcher does not keep a core busy + const bool spin = busy_spin_users.load(std::memory_order_relaxed) != 0 && + std::chrono::steady_clock::now() - last_cmd < RPC_BUSY_SPIN_IDLE_TIME; + if (spin) { + if (!queue.try_pop(&msg_ptr)) { + rpc_cpu_relax(); + continue; + } + } else if (!queue.pop(&msg_ptr)) { break; } if (msg_ptr->cmd != RPC_CMD_NONE) { + LOG_DBG2("[%s] %s (in: %s, out: %s)\n", __func__, rpc_cmd_name(msg_ptr->cmd), + rpc_log_format_size(msg_ptr->input_size).c_str(), rpc_log_format_size(msg_ptr->output_size).c_str()); + const bool trace = rpc_log_debug_level() >= 3; + const auto t0 = std::chrono::steady_clock::now(); if (msg_ptr->output) { bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->output, msg_ptr->output_size); RPC_STATUS_ASSERT(status); } else { - bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size); + bool status = send_rpc_cmd_payload(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->payload, msg_ptr->payload_size); RPC_STATUS_ASSERT(status); } + if (trace) { + const std::chrono::duration<double, std::milli> elapsed = std::chrono::steady_clock::now() - t0; + LOG_DBG3("[%s] %s done in %.3f ms\n", __func__, rpc_cmd_name(msg_ptr->cmd), elapsed.count()); + } + } else { + LOG_DBG3("[%s] sync barrier\n", __func__); } msg_ptr->completion.set_value(); + last_cmd = std::chrono::steady_clock::now(); } } @@ -595,7 +732,9 @@ static std::shared_ptr<rpc_dispatcher> get_dispatcher(const std::string & endpoi } auto dispatcher = std::make_shared<rpc_dispatcher>(); - dispatcher->start(endpoint); + if (!dispatcher->start(endpoint)) { + return nullptr; + } dispatchers[endpoint] = dispatcher; return dispatcher; } @@ -604,7 +743,8 @@ static void ggml_backend_rpc_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; auto request = std::make_shared<rpc_msg_free_buffer_req>(); request->remote_ptr = ctx->remote_ptr; - ctx->dispatcher->send(RPC_CMD_FREE_BUFFER, request, sizeof(*request)); + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request->remote_ptr); + ctx->dispatcher->send(RPC_CMD_FREE_BUFFER, request, sizeof(*request), nullptr, 0, nullptr, 0); delete ctx; } @@ -615,8 +755,9 @@ static void * ggml_backend_rpc_buffer_get_base(ggml_backend_buffer_t buffer) { } auto request = std::make_shared<rpc_msg_buffer_get_base_req>(); request->remote_ptr = ctx->remote_ptr; + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, ctx->remote_ptr); rpc_msg_buffer_get_base_rsp response; - ctx->dispatcher->send(RPC_CMD_BUFFER_GET_BASE, request, sizeof(*request), &response, sizeof(response)); + ctx->dispatcher->send(RPC_CMD_BUFFER_GET_BASE, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); ctx->base_ptr = reinterpret_cast<void *>(response.base_ptr); return ctx->base_ptr; } @@ -625,7 +766,7 @@ static bool ggml_backend_buffer_is_rpc(ggml_backend_buffer_t buffer) { return buffer->iface.free_buffer == ggml_backend_rpc_buffer_free_buffer; } -static rpc_tensor serialize_tensor(const ggml_tensor * tensor, const std::shared_ptr<rpc_dispatcher> & dispatcher = nullptr) { +static rpc_tensor serialize_tensor(const ggml_tensor * tensor, const rpc_dispatcher * dispatcher = nullptr) { rpc_tensor result; if (!tensor) { memset(&result, 0, sizeof(result)); @@ -638,7 +779,7 @@ static rpc_tensor serialize_tensor(const ggml_tensor * tensor, const std::shared ggml_backend_buffer_t buffer = tensor->buffer; ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; // ref: https://github.com/ggml-org/llama.cpp/pull/26500 - if (ctx != nullptr && (dispatcher == nullptr || ctx->dispatcher == dispatcher)) { + if (ctx != nullptr && (dispatcher == nullptr || ctx->dispatcher.get() == dispatcher)) { result.buffer = ctx->remote_ptr; result.data = reinterpret_cast<uint64_t>(tensor->data); } else { @@ -679,9 +820,10 @@ static enum ggml_status ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_ // Due to bandwidth constraints, we only call the server init tensor functions if necessary. // In particular, only quantized tensors need padding if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr)) { + LOG_DBG2("[%s] tensor: %s\n", __func__, rpc_log_format_tensor(tensor).c_str()); auto request = std::make_shared<rpc_msg_init_tensor_req>(); request->tensor = serialize_tensor(tensor); - ctx->dispatcher->send(RPC_CMD_INIT_TENSOR, request, sizeof(*request)); + ctx->dispatcher->send(RPC_CMD_INIT_TENSOR, request, sizeof(*request), nullptr, 0, nullptr, 0); } return GGML_STATUS_SUCCESS; } @@ -694,18 +836,23 @@ static void ggml_backend_rpc_buffer_memset_tensor( request->offset = offset; request->size = size; request->value = value; - ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request)); + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s, value: %u\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str(), value); + ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request), nullptr, 0, nullptr, 0); } // input serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) +// with size == 0 only the header is serialized and data is sent separately as payload static std::shared_ptr<uint8_t> serialize_set_tensor(const rpc_tensor & rpc_tensor, uint8_t cache_flag, uint64_t offset, const void * data, size_t size, size_t & input_size) { input_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset) + size; - uint8_t * input = new uint8_t[input_size](); + uint8_t * input = new uint8_t[input_size]; uint8_t * p = input; memcpy(p, &rpc_tensor, sizeof(rpc_tensor)); p += sizeof(rpc_tensor); memcpy(p, &cache_flag, sizeof(cache_flag)); p += sizeof(cache_flag); memcpy(p, &offset, sizeof(offset)); p += sizeof(offset); - memcpy(p, data, size); + if (size > 0) { + memcpy(p, data, size); + } return std::shared_ptr<uint8_t>(input, std::default_delete<uint8_t[]>()); } @@ -713,21 +860,26 @@ static std::shared_ptr<uint8_t> serialize_set_tensor(const rpc_tensor & rpc_tens // compute-buffer inputs (the activations ggml_backend_sched copies between backends) must not // take this path, otherwise with `rpc-server -c` every ubatch above the threshold is written // to the cache directory and later served from there. -static bool rpc_use_hash_cache(const ggml_tensor * tensor, size_t size) { - return size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; +// hashing is slow (~1 GB/s), so only do it when the server asks for tensor hashes. +static bool rpc_use_hash_cache(const rpc_dispatcher & dispatcher, const ggml_tensor * tensor, size_t size) { + return dispatcher.get_send_tensor_hash() && size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; } static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; rpc_tensor rpc_tensor = serialize_tensor(tensor); uint8_t cache_flag = 0; - if (rpc_use_hash_cache(tensor, size)) { + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str()); + if (rpc_use_hash_cache(*ctx->dispatcher, tensor, size)) { auto request = std::make_shared<rpc_msg_set_tensor_hash_req>(); request->tensor = rpc_tensor; request->offset = offset; request->hash = fnv_hash((const uint8_t*)data, size); rpc_msg_set_tensor_hash_rsp response; - ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response)); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); + LOG_DBG2("[%s] tensor: %s, hash: 0x%" PRIx64 ", cache: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), + request->hash, response.result ? "hit" : "miss"); if (response.result) { // the server has the same data, no need to send it return; @@ -735,18 +887,64 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm // the server has no cache entry for this tensor - ask it to save one cache_flag = 1; } - size_t input_size; - auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size); - ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input, input_size); + size_t header_size; + auto header = serialize_set_tensor(rpc_tensor, cache_flag, offset, nullptr, 0, header_size); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR, header, header_size, data, size, nullptr, 0); +} + +static void ggml_backend_rpc_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + rpc_tensor rpc_tensor = serialize_tensor(tensor); + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s, n_copies: %zu, stride: %zu\n", __func__, + rpc_log_format_tensor(tensor).c_str(), offset, rpc_log_format_size(size).c_str(), n_copies, stride_tensor); + // input serialization format: | rpc_tensor | offset (8 bytes) | size (8 bytes) | n_copies (8 bytes) | stride (8 bytes) | data (size * n_copies bytes) | + size_t input_size = sizeof(rpc_tensor) + 4*sizeof(uint64_t) + size*n_copies; + uint8_t * input = new uint8_t[input_size](); + uint8_t * dest = input; + memcpy(dest, &rpc_tensor, sizeof(rpc_tensor)); + dest += sizeof(rpc_tensor); + uint64_t header[4] = { offset, size, n_copies, stride_tensor }; + memcpy(dest, header, sizeof(header)); + dest += sizeof(header); + for (size_t i = 0; i < n_copies; i++) { + memcpy(dest + i*size, (const char *)data + i*stride_data, size); + } + std::shared_ptr<uint8_t> input_ptr(input, std::default_delete<uint8_t[]>()); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_2D, input_ptr, input_size, nullptr, 0, nullptr, 0); +} + +static void ggml_backend_rpc_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s, n_copies: %zu, stride: %zu\n", __func__, + rpc_log_format_tensor(tensor).c_str(), offset, rpc_log_format_size(size).c_str(), n_copies, stride_tensor); + auto request = std::make_shared<rpc_msg_get_tensor_2d_req>(); + request->tensor = serialize_tensor(tensor); + request->offset = offset; + request->size = size; + request->n_copies = n_copies; + request->stride = stride_tensor; + if (stride_data == size) { + ctx->dispatcher->send(RPC_CMD_GET_TENSOR_2D, request, sizeof(*request), nullptr, 0, data, size*n_copies); + } else { + std::vector<uint8_t> packed(size*n_copies); + ctx->dispatcher->send(RPC_CMD_GET_TENSOR_2D, request, sizeof(*request), nullptr, 0, packed.data(), packed.size()); + for (size_t i = 0; i < n_copies; i++) { + memcpy((char *)data + i*stride_data, packed.data() + i*size, size); + } + } } static void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str()); auto request = std::make_shared<rpc_msg_get_tensor_req>(); request->tensor = serialize_tensor(tensor); request->offset = offset; request->size = size; - ctx->dispatcher->send(RPC_CMD_GET_TENSOR, request, sizeof(*request), data, size); + ctx->dispatcher->send(RPC_CMD_GET_TENSOR, request, sizeof(*request), nullptr, 0, data, size); } static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { @@ -757,14 +955,16 @@ static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con ggml_backend_buffer_t dst_buffer = dst->buffer; ggml_backend_rpc_buffer_context * dst_ctx = (ggml_backend_rpc_buffer_context *)dst_buffer->context; if (src_ctx->dispatcher != dst_ctx->dispatcher) { + LOG_DBG2("[%s] src and dst are on different servers, falling back to a host copy\n", __func__); return false; } ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + LOG_DBG2("[%s] src: %s -> dst: %s\n", __func__, rpc_log_format_tensor(src).c_str(), rpc_log_format_tensor(dst).c_str()); auto request = std::make_shared<rpc_msg_copy_tensor_req>(); request->src = serialize_tensor(src); request->dst = serialize_tensor(dst); rpc_msg_copy_tensor_rsp response; - ctx->dispatcher->send(RPC_CMD_COPY_TENSOR, request, sizeof(*request), &response, sizeof(response)); + ctx->dispatcher->send(RPC_CMD_COPY_TENSOR, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); return response.result; } return false; @@ -775,7 +975,8 @@ static void ggml_backend_rpc_buffer_clear(ggml_backend_buffer_t buffer, uint8_t auto request = std::make_shared<rpc_msg_buffer_clear_req>(); request->remote_ptr = ctx->remote_ptr; request->value = value; - ctx->dispatcher->send(RPC_CMD_BUFFER_CLEAR, request, sizeof(*request)); + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 ", value: %u\n", __func__, request->remote_ptr, request->value); + ctx->dispatcher->send(RPC_CMD_BUFFER_CLEAR, request, sizeof(*request), nullptr, 0, nullptr, 0); } static ggml_backend_buffer_i ggml_backend_rpc_buffer_interface = { @@ -785,11 +986,14 @@ static ggml_backend_buffer_i ggml_backend_rpc_buffer_interface = { /* .memset_tensor = */ ggml_backend_rpc_buffer_memset_tensor, /* .set_tensor = */ ggml_backend_rpc_buffer_set_tensor, /* .get_tensor = */ ggml_backend_rpc_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, + /* .set_tensor_2d = */ ggml_backend_rpc_buffer_set_tensor_2d, + /* .get_tensor_2d = */ ggml_backend_rpc_buffer_get_tensor_2d, /* .cpy_tensor = */ ggml_backend_rpc_buffer_cpy_tensor, /* .clear = */ ggml_backend_rpc_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; static const char * ggml_backend_rpc_buffer_type_name(ggml_backend_buffer_type_t buft) { @@ -805,14 +1009,22 @@ static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer(ggml_back rpc_msg_alloc_buffer_rsp response; auto dispatcher = get_dispatcher(buft_ctx->endpoint); - dispatcher->send(RPC_CMD_ALLOC_BUFFER, request, sizeof(*request), &response, sizeof(response)); + if (dispatcher == nullptr) { + return nullptr; + } + dispatcher->send(RPC_CMD_ALLOC_BUFFER, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); if (response.remote_ptr != 0) { + LOG_DBG("[%s] endpoint: %s, device: %u, size: %s -> remote_ptr: 0x%" PRIx64 ", remote_size: %s\n", + __func__, buft_ctx->endpoint.c_str(), buft_ctx->device, rpc_log_format_size(request->size).c_str(), + response.remote_ptr, rpc_log_format_size(response.remote_size).c_str()); ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft, ggml_backend_rpc_buffer_interface, new ggml_backend_rpc_buffer_context{dispatcher, nullptr, response.remote_ptr}, response.remote_size); return buffer; } else { + LOG_WARN("[%s] endpoint: %s, device: %u, size: %s -> failed\n", + __func__, buft_ctx->endpoint.c_str(), buft_ctx->device, rpc_log_format_size(request->size).c_str()); return nullptr; } } @@ -821,7 +1033,8 @@ static size_t get_alignment(const std::shared_ptr<rpc_dispatcher> & dispatcher, auto request = std::make_shared<rpc_msg_get_alignment_req>(); request->device = device; rpc_msg_get_alignment_rsp response; - dispatcher->send(RPC_CMD_GET_ALIGNMENT, request, sizeof(*request), &response, sizeof(response)); + dispatcher->send(RPC_CMD_GET_ALIGNMENT, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); + LOG_DBG2("[%s] device: %u, alignment: %s\n", __func__, device, rpc_log_format_size(response.alignment).c_str()); return response.alignment; } @@ -834,7 +1047,8 @@ static size_t get_max_size(const std::shared_ptr<rpc_dispatcher> & dispatcher, u auto request = std::make_shared<rpc_msg_get_max_size_req>(); request->device = device; rpc_msg_get_max_size_rsp response; - dispatcher->send(RPC_CMD_GET_MAX_SIZE, request, sizeof(*request), &response, sizeof(response)); + dispatcher->send(RPC_CMD_GET_MAX_SIZE, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); + LOG_DBG2("[%s] device: %u, max_size: %s\n", __func__, device, rpc_log_format_size(response.max_size).c_str()); return response.max_size; } @@ -900,6 +1114,8 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty std::lock_guard<std::mutex> lock(cache_mutex); auto it = cache.find(cache_hash); if (it != cache.end()) { + LOG_DBG3("[%s] cache hit: %s -> %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), + rpc_log_format_size(it->second).c_str()); return std::max<size_t>(it->second, min_size); } } @@ -913,15 +1129,22 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty request->srcs[i] = serialize_tensor(tensor->src[i]); } + LOG_DBG2("[%s] cache miss: %s, querying server\n", __func__, rpc_log_format_tensor(tensor).c_str()); rpc_msg_get_alloc_size_rsp response; auto dispatcher = get_dispatcher(buft_ctx->endpoint); - dispatcher->send(RPC_CMD_GET_ALLOC_SIZE, request, sizeof(*request), &response, sizeof(response)); + if (dispatcher == nullptr) { + return min_size; + } + dispatcher->send(RPC_CMD_GET_ALLOC_SIZE, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); { std::lock_guard<std::mutex> lock(cache_mutex); cache[cache_hash] = response.alloc_size; } + LOG_DBG2("[%s] %s -> alloc_size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), + rpc_log_format_size(response.alloc_size).c_str()); + return std::max<size_t>(response.alloc_size, min_size); } @@ -955,14 +1178,18 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context; rpc_tensor rpc_tensor = serialize_tensor(tensor); uint8_t cache_flag = 0; - if (rpc_use_hash_cache(tensor, size)) { + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str()); + if (rpc_use_hash_cache(*ctx->dispatcher, tensor, size)) { auto request = std::make_shared<rpc_msg_set_tensor_hash_req>(); request->tensor = rpc_tensor; request->offset = offset; request->hash = fnv_hash((const uint8_t*)data, size); rpc_msg_set_tensor_hash_rsp response; // TODO: make this async - ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response)); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); + LOG_DBG2("[%s] tensor: %s, hash: 0x%" PRIx64 ", cache: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), + request->hash, response.result ? "hit" : "miss"); if (response.result) { // the server has the same data, no need to send it return; @@ -977,6 +1204,8 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso static void ggml_backend_rpc_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context; + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str()); auto request = std::make_shared<rpc_msg_get_tensor_req>(); request->tensor = serialize_tensor(tensor); request->offset = offset; @@ -989,7 +1218,7 @@ static void ggml_backend_rpc_synchronize(ggml_backend_t backend) { rpc_ctx->dispatcher->synchronize(); } -static void add_tensor(ggml_tensor * tensor, const ggml_cgraph * cgraph, const std::shared_ptr<rpc_dispatcher> & dispatcher, std::vector<rpc_tensor> & tensors, std::unordered_set<ggml_tensor*> & visited) { +static void add_tensor(ggml_tensor * tensor, const ggml_cgraph * cgraph, const rpc_dispatcher * dispatcher, std::vector<rpc_tensor> & tensors, std::unordered_set<ggml_tensor*> & visited) { if (tensor == nullptr) { return; } @@ -1009,7 +1238,7 @@ static void add_tensor(ggml_tensor * tensor, const ggml_cgraph * cgraph, const s tensors.push_back(result); } -static uint8_t * serialize_graph(uint32_t device, const ggml_cgraph * cgraph, const std::shared_ptr<rpc_dispatcher> & dispatcher, size_t * output_size) { +static uint8_t * serialize_graph(uint32_t device, const ggml_cgraph * cgraph, const rpc_dispatcher * dispatcher, size_t * output_size) { uint32_t n_nodes = cgraph->n_nodes; std::vector<rpc_tensor> tensors; std::unordered_set<ggml_tensor*> visited; @@ -1017,13 +1246,15 @@ static uint8_t * serialize_graph(uint32_t device, const ggml_cgraph * cgraph, co add_tensor(cgraph->nodes[i], cgraph, dispatcher, tensors, visited); } // serialization format: - // | device (4 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | + // | device (4 bytes) | uid (8 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | uint32_t n_tensors = tensors.size(); - *output_size = 2*sizeof(uint32_t) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t) + n_tensors * sizeof(rpc_tensor); + *output_size = 2*sizeof(uint32_t) + sizeof(uint64_t) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t) + n_tensors * sizeof(rpc_tensor); uint8_t * output = new uint8_t[*output_size](); uint8_t * dest = output; memcpy(dest, &device, sizeof(device)); dest += sizeof(device); + memcpy(dest, &cgraph->uid, sizeof(cgraph->uid)); + dest += sizeof(cgraph->uid); memcpy(dest, &n_nodes, sizeof(n_nodes)); dest += sizeof(n_nodes); for (uint32_t i = 0; i < n_nodes; i++) { @@ -1037,24 +1268,37 @@ static uint8_t * serialize_graph(uint32_t device, const ggml_cgraph * cgraph, co return output; } -static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; - ggml_backend_dev_t rpc_dev = ggml_backend_get_device(backend); - ggml_backend_rpc_device_context * rpc_dev_ctx = (ggml_backend_rpc_device_context *)rpc_dev->context; - +void rpc_dispatcher::graph_compute(uint32_t device, const ggml_cgraph * cgraph) { + std::lock_guard<std::mutex> lock(graph_mutex); GGML_ASSERT(cgraph->n_nodes > 0); - bool reuse = cgraph->uid != 0 && rpc_dev_ctx->last_graph_uid == cgraph->uid; + auto & device_graph_uids = graph_uids[device]; + bool reuse = cgraph->uid != 0 && device_graph_uids.count(cgraph->uid) > 0; + LOG_DBG("[%s] device: %u, uid: %" PRIu64 ", n_nodes: %u, reuse: %s\n", __func__, device, cgraph->uid, cgraph->n_nodes, reuse ? "yes" : "no"); if (reuse) { auto request = std::make_shared<rpc_msg_graph_recompute_req>(); - request->device = rpc_ctx->device; - rpc_ctx->dispatcher->send_async(RPC_CMD_GRAPH_RECOMPUTE, request, sizeof(*request)); + request->device = device; + request->uid = cgraph->uid; + send_async(RPC_CMD_GRAPH_RECOMPUTE, request, sizeof(*request)); } else { - rpc_dev_ctx->last_graph_uid = cgraph->uid; + if (cgraph->uid != 0) { + if (device_graph_uids.size() >= GRAPH_CACHE_MAX) { + LOG_DBG("[%s] clearing graph cache for device %u (%zu entries)\n", + __func__, device, device_graph_uids.size()); + device_graph_uids.clear(); + } + device_graph_uids.insert(cgraph->uid); + } size_t input_size = 0; - uint8_t * input = serialize_graph(rpc_ctx->device, cgraph, rpc_ctx->dispatcher, &input_size); + uint8_t * input = serialize_graph(device, cgraph, this, &input_size); + LOG_DBG2("[%s] serialized graph: %s\n", __func__, rpc_log_format_size(input_size).c_str()); std::shared_ptr<uint8_t> input_ptr(input, std::default_delete<uint8_t[]>()); - rpc_ctx->dispatcher->send_async(RPC_CMD_GRAPH_COMPUTE, input_ptr, input_size); + send_async(RPC_CMD_GRAPH_COMPUTE, input_ptr, input_size); } +} + +static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; + rpc_ctx->dispatcher->graph_compute(rpc_ctx->device, cgraph); return GGML_STATUS_SUCCESS; } @@ -1069,13 +1313,25 @@ static void ggml_backend_rpc_event_wait(ggml_backend_t backend, ggml_backend_eve GGML_UNUSED(event); } +static void ggml_backend_rpc_set_tensor_2d_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + ggml_backend_tensor_set_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); + GGML_UNUSED(backend); +} + +static void ggml_backend_rpc_get_tensor_2d_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + ggml_backend_tensor_get_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); + GGML_UNUSED(backend); +} + static ggml_backend_i ggml_backend_rpc_interface = { /* .get_name = */ ggml_backend_rpc_name, /* .free = */ ggml_backend_rpc_free, /* .set_tensor_async = */ ggml_backend_rpc_set_tensor_async, /* .get_tensor_async = */ ggml_backend_rpc_get_tensor_async, - /* .set_tensor_2d_async = */ NULL, - /* .get_tensor_2d_async = */ NULL, + /* .set_tensor_2d_async = */ ggml_backend_rpc_set_tensor_2d_async, + /* .get_tensor_2d_async = */ ggml_backend_rpc_get_tensor_2d_async, /* .cpy_tensor_async = */ NULL, /* .synchronize = */ ggml_backend_rpc_synchronize, /* .graph_plan_create = */ NULL, @@ -1099,6 +1355,9 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u return it->second; } auto dispatcher = get_dispatcher(endpoint); + if (dispatcher == nullptr) { + return nullptr; + } size_t alignment = get_alignment(dispatcher, device); size_t max_size = get_max_size(dispatcher, device); ggml_backend_rpc_buffer_type_context * buft_ctx = new ggml_backend_rpc_buffer_type_context { @@ -1115,18 +1374,25 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u /* .context = */ buft_ctx }; buft_map[buft_name] = buft; + LOG_DBG("[%s] created %s (alignment: %s, max_size: %s)\n", __func__, buft_name.c_str(), + rpc_log_format_size(alignment).c_str(), rpc_log_format_size(max_size).c_str()); return buft; } ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device) { std::string dev_name = "RPC" + std::to_string(device) + "[" + std::string(endpoint) + "]"; auto dispatcher = get_dispatcher(endpoint); + if (dispatcher == nullptr) { + return nullptr; + } ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context { /* .dispatcher = */ dispatcher, + /* .endpoint = */ endpoint, /* .device = */ device, /* .name = */ dev_name, }; auto reg = ggml_backend_rpc_add_server(endpoint); + LOG_DBG("[%s] %s\n", __func__, dev_name.c_str()); ggml_backend_t backend = new ggml_backend { /* .guid = */ ggml_backend_rpc_guid(), /* .iface = */ ggml_backend_rpc_interface, @@ -1142,10 +1408,17 @@ bool ggml_backend_is_rpc(ggml_backend_t backend) { void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total) { auto dispatcher = get_dispatcher(endpoint); + if (dispatcher == nullptr) { + *free = 0; + *total = 0; + return; + } auto request = std::make_shared<rpc_msg_get_device_memory_req>(); request->device = device; rpc_msg_get_device_memory_rsp response; - dispatcher->send(RPC_CMD_GET_DEVICE_MEMORY, request, sizeof(*request), &response, sizeof(response)); + dispatcher->send(RPC_CMD_GET_DEVICE_MEMORY, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); + LOG_DBG2("[%s] device: %u, free: %s, total: %s\n", __func__, device, + rpc_log_format_size(response.free_mem).c_str(), rpc_log_format_size(response.total_mem).c_str()); *free = response.free_mem; *total = response.total_mem; } @@ -1157,6 +1430,7 @@ class rpc_server { rpc_server(std::vector<ggml_backend_t> all_backends, const char * cache_dir) : backends(std::move(all_backends)), cache_dir(cache_dir) { stored_graphs.resize(backends.size()); + comm_states.resize(backends.size()); } ~rpc_server(); @@ -1168,12 +1442,17 @@ class rpc_server { bool free_buffer(const rpc_msg_free_buffer_req & request); bool buffer_clear(const rpc_msg_buffer_clear_req & request); bool memset_tensor(const rpc_msg_memset_tensor_req & request); - bool set_tensor(const std::vector<uint8_t> & input); + bool set_tensor(socket_ptr sock, uint64_t input_size); + bool set_tensor_2d(const std::vector<uint8_t> & input); bool set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response); bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector<uint8_t> & response); + bool get_tensor_2d(const rpc_msg_get_tensor_2d_req & request, std::vector<uint8_t> & response); bool copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response); bool graph_compute(const std::vector<uint8_t> & input); bool graph_recompute(const rpc_msg_graph_recompute_req & request); + bool comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_init_rsp & response); + bool comm_allreduce(const rpc_msg_comm_allreduce_req & request); + bool comm_free(const rpc_msg_comm_free_req & request); bool init_tensor(const rpc_msg_init_tensor_req & request); bool get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response); bool get_device_memory(const rpc_msg_get_device_memory_req & request, rpc_msg_get_device_memory_rsp & response); @@ -1184,6 +1463,10 @@ class rpc_server { }; private: + void sync_all_backends(); + void comm_free_device(uint32_t dev_id); + static bool comm_allreduce_pairwise(void * comm_ctx_v, ggml_tensor ** tensors); + static bool comm_bootstrap_exchange(void * user_data, const void * send_buf, void * recv_buf, size_t size); bool get_cached_file(uint64_t hash, std::vector<uint8_t> & data); ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor); ggml_tensor * create_node(uint64_t id, @@ -1192,17 +1475,43 @@ class rpc_server { std::unordered_map<uint64_t, struct ggml_tensor*> & tensor_map); + // Per-device state for one communicator. + struct comm_state { + // Sockets to the other servers. Rank 0 holds one per peer in rank order, so peers[i] is + // rank i + 1. Every other rank holds one to rank 0. The pairwise path reduces over peers[0]. + std::vector<socket_ptr> peers; + uint32_t rank = 0; + uint32_t world = 0; + ggml_backend_t pairwise_backend = nullptr; + ggml_backend_buffer_ptr scratch; + size_t scratch_size = 0; + std::vector<uint8_t> send_buf; + std::vector<uint8_t> recv_buf; + + // The backend's communicator. Null on the pairwise path, which has none. + void * backend_comm_ctx = nullptr; + + // The chosen reduction, set once a path is agreed. Receives backend_comm_ctx, or + // comm_state on the pairwise path. A communicator covers one device, so the caller + // passes a one-element tensor array. + ggml_backend_comm_allreduce_tensor_t allreduce_fn = nullptr; + }; + std::vector<ggml_backend_t> backends; const char * cache_dir; std::unordered_set<ggml_backend_buffer_t> buffers; - // store the last computed graph for each backend - std::vector<stored_graph> stored_graphs; + // computed graphs cached per backend, keyed by uid + std::vector<std::unordered_map<uint64_t, stored_graph>> stored_graphs; + std::vector<comm_state> comm_states; + // SET_TENSOR data is received in chunks through this buffer + std::vector<uint8_t> set_tensor_buf; }; void rpc_server::hello(rpc_msg_hello_rsp & response) { response.major = RPC_PROTO_MAJOR_VERSION; response.minor = RPC_PROTO_MINOR_VERSION; response.patch = RPC_PROTO_PATCH_VERSION; + response.flags = cache_dir ? RPC_DISPATCHER_FLAGS_SEND_TENSOR_HASH : 0; LOG_DBG("[%s] version: %d.%d.%d\n", __func__, response.major, response.minor, response.patch); } @@ -1224,7 +1533,7 @@ bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_ ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr) { - GGML_LOG_ERROR("Null tensor pointer passed to server get_alloc_size function.\n"); + LOG_ERROR("Null tensor pointer passed to server get_alloc_size function.\n"); return false; } for (int i = 0; i < GGML_MAX_SRC; i++) { @@ -1233,7 +1542,8 @@ bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_ } } - LOG_DBG("[%s] device: %d, buffer: %p, data: %p\n", __func__, dev_id, (void*)tensor->buffer, tensor->data); + LOG_DBG2("[%s] device: %u, tensor: %s, buffer: %p, data: %p\n", __func__, dev_id, rpc_log_format_tensor(tensor).c_str(), + (void*)tensor->buffer, tensor->data); if (tensor->buffer == nullptr) { //No buffer allocated. buft = ggml_backend_get_default_buffer_type(backends[dev_id]); @@ -1258,11 +1568,12 @@ bool rpc_server::alloc_buffer(const rpc_msg_alloc_buffer_req & request, rpc_msg_ if (buffer != nullptr) { response.remote_ptr = reinterpret_cast<uint64_t>(buffer); response.remote_size = buffer->size; - LOG_DBG("[%s] device: %d, size: %" PRIu64 " -> remote_ptr: %" PRIx64 ", remote_size: %" PRIu64 "\n", - __func__, dev_id, request.size, response.remote_ptr, response.remote_size); + LOG_DBG("[%s] device: %u, size: %s -> remote_ptr: 0x%" PRIx64 ", remote_size: %s\n", + __func__, dev_id, rpc_log_format_size(request.size).c_str(), response.remote_ptr, + rpc_log_format_size(response.remote_size).c_str()); buffers.insert(buffer); } else { - LOG_DBG("[%s] device: %d, size: %" PRIu64 " -> failed\n", __func__, dev_id, request.size); + LOG_WARN("[%s] device: %u, size: %s -> failed\n", __func__, dev_id, rpc_log_format_size(request.size).c_str()); } return true; } @@ -1274,7 +1585,7 @@ bool rpc_server::get_alignment(const rpc_msg_get_alignment_req & request, rpc_ms } ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]); size_t alignment = ggml_backend_buft_get_alignment(buft); - LOG_DBG("[%s] device: %d, alignment: %lu\n", __func__, dev_id, alignment); + LOG_DBG2("[%s] device: %u, alignment: %s\n", __func__, dev_id, rpc_log_format_size(alignment).c_str()); response.alignment = alignment; return true; } @@ -1286,16 +1597,16 @@ bool rpc_server::get_max_size(const rpc_msg_get_max_size_req & request, rpc_msg_ } ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]); size_t max_size = ggml_backend_buft_get_max_size(buft); - LOG_DBG("[%s] device: %d, max_size: %lu\n", __func__, dev_id, max_size); + LOG_DBG2("[%s] device: %u, max_size: %s\n", __func__, dev_id, rpc_log_format_size(max_size).c_str()); response.max_size = max_size; return true; } bool rpc_server::buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response) { - LOG_DBG("[%s] remote_ptr: %" PRIx64 "\n", __func__, request.remote_ptr); + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request.remote_ptr); ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr); if (buffers.find(buffer) == buffers.end()) { - GGML_LOG_ERROR("[%s] buffer not found\n", __func__); + LOG_ERROR("[%s] buffer not found\n", __func__); return false; } void * base = ggml_backend_buffer_get_base(buffer); @@ -1304,16 +1615,20 @@ bool rpc_server::buffer_get_base(const rpc_msg_buffer_get_base_req & request, rp } bool rpc_server::free_buffer(const rpc_msg_free_buffer_req & request) { - LOG_DBG("[%s] remote_ptr: %" PRIx64 "\n", __func__, request.remote_ptr); + sync_all_backends(); + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request.remote_ptr); ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr); if (buffers.find(buffer) == buffers.end()) { - GGML_LOG_ERROR("[%s] buffer not found\n", __func__); + LOG_ERROR("[%s] buffer not found\n", __func__); return false; } // Discard all cached graphs to avoid use-after-free in graph_recompute, - // since their nodes may hold pointers to the buffer being freed. - for (auto & sg : stored_graphs) { - sg.graph = nullptr; + // since their nodes may hold pointers to the buffer being freed. The client + // will have to re-serialize its graphs from now on. + for (auto & sgs : stored_graphs) { + for (auto & sg : sgs) { + sg.second.graph = nullptr; + } } ggml_backend_buffer_free(buffer); buffers.erase(buffer); @@ -1321,10 +1636,11 @@ bool rpc_server::free_buffer(const rpc_msg_free_buffer_req & request) { } bool rpc_server::buffer_clear(const rpc_msg_buffer_clear_req & request) { - LOG_DBG("[%s] remote_ptr: %" PRIx64 ", value: %u\n", __func__, request.remote_ptr, request.value); + sync_all_backends(); + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 ", value: %u\n", __func__, request.remote_ptr, request.value); ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr); if (buffers.find(buffer) == buffers.end()) { - GGML_LOG_ERROR("[%s] buffer not found\n", __func__); + LOG_ERROR("[%s] buffer not found\n", __func__); return false; } ggml_backend_buffer_clear(buffer, request.value); @@ -1332,6 +1648,7 @@ bool rpc_server::buffer_clear(const rpc_msg_buffer_clear_req & request) { } bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) { + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1342,13 +1659,13 @@ bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) { ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr || tensor->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__); + LOG_ERROR("[%s] error deserializing tensor\n", __func__); return false; } const uint64_t tensor_size = ggml_nbytes(tensor); if (request.offset > tensor_size || request.size > tensor_size - request.offset) { - GGML_LOG_ERROR("[%s] tensor region (offset=%" PRIu64 ", size=%" PRIu64 ") out of tensor bounds [0, %" PRIu64 ")\n", + LOG_ERROR("[%s] tensor region (offset=%" PRIu64 ", size=%" PRIu64 ") out of tensor bounds [0, %" PRIu64 ")\n", __func__, request.offset, request.size, tensor_size); return false; } @@ -1356,23 +1673,24 @@ bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) { const uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer); const uint64_t buffer_size = ggml_backend_buffer_get_size(tensor->buffer); if (request.tensor.data < buffer_start) { - GGML_LOG_ERROR("[%s] tensor data before buffer start\n", __func__); + LOG_ERROR("[%s] tensor data before buffer start\n", __func__); return false; } const uint64_t data_offset = request.tensor.data - buffer_start; if (data_offset > buffer_size || request.offset > buffer_size - data_offset || request.size > buffer_size - data_offset - request.offset) { - GGML_LOG_ERROR("[%s] tensor region out of buffer bounds\n", __func__); + LOG_ERROR("[%s] tensor region out of buffer bounds\n", __func__); return false; } if (tensor->buffer->iface.memset_tensor == nullptr) { - GGML_LOG_ERROR("[%s] memset not implemented by backend buffer\n", __func__); + LOG_ERROR("[%s] memset not implemented by backend buffer\n", __func__); return false; } - LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", value: %u\n", - __func__, (void *) tensor->buffer, tensor->data, request.offset, request.size, request.value); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s, value: %u\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void *) tensor->buffer, tensor->data, request.offset, + rpc_log_format_size(request.size).c_str(), request.value); ggml_backend_tensor_memset(tensor, request.value, request.offset, request.size); return true; } @@ -1380,13 +1698,13 @@ bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) { ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) { // Validate tensor type before using it if (tensor->type >= GGML_TYPE_COUNT) { - GGML_LOG_ERROR("[%s] invalid tensor type received: %u\n", __func__, tensor->type); + LOG_ERROR("[%s] invalid tensor type received: %u\n", __func__, tensor->type); return nullptr; } // Fix: Prevent division by zero if blck_size is 0 (e.g., deprecated types) if (ggml_blck_size((enum ggml_type)tensor->type) == 0) { - GGML_LOG_ERROR("[%s] invalid tensor type received (blck_size is 0): %u\n", __func__, tensor->type); + LOG_ERROR("[%s] invalid tensor type received (blck_size is 0): %u\n", __func__, tensor->type); return nullptr; } @@ -1395,7 +1713,7 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp // ggml_new_tensor_4d might fail if dimensions are invalid, although less likely to crash than invalid type if (result == nullptr) { - GGML_LOG_ERROR("[%s] ggml_new_tensor_4d failed for type %u\n", __func__, tensor->type); + LOG_ERROR("[%s] ggml_new_tensor_4d failed for type %u\n", __func__, tensor->type); return nullptr; } @@ -1407,13 +1725,20 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp result->buffer = nullptr; } - if (result->buffer) { + if (result->buffer && ggml_nelements(result) > 0) { // require that the tensor data does not go beyond the buffer end uint64_t tensor_size = (uint64_t) ggml_nbytes(result); uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(result->buffer); uint64_t buffer_size = (uint64_t) ggml_backend_buffer_get_size(result->buffer); - GGML_ASSERT(tensor->data + tensor_size >= tensor->data); // check for overflow - GGML_ASSERT(tensor->data >= buffer_start && tensor->data + tensor_size <= buffer_start + buffer_size); + if (tensor->data + tensor_size < tensor->data || + tensor->data < buffer_start || tensor->data + tensor_size > buffer_start + buffer_size) { + LOG_ERROR("[%s] tensor '%s' (op %s, type %s, ne [%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "]) " + "data [0x%" PRIx64 ", 0x%" PRIx64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n", + __func__, tensor->name, ggml_op_name((ggml_op) tensor->op), ggml_type_name(result->type), + result->ne[0], result->ne[1], result->ne[2], result->ne[3], + tensor->data, tensor->data + tensor_size, buffer_start, buffer_start + buffer_size); + return nullptr; + } } result->op = (ggml_op) tensor->op; @@ -1427,18 +1752,21 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp } -bool rpc_server::set_tensor(const std::vector<uint8_t> & input) { +bool rpc_server::set_tensor(socket_ptr sock, uint64_t input_size) { + sync_all_backends(); // serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) | + // only the header is read here, the data is streamed from the socket below uint8_t cache_flag; uint64_t offset; - const size_t header_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset); - if (input.size() < header_size) { + uint8_t header[sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset)]; + const size_t header_size = sizeof(header); + if (input_size < header_size || !sock->recv_data(header, header_size)) { return false; } - const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); - memcpy(&cache_flag, input.data() + sizeof(rpc_tensor), sizeof(cache_flag)); - memcpy(&offset, input.data() + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset)); - const size_t size = input.size() - header_size; + const rpc_tensor * in_tensor = (const rpc_tensor *)header; + memcpy(&cache_flag, header + sizeof(rpc_tensor), sizeof(cache_flag)); + memcpy(&offset, header + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset)); + const size_t size = input_size - header_size; struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), @@ -1450,10 +1778,12 @@ bool rpc_server::set_tensor(const std::vector<uint8_t> & input) { ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor); if (tensor == nullptr || tensor->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__); + LOG_ERROR("[%s] error deserializing tensor\n", __func__); return false; } - LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu\n", __func__, (void*)tensor->buffer, tensor->data, offset, size); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, offset, + rpc_log_format_size(size).c_str()); // sanitize tensor->data { @@ -1461,24 +1791,105 @@ bool rpc_server::set_tensor(const std::vector<uint8_t> & input) { const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer); if (in_tensor->data + offset < p0 || in_tensor->data + offset >= p1 || size > (p1 - in_tensor->data - offset)) { - GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu) out of buffer bounds [0x%zx, 0x%zx)\n", + LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu) out of buffer bounds [0x%zx, 0x%zx)\n", __func__, in_tensor->data, offset, size, p0, p1); return false; } } - const void * data = input.data() + header_size; if (cache_dir && cache_flag) { - uint64_t hash = fnv_hash((const uint8_t*)data, size); + // the whole data is needed to compute the hash + std::vector<uint8_t> data(size); + if (!sock->recv_data(data.data(), size)) { + return false; + } + uint64_t hash = fnv_hash(data.data(), size); char hash_str[17]; snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash); // save to cache_dir/hash_str fs::path cache_file = fs::path(cache_dir) / hash_str; std::ofstream ofs(cache_file, std::ios::binary); - ofs.write((const char *)data, size); - GGML_LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str()); + ofs.write((const char *)data.data(), size); + LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str()); + ggml_backend_tensor_set(tensor, data.data(), offset, size); + return true; + } + + // a buffer of the tensor size would be allocated and page faulted for every tensor, which is slow + const size_t chunk_size = 16*1024*1024; + if (set_tensor_buf.size() < std::min(size, chunk_size)) { + set_tensor_buf.resize(std::min(size, chunk_size)); + } + for (size_t done = 0; done < size; ) { + const size_t n = std::min(size - done, chunk_size); + if (!sock->recv_data(set_tensor_buf.data(), n)) { + return false; + } + ggml_backend_tensor_set(tensor, set_tensor_buf.data(), offset + done, n); + done += n; + } + return true; +} + +bool rpc_server::set_tensor_2d(const std::vector<uint8_t> & input) { + sync_all_backends(); + // serialization format: | rpc_tensor | offset (8 bytes) | size (8 bytes) | n_copies (8 bytes) | stride (8 bytes) | data (size * n_copies bytes) | + if (input.size() < sizeof(rpc_tensor) + 4*sizeof(uint64_t)) { + return false; + } + const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); + uint64_t header[4]; + memcpy(header, input.data() + sizeof(rpc_tensor), sizeof(header)); + const uint64_t offset = header[0]; + const uint64_t size = header[1]; + const uint64_t n_copies = header[2]; + const uint64_t stride = header[3]; + + const uint64_t data_size = input.size() - sizeof(rpc_tensor) - 4*sizeof(uint64_t); + if (n_copies == 0 || size == 0 || size > data_size / n_copies || size * n_copies != data_size) { + return false; + } + + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_context * ctx = ctx_ptr.get(); + ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor); + if (tensor == nullptr || tensor->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s, n_copies: %" PRIu64 ", stride: %" PRIu64 "\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, offset, + rpc_log_format_size(size).c_str(), n_copies, stride); + + // sanitize tensor->data + { + if (stride != 0 && n_copies - 1 > (UINT64_MAX - size) / stride) { + return false; + } + const uint64_t span = (n_copies - 1)*stride + size; + const uint64_t p0 = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer); + const uint64_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer); + + if (in_tensor->data < p0 || in_tensor->data > p1 || offset > p1 - in_tensor->data || span > p1 - in_tensor->data - offset) { + LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", span=%" PRIu64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n", + __func__, in_tensor->data, offset, span, p0, p1); + return false; + } + if (offset > ggml_nbytes(tensor) || span > ggml_nbytes(tensor) - offset) { + LOG_ERROR("[%s] tensor write region (offset=%" PRIu64 ", span=%" PRIu64 ") out of tensor bounds (%zu)\n", + __func__, offset, span, ggml_nbytes(tensor)); + return false; + } } - ggml_backend_tensor_set(tensor, data, offset, size); + + const void * data = input.data() + sizeof(rpc_tensor) + 4*sizeof(uint64_t); + ggml_backend_tensor_set_2d(tensor, data, offset, size, n_copies, stride, size); return true; } @@ -1504,12 +1915,7 @@ bool rpc_server::get_cached_file(uint64_t hash, std::vector<uint8_t> & data) { bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response) { - std::vector<uint8_t> cached_file; - if (!get_cached_file(request.hash, cached_file)) { - response.result = 0; - return true; - } - size_t size = cached_file.size(); + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1520,11 +1926,21 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr || tensor->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__); + LOG_ERROR("[%s] error deserializing tensor\n", __func__); return false; } - LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu, hash: %" PRIx64 "\n", - __func__, (void*)tensor->buffer, tensor->data, request.offset, size, request.hash); + + std::vector<uint8_t> cached_file; + if (!get_cached_file(request.hash, cached_file)) { + LOG_DBG2("[%s] tensor: %s, offset: %" PRIu64 ", hash: 0x%" PRIx64 ", cache miss\n", + __func__, rpc_log_format_tensor(tensor).c_str(), request.offset, request.hash); + response.result = 0; + return true; + } + size_t size = cached_file.size(); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s, hash: 0x%" PRIx64 ", cache hit\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, request.offset, + rpc_log_format_size(size).c_str(), request.hash); // sanitize tensor->data { @@ -1534,7 +1950,7 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp if (request.tensor.data + request.offset < p0 || request.tensor.data + request.offset >= p1 || size > (p1 - request.tensor.data - request.offset)) { - GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu, hash=0x%" PRIx64 ") out of buffer bounds [0x%zx, 0x%zx)\n", + LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu, hash=0x%" PRIx64 ") out of buffer bounds [0x%zx, 0x%zx)\n", __func__, request.tensor.data, request.offset, size, request.hash, p0, p1); return false; } @@ -1545,6 +1961,7 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp } bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1555,24 +1972,25 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr) { - GGML_LOG_ERROR("Null tensor pointer passed to server init_tensor function.\n"); + LOG_ERROR("Null tensor pointer passed to server init_tensor function.\n"); return false; } - LOG_DBG("[%s] buffer: %p, data: %p\n", __func__, (void*)tensor->buffer, tensor->data); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p\n", __func__, rpc_log_format_tensor(tensor).c_str(), + (void*)tensor->buffer, tensor->data); // Call the backend's buffer_init_tensor function ggml_backend_buffer_t buffer = tensor->buffer; if (buffer && buffer->iface.init_tensor) { buffer->iface.init_tensor(buffer, tensor); } else { if (!buffer) { - GGML_LOG_ERROR("Tensor with null buffer passed to init_tensor function\n"); + LOG_ERROR("Tensor with null buffer passed to init_tensor function\n"); } } if (tensor->extra != nullptr) { // This pointer can either be passed around client/server, or probably better stored server-side and kept track of. // Currently unimplemented. - GGML_LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n"); + LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n"); return false; } @@ -1580,6 +1998,7 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { } bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector<uint8_t> & response) { + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1590,10 +2009,12 @@ bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector< ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr || tensor->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__); + LOG_ERROR("[%s] error deserializing tensor\n", __func__); return false; } - LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 "\n", __func__, (void*)tensor->buffer, tensor->data, request.offset, request.size); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s\n", __func__, + rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, request.offset, + rpc_log_format_size(request.size).c_str()); // sanitize tensor->data { @@ -1603,7 +2024,7 @@ bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector< if (request.tensor.data + request.offset < p0 || request.tensor.data + request.offset >= p1 || request.size > (p1 - request.tensor.data - request.offset)) { - GGML_LOG_ERROR("[%s] requested tensor region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%" PRIu64 ") out of buffer bounds [0x%zx, 0x%zx)\n", + LOG_ERROR("[%s] requested tensor region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%" PRIu64 ") out of buffer bounds [0x%zx, 0x%zx)\n", __func__, request.tensor.data, request.offset, request.size, p0, p1); return false; } @@ -1614,7 +2035,57 @@ bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector< return true; } +bool rpc_server::get_tensor_2d(const rpc_msg_get_tensor_2d_req & request, std::vector<uint8_t> & response) { + sync_all_backends(); + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_context * ctx = ctx_ptr.get(); + ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); + if (tensor == nullptr || tensor->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s, n_copies: %" PRIu64 ", stride: %" PRIu64 "\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, request.offset, + rpc_log_format_size(request.size).c_str(), request.n_copies, request.stride); + + // sanitize tensor->data + { + if (request.n_copies == 0 || request.size == 0 || request.size > UINT64_MAX / request.n_copies) { + return false; + } + if (request.stride != 0 && request.n_copies - 1 > (UINT64_MAX - request.size) / request.stride) { + return false; + } + const uint64_t span = (request.n_copies - 1)*request.stride + request.size; + const uint64_t p0 = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer); + const uint64_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer); + + if (request.tensor.data < p0 || request.tensor.data > p1 || request.offset > p1 - request.tensor.data || + span > p1 - request.tensor.data - request.offset) { + LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", span=%" PRIu64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n", + __func__, request.tensor.data, request.offset, span, p0, p1); + return false; + } + if (request.offset > ggml_nbytes(tensor) || span > ggml_nbytes(tensor) - request.offset) { + LOG_ERROR("[%s] tensor read region (offset=%" PRIu64 ", span=%" PRIu64 ") out of tensor bounds (%zu)\n", + __func__, request.offset, span, ggml_nbytes(tensor)); + return false; + } + } + + response.resize(request.size * request.n_copies, 0); + ggml_backend_tensor_get_2d(tensor, response.data(), request.offset, request.size, request.n_copies, request.stride, request.size); + return true; +} + bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response) { + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ 2*ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1627,7 +2098,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co ggml_tensor * src = deserialize_tensor(ctx, &request.src); ggml_tensor * dst = deserialize_tensor(ctx, &request.dst); if (src == nullptr || dst == nullptr || src->buffer == nullptr || dst->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensors\n", __func__); + LOG_ERROR("[%s] error deserializing tensors\n", __func__); return false; } @@ -1637,7 +2108,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co uint64_t dst_buf_sz = (uint64_t) ggml_backend_buffer_get_size(dst->buffer); if (dst_data + src_size > dst_base + dst_buf_sz) { - GGML_LOG_ERROR("[%s] out-of-bounds write in rpc_server::copy_tensor:\n" + LOG_ERROR("[%s] out-of-bounds write in rpc_server::copy_tensor:\n" " write range : [0x%" PRIx64 ", 0x%" PRIx64 "]\n" " buffer base: [0x%" PRIx64 ", 0x%" PRIx64 "]\n", __func__, @@ -1648,8 +2119,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co return false; } - LOG_DBG("[%s] src->buffer: %p, dst->buffer: %p\n", - __func__, (void*) src->buffer, (void*) dst->buffer); + LOG_DBG2("[%s] src: %s -> dst: %s\n", __func__, rpc_log_format_tensor(src).c_str(), rpc_log_format_tensor(dst).c_str()); response.result = ggml_backend_buffer_copy_tensor(src, dst); return true; @@ -1674,7 +2144,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id, return nullptr; } if (result->buffer == nullptr && result->data != nullptr) { - GGML_LOG_ERROR("[%s] invalid data ptr", __func__); + LOG_ERROR("[%s] invalid data ptr\n", __func__); return nullptr; } tensor_map[id] = result; @@ -1686,7 +2156,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id, result->src[i] = create_node(tensor->src[i], ctx, tensor_ptrs, tensor_map); // If the recursive call failed for a non-zero ID, propagate the error if (result->src[i] == nullptr) { - GGML_LOG_ERROR("[%s] failed to create source node %d (src_id=%" PRIu64 ") for node id %" PRIu64 "\n", + LOG_ERROR("[%s] failed to create source node %d (src_id=%" PRIu64 ") for node id %" PRIu64 "\n", __func__, i, tensor->src[i], id); // Must return nullptr to signal failure up the call stack return nullptr; @@ -1701,7 +2171,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id, result->view_src = create_node(tensor->view_src, ctx, tensor_ptrs, tensor_map); // If the recursive call failed for a non-zero ID, propagate the error if (result->view_src == nullptr) { - GGML_LOG_ERROR("[%s] failed to create view_src node (view_src_id=%" PRIu64 ") for node id %" PRIu64 "\n", + LOG_ERROR("[%s] failed to create view_src node (view_src_id=%" PRIu64 ") for node id %" PRIu64 "\n", __func__, tensor->view_src, id); // Must return nullptr to signal failure up the call stack return nullptr; @@ -1713,8 +2183,8 @@ ggml_tensor * rpc_server::create_node(uint64_t id, bool rpc_server::graph_compute(const std::vector<uint8_t> & input) { // serialization format: - // | device (4 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | - if (input.size() < 2*sizeof(uint32_t)) { + // | device (4 bytes) | uid (8 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | + if (input.size() < 2*sizeof(uint32_t) + sizeof(uint64_t)) { return false; } const uint8_t * src = input.data(); @@ -1724,10 +2194,13 @@ bool rpc_server::graph_compute(const std::vector<uint8_t> & input) { if (device >= backends.size()) { return false; } + uint64_t uid; + memcpy(&uid, src, sizeof(uid)); + src += sizeof(uid); uint32_t n_nodes; memcpy(&n_nodes, src, sizeof(n_nodes)); src += sizeof(n_nodes); - if (input.size() < 2*sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t)) { + if (input.size() < 2*sizeof(uint32_t) + sizeof(uint64_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t)) { return false; } const uint64_t * nodes = (const uint64_t *)src; @@ -1735,19 +2208,28 @@ bool rpc_server::graph_compute(const std::vector<uint8_t> & input) { uint32_t n_tensors; memcpy(&n_tensors, src, sizeof(n_tensors)); src += sizeof(n_tensors); - if (input.size() < 2*sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t) + n_tensors*sizeof(rpc_tensor)) { + if (input.size() < 2*sizeof(uint32_t) + sizeof(uint64_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t) + n_tensors*sizeof(rpc_tensor)) { return false; } const rpc_tensor * tensors = (const rpc_tensor *)src; - LOG_DBG("[%s] device: %u, n_nodes: %u, n_tensors: %u\n", __func__, device, n_nodes, n_tensors); + LOG_DBG("[%s] device: %u, uid: %" PRIu64 ", n_nodes: %u, n_tensors: %u\n", __func__, device, uid, n_nodes, n_tensors); + + // graphs with uid == 0 are not cached, see GRAPH_CACHE_MAX for the eviction policy + if (uid != 0 && stored_graphs[device].size() >= GRAPH_CACHE_MAX) { + LOG_DBG("[%s] clearing graph cache for device %u (%zu entries)\n", + __func__, device, stored_graphs[device].size()); + stored_graphs[device].clear(); + } + stored_graph sg_tmp; + stored_graph & sg = uid != 0 ? stored_graphs[device][uid] : sg_tmp; size_t buf_size = ggml_tensor_overhead()*(n_nodes + n_tensors) + ggml_graph_overhead_custom(n_nodes, false); - if (stored_graphs[device].buffer.size() < buf_size) { - stored_graphs[device].buffer.resize(buf_size); + if (sg.buffer.size() < buf_size) { + sg.buffer.resize(buf_size); } struct ggml_init_params params = { /*.mem_size =*/ buf_size, - /*.mem_buffer =*/ stored_graphs[device].buffer.data(), + /*.mem_buffer =*/ sg.buffer.data(), /*.no_alloc =*/ true, }; ggml_context_ptr ctx_ptr { ggml_init(params) }; @@ -1762,6 +2244,9 @@ bool rpc_server::graph_compute(const std::vector<uint8_t> & input) { } std::unordered_map<uint64_t, ggml_tensor*> tensor_map; tensor_map.reserve(n_nodes); + // the compute itself is asynchronous, so building the graph is the only server-side cost + // observable here - the round trip is timed on the client + const auto t_deserialize = std::chrono::steady_clock::now(); for (uint32_t i = 0; i < n_nodes; i++) { int64_t id; memcpy(&id, &nodes[i], sizeof(id)); @@ -1770,17 +2255,21 @@ bool rpc_server::graph_compute(const std::vector<uint8_t> & input) { // If id was 0, create_node returning nullptr is expected. // If id was non-zero and create_node returned nullptr, it indicates a deserialization error. if (graph->nodes[i] == nullptr && id != 0) { - GGML_LOG_ERROR("[%s] failed to create graph node %d (id=%" PRId64 ")\n", __func__, i, id); + LOG_ERROR("[%s] failed to create graph node %d (id=%" PRId64 ")\n", __func__, i, id); return false; } if (graph->nodes[i] != nullptr) { + LOG_DBG3("[%s] node %u: id=%" PRId64 ", %s\n", __func__, i, id, rpc_log_format_tensor(graph->nodes[i]).c_str()); const size_t hash_pos = ggml_hash_insert(&graph->visited_hash_set, graph->nodes[i]); graph->use_counts[hash_pos] = tensor_ptrs.at(id)->use_count; } } - ggml_status status = ggml_backend_graph_compute(backends[device], graph); + const std::chrono::duration<double, std::milli> deserialize_ms = std::chrono::steady_clock::now() - t_deserialize; + + ggml_status status = ggml_backend_graph_compute_async(backends[device], graph); GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC"); - stored_graphs[device].graph = graph; + sg.graph = graph; + LOG_DBG3("[%s] device: %u, graph build: %.3f ms\n", __func__, device, deserialize_ms.count()); return true; } @@ -1789,16 +2278,356 @@ bool rpc_server::graph_recompute(const rpc_msg_graph_recompute_req & request) { if (device >= backends.size()) { return false; } - if (stored_graphs[device].graph == nullptr) { + auto it = stored_graphs[device].find(request.uid); + if (it == stored_graphs[device].end() || it->second.graph == nullptr) { + LOG_ERROR("[%s] device: %u, graph with uid %" PRIu64 " not found\n", __func__, device, request.uid); return false; } - ggml_cgraph * graph = stored_graphs[device].graph; - LOG_DBG("[%s] device: %u\n", __func__, device); - ggml_status status = ggml_backend_graph_compute(backends[device], graph); + ggml_cgraph * graph = it->second.graph; + LOG_DBG("[%s] device: %u, uid: %" PRIu64 "\n", __func__, device, request.uid); + ggml_status status = ggml_backend_graph_compute_async(backends[device], graph); GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC"); return true; } +// graph compute is asynchronous; commands that read or write buffer data synchronize first +void rpc_server::sync_all_backends() { + for (ggml_backend_t backend : backends) { + ggml_backend_synchronize(backend); + } +} + +void rpc_server::comm_free_device(uint32_t dev_id) { + comm_state & state = comm_states[dev_id]; + if (state.backend_comm_ctx != nullptr) { + auto comm_free = (ggml_backend_comm_free_t) ggml_backend_reg_get_proc_address( + ggml_backend_dev_backend_reg(ggml_backend_get_device(backends[dev_id])), "ggml_backend_comm_free"); + if (comm_free != nullptr) { + comm_free(state.backend_comm_ctx); + } + } + state = comm_state(); +} + +// Exchange RPC transport capabilities over a peer connection. recv_first selects which side +// receives first. Both sides receiving first would deadlock. +static bool comm_caps_handshake(const socket_ptr & peer, bool recv_first) { + uint8_t local_caps[RPC_CONN_CAPS_SIZE] = {}; + uint8_t remote_caps[RPC_CONN_CAPS_SIZE] = {}; + peer->get_caps(local_caps); + if (recv_first) { + if (!peer->recv_data(remote_caps, sizeof(remote_caps)) || + !peer->send_data(local_caps, sizeof(local_caps))) { + return false; + } + } else { + if (!peer->send_data(local_caps, sizeof(local_caps)) || + !peer->recv_data(remote_caps, sizeof(remote_caps))) { + return false; + } + } + peer->update_caps(remote_caps); + return true; +} + +// The comm link between servers uses the same caps negotiation as the client HELLO, +// so it gets the same transport upgrades (e.g. RDMA). +bool rpc_server::comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_init_rsp & response) { + response.ok = 0; + if (request.device >= backends.size() || request.world < 2 || request.rank >= request.world) { + return true; + } + const uint32_t dev_id = request.device; + comm_state & state = comm_states[dev_id]; + if (state.allreduce_fn != nullptr) { + response.ok = 1; + return true; + } + + // Joining a communicator blocks until every rank arrives, so drain outstanding work first. + sync_all_backends(); + + // Rank 0 listens and accepts world-1 peers; every other rank connects back to it. + if (request.rank == 0) { + socket_ptr srv = socket_t::create_server("0.0.0.0", request.port); + if (srv == nullptr) { + LOG_ERROR("[%s] failed to listen on comm port %u\n", __func__, request.port); + return true; + } + // Store peers[i] as rank i + 1, as comm_bootstrap_exchange needs. + // Each peer sends its rank because peers may connect in any order. + state.peers.resize(request.world - 1); + for (uint32_t i = 1; i < request.world; i++) { + socket_ptr peer = srv->accept(); + uint32_t peer_rank = 0; + if (peer == nullptr || !comm_caps_handshake(peer, /*recv_first =*/ true) || + !peer->recv_data(&peer_rank, sizeof(peer_rank)) || + peer_rank == 0 || peer_rank >= request.world || state.peers[peer_rank - 1] != nullptr) { + LOG_ERROR("[%s] bad peer connection (rank %u)\n", __func__, peer_rank); + state.peers.clear(); + return true; + } + state.peers[peer_rank - 1] = std::move(peer); + } + } else { + const std::string host(request.host, strnlen(request.host, sizeof(request.host))); + socket_ptr peer; + // rank 0 may not be listening yet, retry for a few seconds + for (int i = 0; i < 100 && peer == nullptr; i++) { + peer = socket_t::connect(host.c_str(), request.port); + if (peer == nullptr) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + } + if (peer == nullptr) { + LOG_ERROR("[%s] failed to connect to peer %s:%u\n", __func__, host.c_str(), request.port); + return true; + } + if (!comm_caps_handshake(peer, /*recv_first =*/ false) || + !peer->send_data(&request.rank, sizeof(request.rank)) || !peer->flush()) { + return true; + } + state.peers.push_back(std::move(peer)); + } + + state.rank = request.rank; + state.world = request.world; + + // Forces the pairwise path, for comparing the two on one build. Setting it on any one rank + // is enough, because the negotiation spreads the decision. + const bool no_backend_communicator = std::getenv("GGML_RPC_NO_BACKEND_COMMUNICATOR") != nullptr; + + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backends[dev_id])); + auto init_rank = (ggml_backend_comm_init_rank_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_comm_init_rank"); + auto allreduce_fn = (ggml_backend_comm_allreduce_tensor_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_comm_allreduce_tensor"); + + // The ranks must agree on backend communicator or pairwise. Every rank shares whether its + // backend provides a communicator, and all ranks use it only if every rank can. + const uint8_t self_can_use_backend_comm = !no_backend_communicator && init_rank != nullptr && allreduce_fn != nullptr; + std::vector<uint8_t> all_can_use_backend_comm(request.world); + if (!comm_bootstrap_exchange(&state, &self_can_use_backend_comm, all_can_use_backend_comm.data(), + sizeof(self_can_use_backend_comm))) { + state.peers.clear(); + return true; + } + const bool use_backend_communicator = std::all_of(all_can_use_backend_comm.begin(), all_can_use_backend_comm.end(), + [](uint8_t c) { return c != 0; }); + + if (use_backend_communicator) { + void * backend_ctx = init_rank(backends[dev_id], (int) request.rank, (int) request.world, comm_bootstrap_exchange, &state); + if (backend_ctx == nullptr) { + LOG_ERROR("[%s] failed to join communicator as rank %u/%u\n", __func__, request.rank, request.world); + state.peers.clear(); + return true; + } + state.backend_comm_ctx = backend_ctx; + state.allreduce_fn = allreduce_fn; + // The sockets have done their job; the collective runs between the backends from here on. + state.peers.clear(); + LOG_INFO("[%s] device %u using backend communicator, rank %u/%u\n", __func__, dev_id, request.rank, request.world); + response.ok = 1; + return true; + } + + // No backend communicator, so fall back to the pairwise exchange, which only covers two ranks. + if (request.world != 2) { + LOG_ERROR("[%s] world size %u requires a backend communicator\n", __func__, request.world); + state.peers.clear(); + return true; + } + state.pairwise_backend = backends[dev_id]; + state.allreduce_fn = comm_allreduce_pairwise; + LOG_INFO("[%s] device %u using pairwise communicator, rank %u\n", __func__, dev_id, request.rank); + response.ok = 1; + return true; +} + +// Bootstrap callback for ggml_backend_comm_init_rank_t, over the peer sockets. Rank 0 collects +// size bytes from every rank in rank order, then sends the whole array back to every rank. +// user_data is the comm_state. +bool rpc_server::comm_bootstrap_exchange(void * user_data, const void * send_buf, void * recv_buf, size_t size) { + comm_state & state = *(comm_state *) user_data; + uint8_t * recv = (uint8_t *) recv_buf; + const size_t total = (size_t) state.world * size; + + if (state.rank == 0) { + memcpy(recv, send_buf, size); + for (size_t i = 0; i < state.peers.size(); i++) { + if (!state.peers[i]->recv_data(recv + (i + 1) * size, size)) { + return false; + } + } + for (const socket_ptr & peer : state.peers) { + if (!peer->send_data(recv, total) || !peer->flush()) { + return false; + } + } + return true; + } + return state.peers[0]->send_data(send_buf, size) && state.peers[0]->flush() && + state.peers[0]->recv_data(recv, total); +} + +// Reduces this rank's contribution in place, exchanging partials with the peer over peers[0]. +// Used when the backend provides no communicator. comm_ctx_v is the comm_state. +bool rpc_server::comm_allreduce_pairwise(void * comm_ctx_v, ggml_tensor ** tensors) { + comm_state & state = *(comm_state *) comm_ctx_v; + ggml_tensor * t_dst = tensors[0]; + ggml_backend_t backend = state.pairwise_backend; + + const size_t nbytes = ggml_nbytes(t_dst); + const int64_t ne = ggml_nelements(t_dst); + + size_t ctx_size = 16*ggml_tensor_overhead() + 2*ggml_graph_overhead_custom(8, false); + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_context * ctx = ctx_ptr.get(); + + // reduce large partials in bf16 to halve the wire bytes; small (decode-sized) ones + // stay f32 since the extra casts and sync cost more than the bytes saved + const bool wire_bf16 = t_dst->type == GGML_TYPE_F32 && ne >= 32768; + const size_t wire_bytes = wire_bf16 ? (size_t) ne*2 : nbytes; + const size_t need = wire_bf16 ? 2*nbytes : nbytes; + if (state.scratch_size < need) { + state.scratch.reset(ggml_backend_alloc_buffer(backend, need)); + state.scratch_size = need; + } + char * scratch_base = (char *) ggml_backend_buffer_get_base(state.scratch.get()); + state.send_buf.resize(wire_bytes); + state.recv_buf.resize(wire_bytes); + + auto new_scratch_tensor = [&](ggml_type type, size_t offset) { + ggml_tensor * t = ggml_new_tensor_4d(ctx, type, t_dst->ne[0], t_dst->ne[1], t_dst->ne[2], t_dst->ne[3]); + t->buffer = state.scratch.get(); + t->data = scratch_base + offset; + return t; + }; + auto new_cpy_node = [&](ggml_tensor * src, ggml_tensor * dst) { + ggml_tensor * t = ggml_new_tensor_4d(ctx, dst->type, dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + t->op = GGML_OP_CPY; + t->src[0] = src; + t->src[1] = dst; + t->buffer = dst->buffer; + t->data = dst->data; + t->flags |= GGML_TENSOR_FLAG_COMPUTE; + return t; + }; + auto compute_nodes = [&](ggml_tensor * n0, ggml_tensor * n1) { + ggml_cgraph * graph = ggml_new_graph_custom(ctx, 2, false); + graph->nodes[0] = n0; + graph->nodes[1] = n1; + graph->n_nodes = n1 != nullptr ? 2 : 1; + ggml_status status = ggml_backend_graph_compute_async(backend, graph); + GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC"); + }; + + // wait for the pending subgraph that produced this partial + ggml_backend_synchronize(backend); + + ggml_tensor * t_wire_send = nullptr; + ggml_tensor * t_wire_recv = nullptr; + if (wire_bf16) { + t_wire_send = new_scratch_tensor(GGML_TYPE_BF16, 0); + t_wire_recv = new_scratch_tensor(GGML_TYPE_BF16, ne*2); + compute_nodes(new_cpy_node(t_dst, t_wire_send), nullptr); + ggml_backend_synchronize(backend); + ggml_backend_tensor_get(t_wire_send, state.send_buf.data(), 0, wire_bytes); + } else { + ggml_backend_tensor_get(t_dst, state.send_buf.data(), 0, wire_bytes); + } + + // rank 0 sends first, rank 1 receives first, so large payloads cannot deadlock + if (state.rank == 0) { + if (!state.peers[0]->send_data(state.send_buf.data(), wire_bytes) || !state.peers[0]->flush() || + !state.peers[0]->recv_data(state.recv_buf.data(), wire_bytes)) { + return false; + } + } else { + if (!state.peers[0]->recv_data(state.recv_buf.data(), wire_bytes) || + !state.peers[0]->send_data(state.send_buf.data(), wire_bytes) || !state.peers[0]->flush()) { + return false; + } + } + + ggml_tensor * t_peer = new_scratch_tensor(t_dst->type, wire_bf16 ? (size_t) ne*4 : 0); + ggml_tensor * t_cast = nullptr; + if (wire_bf16) { + ggml_backend_tensor_set(t_wire_recv, state.recv_buf.data(), 0, wire_bytes); + t_cast = new_cpy_node(t_wire_recv, t_peer); + } else { + ggml_backend_tensor_set(t_peer, state.recv_buf.data(), 0, wire_bytes); + } + + ggml_tensor * t_red = ggml_new_tensor_4d(ctx, t_dst->type, t_dst->ne[0], t_dst->ne[1], t_dst->ne[2], t_dst->ne[3]); + t_red->op = GGML_OP_ADD; + t_red->src[0] = t_dst; + t_red->src[1] = t_peer; + t_red->buffer = t_dst->buffer; + t_red->data = t_dst->data; + t_red->flags |= GGML_TENSOR_FLAG_COMPUTE; + + if (t_cast != nullptr) { + compute_nodes(t_cast, t_red); + } else { + compute_nodes(t_red, nullptr); + } + return true; +} + +bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { + if (request.device >= backends.size()) { + return false; + } + comm_state & state = comm_states[request.device]; + if (state.allreduce_fn == nullptr) { + LOG_ERROR("[%s] no communicator for device %u\n", __func__, request.device); + return false; + } + + struct ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_tensor * t_dst = deserialize_tensor(ctx_ptr.get(), &request.tensor); + if (t_dst == nullptr || t_dst->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + if (ggml_nbytes(t_dst) == 0) { + return true; + } + + // Neither path needs a wait here. The pairwise path waits internally, and the backend orders + // its reduction after the work that produced the partial sum. + void * comm_ctx = state.backend_comm_ctx != nullptr ? state.backend_comm_ctx : &state; + if (!state.allreduce_fn(comm_ctx, &t_dst)) { + LOG_ERROR("[%s] allreduce failed for tensor %s\n", __func__, t_dst->name); + return false; + } + return true; +} + +bool rpc_server::comm_free(const rpc_msg_comm_free_req & request) { + if (request.device >= backends.size()) { + return false; + } + // Wait for outstanding work. Destroying a communicator while its kernels are still running + // is unsafe. + sync_all_backends(); + comm_free_device(request.device); + return true; +} + bool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request, rpc_msg_get_device_memory_rsp & response) { uint32_t dev_id = request.device; if (dev_id >= backends.size()) { @@ -1809,11 +2638,20 @@ bool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request ggml_backend_dev_memory(dev, &free, &total); response.free_mem = free; response.total_mem = total; - LOG_DBG("[%s] device: %u, free_mem: %" PRIu64 ", total_mem: %" PRIu64 "\n", __func__, dev_id, response.free_mem, response.total_mem); + LOG_DBG2("[%s] device: %u, free: %s, total: %s\n", __func__, dev_id, + rpc_log_format_size(response.free_mem).c_str(), rpc_log_format_size(response.total_mem).c_str()); return true; } rpc_server::~rpc_server() { + // Wait for outstanding work. Destroying a communicator while its kernels are still running + // is unsafe. + sync_all_backends(); + + // Free any communicator still open. A client can disconnect without sending COMM_FREE. + for (uint32_t dev_id = 0; dev_id < comm_states.size(); dev_id++) { + comm_free_device(dev_id); + } for (auto buffer : buffers) { ggml_backend_buffer_free(buffer); } @@ -1827,7 +2665,7 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const return; } if (cmd != RPC_CMD_HELLO) { - GGML_LOG_ERROR("Expected HELLO command, update client\n"); + LOG_ERROR("Expected HELLO command, update client\n"); return; } @@ -1838,7 +2676,7 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const } if (hello_input_size != sizeof(rpc_msg_hello_req)) { - GGML_LOG_ERROR("HELLO request size mismatch (%zu vs %zu) — client needs upgrade to protocol v%d.x\n", + LOG_ERROR("HELLO request size mismatch (%zu vs %zu) — client needs upgrade to protocol v%d.x\n", (size_t)hello_input_size, sizeof(rpc_msg_hello_req), RPC_PROTO_MAJOR_VERSION); return; } @@ -1858,15 +2696,17 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const // Activate transport upgrade using client's caps sock->update_caps(req.conn_caps); + LOG_DBG("[%s] client connected, transport: %s\n", __func__, sock->transport_name()); while (true) { if (!sock->recv_data(&cmd, 1)) { break; } if (cmd >= RPC_CMD_COUNT) { // fail fast if the command is invalid - GGML_LOG_ERROR("Unknown command: %d\n", cmd); + LOG_ERROR("Unknown command: %d\n", cmd); break; } + LOG_DBG2("[%s] %s\n", __func__, rpc_cmd_name((enum rpc_cmd)cmd)); switch (cmd) { case RPC_CMD_HELLO: { // HELLO command is handled above @@ -1984,11 +2824,35 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const break; } case RPC_CMD_SET_TENSOR: { + uint64_t input_size; + if (!sock->recv_data(&input_size, sizeof(input_size))) { + return; + } + if (!server.set_tensor(sock, input_size)) { + return; + } + break; + } + case RPC_CMD_SET_TENSOR_2D: { std::vector<uint8_t> input; if (!recv_msg(sock, input)) { return; } - if (!server.set_tensor(input)) { + if (!server.set_tensor_2d(input)) { + return; + } + break; + } + case RPC_CMD_GET_TENSOR_2D: { + rpc_msg_get_tensor_2d_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + std::vector<uint8_t> response; + if (!server.get_tensor_2d(request, response)) { + return; + } + if (!send_msg(sock, response.data(), response.size())) { return; } break; @@ -2065,6 +2929,40 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const } break; } + case RPC_CMD_COMM_INIT: { + rpc_msg_comm_init_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + rpc_msg_comm_init_rsp response; + if (!server.comm_init(request, response)) { + return; + } + if (!send_msg(sock, &response, sizeof(response))) { + return; + } + break; + } + case RPC_CMD_COMM_ALLREDUCE: { + rpc_msg_comm_allreduce_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + if (!server.comm_allreduce(request)) { + return; + } + break; + } + case RPC_CMD_COMM_FREE: { + rpc_msg_comm_free_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + if (!server.comm_free(request)) { + return; + } + break; + } case RPC_CMD_GET_DEVICE_MEMORY: { rpc_msg_get_device_memory_req request; if (!recv_msg(sock, &request, sizeof(request))) { @@ -2080,7 +2978,7 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const break; } default: { - GGML_LOG_ERROR("Unknown command: %d\n", cmd); + LOG_ERROR("Unhandled command: %s\n", rpc_cmd_name((enum rpc_cmd)cmd)); return; } } @@ -2090,26 +2988,26 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir, size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices) { if (n_devices == 0 || devices == nullptr) { - fprintf(stderr, "Invalid arguments to ggml_backend_rpc_start_server\n"); + LOG_ERROR("Invalid arguments to ggml_backend_rpc_start_server\n"); return; } std::vector<ggml_backend_t> backends; - printf("Starting RPC server v%d.%d.%d\n", + LOG_INFO("Starting RPC server v%d.%d.%d\n", RPC_PROTO_MAJOR_VERSION, RPC_PROTO_MINOR_VERSION, RPC_PROTO_PATCH_VERSION); - printf(" endpoint : %s\n", endpoint); - printf(" local cache : %s\n", cache_dir ? cache_dir : "n/a"); - printf("Devices:\n"); + LOG_INFO(" endpoint : %s\n", endpoint); + LOG_INFO(" local cache : %s\n", cache_dir ? cache_dir : "n/a"); + LOG_INFO("Devices:\n"); for (size_t i = 0; i < n_devices; i++) { auto dev = devices[i]; size_t free, total; ggml_backend_dev_memory(dev, &free, &total); - printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), - total / 1024 / 1024, free / 1024 / 1024); + LOG_INFO(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), + total / 1024 / 1024, free / 1024 / 1024); auto backend = ggml_backend_dev_init(dev, nullptr); if (!backend) { - fprintf(stderr, "Failed to create backend for device %s\n", dev->iface.get_name(dev)); + LOG_ERROR("Failed to create backend for device %s\n", dev->iface.get_name(dev)); return; } backends.push_back(backend); @@ -2129,30 +3027,28 @@ void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir } #ifdef GGML_RPC_RDMA - printf(" transport : TCP (RDMA auto-negotiate enabled)\n"); + LOG_INFO(" transport : TCP (RDMA auto-negotiate enabled)\n"); #else - printf(" transport : TCP\n"); + LOG_INFO(" transport : TCP\n"); #endif // GGML_RPC_RDMA if (!rpc_transport_init()) { - fprintf(stderr, "Failed to initialize RPC transport\n"); + LOG_ERROR("Failed to initialize RPC transport\n"); return; } auto server_socket = socket_t::create_server(host.c_str(), port); if (server_socket == nullptr) { - fprintf(stderr, "Failed to create server socket\n"); + LOG_ERROR("Failed to create server socket\n"); return; } while (true) { auto client_socket = server_socket->accept(); if (client_socket == nullptr) { - fprintf(stderr, "Failed to accept client connection\n"); + LOG_ERROR("Failed to accept client connection\n"); return; } - printf("Accepted client connection\n"); - fflush(stdout); + LOG_INFO("Accepted client connection\n"); rpc_serve_client(backends, cache_dir, client_socket); - printf("Client connection closed\n"); - fflush(stdout); + LOG_INFO("Client connection closed\n"); } rpc_transport_shutdown(); for (auto backend : backends) { @@ -2234,18 +3130,27 @@ static bool ggml_backend_rpc_device_supports_buft(ggml_backend_dev_t dev, ggml_b static ggml_backend_event_t ggml_backend_rpc_device_event_new(ggml_backend_dev_t dev) { ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; auto dispatcher = get_dispatcher(ctx->endpoint); + if (dispatcher == nullptr) { + return nullptr; + } return dispatcher->event_new(dev); } static void ggml_backend_rpc_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; auto dispatcher = get_dispatcher(ctx->endpoint); + if (dispatcher == nullptr) { + return; + } dispatcher->event_free(event); } static void ggml_backend_rpc_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) { ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; auto dispatcher = get_dispatcher(ctx->endpoint); + if (dispatcher == nullptr) { + return; + } dispatcher->event_synchronize(event); } @@ -2294,6 +3199,137 @@ static ggml_backend_dev_t ggml_backend_rpc_reg_get_device(ggml_backend_reg_t reg } } +// Pairwise allreduce between two RPC servers over a direct server-to-server connection. +// The client only sends fire-and-forget COMM_ALLREDUCE commands; the tensor data is +// exchanged between the servers and never passes through the client. +struct ggml_backend_rpc_comm_context { + struct rank_info { + std::string endpoint; + uint32_t device; + std::shared_ptr<rpc_dispatcher> dispatcher; + }; + std::vector<rank_info> ranks; +}; + +static void ggml_backend_rpc_comm_free(void * comm_ctx_v) { + ggml_backend_rpc_comm_context * comm_ctx = (ggml_backend_rpc_comm_context *) comm_ctx_v; + if (comm_ctx == nullptr) { + return; + } + for (const auto & rank : comm_ctx->ranks) { + auto request = std::make_shared<rpc_msg_comm_free_req>(); + request->device = rank.device; + rank.dispatcher->send(RPC_CMD_COMM_FREE, request, sizeof(*request), nullptr, 0, nullptr, 0); + rank.dispatcher->busy_spin_release(); + } + delete comm_ctx; +} + +static void * ggml_backend_rpc_comm_init(ggml_backend_t * backends, size_t n_backends) { + if (n_backends < 2 || std::getenv("GGML_RPC_NO_COMM") != nullptr) { + return nullptr; + } + std::vector<ggml_backend_rpc_comm_context::rank_info> ranks; + ranks.reserve(n_backends); + for (size_t i = 0; i < n_backends; i++) { + if (!ggml_backend_is_rpc(backends[i])) { + return nullptr; + } + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *) backends[i]->context; + // one rank per endpoint: a server processes its socket sequentially, so a second + // COMM_INIT on the same connection would deadlock behind the first + for (const auto & rank : ranks) { + if (rank.endpoint == rpc_ctx->endpoint) { + LOG_WARN("%s: multiple ranks on endpoint %s are not supported\n", __func__, rpc_ctx->endpoint.c_str()); + return nullptr; + } + } + ranks.push_back({rpc_ctx->endpoint, rpc_ctx->device, rpc_ctx->dispatcher}); + } + + // TODO: simplify this logic + // rank 1 connects to rank 0 on its serving host; endpoints must be mutually reachable + // (e.g. do not bind the servers to 127.0.0.1 when they run on different machines) + std::string host0; + int port0; + if (!parse_endpoint(ranks[0].endpoint, host0, port0)) { + return nullptr; + } + const uint32_t comm_port = (uint32_t) port0 + 1000; + if (host0.size() >= 64) { + return nullptr; + } + + for (const auto & rank : ranks) { + rank.dispatcher->busy_spin_acquire(); + } + + // Send all init requests before reading any response: rank 0 blocks in accept + // until rank 1 has connected. + std::vector<rpc_msg_comm_init_rsp> responses(n_backends); + for (size_t i = 0; i < n_backends; i++) { + auto request = std::make_shared<rpc_msg_comm_init_req>(); + request->device = ranks[i].device; + request->rank = (uint32_t) i; + request->world = (uint32_t) n_backends; + request->port = comm_port; + if (i > 0) { + memcpy(request->host, host0.c_str(), host0.size()); + } + ranks[i].dispatcher->send_async(RPC_CMD_COMM_INIT, request, sizeof(*request), &responses[i], sizeof(responses[i])); + } + for (size_t i = 0; i < n_backends; i++) { + ranks[i].dispatcher->synchronize(); + } + bool ok = true; + for (size_t i = 0; i < n_backends; i++) { + if (!responses[i].ok) { + LOG_WARN("%s: rank %zu (%s) failed to initialize\n", __func__, i, ranks[i].endpoint.c_str()); + ok = false; + } + } + if (!ok) { + for (const auto & rank : ranks) { + rank.dispatcher->busy_spin_release(); + } + return nullptr; + } + LOG_INFO("%s: pairwise communicator initialized (%s <-> %s)\n", __func__, + ranks[0].endpoint.c_str(), ranks[1].endpoint.c_str()); + return new ggml_backend_rpc_comm_context{std::move(ranks)}; +} + +static bool ggml_backend_rpc_comm_allreduce_tensor(void * comm_ctx_v, ggml_tensor ** tensors) { + ggml_backend_rpc_comm_context * comm_ctx = (ggml_backend_rpc_comm_context *) comm_ctx_v; + if (comm_ctx == nullptr) { + return false; + } + const size_t n_ranks = comm_ctx->ranks.size(); + const int64_t ne = ggml_nelements(tensors[0]); + if (ne == 0) { + return true; + } + for (size_t i = 0; i < n_ranks; i++) { + if (tensors[i] == nullptr || tensors[i]->type != GGML_TYPE_F32 || ggml_nelements(tensors[i]) != ne || + !ggml_is_contiguously_allocated(tensors[i]) || + tensors[i]->buffer == nullptr || !ggml_backend_buffer_is_rpc(tensors[i]->buffer)) { + return false; + } + // a rank with a disabled node has garbage in its partial and must contribute zeros, + // which only the fallback path handles + if ((tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + return false; + } + } + for (size_t i = 0; i < n_ranks; i++) { + auto request = std::make_shared<rpc_msg_comm_allreduce_req>(); + request->device = comm_ctx->ranks[i].device; + request->tensor = serialize_tensor(tensors[i]); + comm_ctx->ranks[i].dispatcher->send_async(RPC_CMD_COMM_ALLREDUCE, request, sizeof(*request)); + } + return true; +} + static void * ggml_backend_rpc_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (std::strcmp(name, "ggml_backend_rpc_add_server") == 0) { return (void *)ggml_backend_rpc_add_server; @@ -2301,6 +3337,15 @@ static void * ggml_backend_rpc_get_proc_address(ggml_backend_reg_t reg, const ch if (std::strcmp(name, "ggml_backend_rpc_start_server") == 0) { return (void *)ggml_backend_rpc_start_server; } + if (std::strcmp(name, "ggml_backend_comm_init") == 0) { + return (void *)ggml_backend_rpc_comm_init; + } + if (std::strcmp(name, "ggml_backend_comm_free") == 0) { + return (void *)ggml_backend_rpc_comm_free; + } + if (std::strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { + return (void *)ggml_backend_rpc_comm_allreduce_tensor; + } return NULL; GGML_UNUSED(reg); @@ -2325,8 +3370,11 @@ ggml_backend_reg_t ggml_backend_rpc_reg(void) { static uint32_t ggml_backend_rpc_get_device_count(const char * endpoint) { auto dispatcher = get_dispatcher(endpoint); + if (dispatcher == nullptr) { + return 0; + } rpc_msg_device_count_rsp response; - dispatcher->send(RPC_CMD_DEVICE_COUNT, nullptr, 0, &response, sizeof(response)); + dispatcher->send(RPC_CMD_DEVICE_COUNT, nullptr, 0, nullptr, 0, &response, sizeof(response)); return response.device_count; } @@ -2343,9 +3391,11 @@ ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint) { static uint32_t dev_id = 0; std::lock_guard<std::mutex> lock(mutex); if (reg_map.find(endpoint) != reg_map.end()) { + LOG_DBG3("[%s] endpoint %s already registered\n", __func__, endpoint); return reg_map[endpoint]; } uint32_t dev_count = ggml_backend_rpc_get_device_count(endpoint); + LOG_DBG2("[%s] endpoint: %s, device count: %u\n", __func__, endpoint, dev_count); if (dev_count == 0) { return nullptr; } @@ -2359,7 +3409,6 @@ ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint) { /* .device = */ ind, /* .name = */ dev_name, /* .description = */ dev_desc, - /* .last_graph_uid = */ 0, }; ggml_backend_dev_t dev = new ggml_backend_device { diff --git a/ggml/src/ggml-rpc/log.cpp b/ggml/src/ggml-rpc/log.cpp new file mode 100644 index 000000000..89f0ce705 --- /dev/null +++ b/ggml/src/ggml-rpc/log.cpp @@ -0,0 +1,34 @@ +#include "log.h" + +#include <cinttypes> +#include <cstdio> + +std::string rpc_log_format_size(size_t size) { + static const char * units[] = { "B", "KiB", "MiB", "GiB", "TiB", "PiB" }; + + double value = (double) size; + size_t unit = 0; + while (value >= 1024.0 && unit + 1 < sizeof(units) / sizeof(units[0])) { + value /= 1024.0; + unit++; + } + + char buf[32]; + if (unit == 0) { + snprintf(buf, sizeof(buf), "%zu B", size); + } else { + snprintf(buf, sizeof(buf), "%.2f %s", value, units[unit]); + } + return std::string(buf); +} + +std::string rpc_log_format_tensor(const ggml_tensor * tensor) { + char shape[64]; + snprintf(shape, sizeof(shape), "[%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "]", tensor->ne[0], tensor->ne[1], + tensor->ne[2], tensor->ne[3]); + + char buf[GGML_MAX_NAME + 192]; + snprintf(buf, sizeof(buf), "%s [%s, %s, %s]", ggml_get_name(tensor), ggml_op_name(tensor->op), + ggml_type_name(tensor->type), shape); + return std::string(buf); +} diff --git a/ggml/src/ggml-rpc/log.h b/ggml/src/ggml-rpc/log.h new file mode 100644 index 000000000..96a383d77 --- /dev/null +++ b/ggml/src/ggml-rpc/log.h @@ -0,0 +1,49 @@ +#pragma once + +// Log infrastructure for the RPC backend: LOG_* are unconditional, LOG_DBG* are gated by GGML_RPC_DEBUG + +#include "ggml-impl.h" + +#include <cstddef> +#include <cstdlib> +#include <string> + +// GGML_RPC_DEBUG verbosity: unset / 0 disabled, 1 connections and allocations, 2 per-command trace, +// 3 per-message detail. Non-numeric values fall back to 1 +static inline int rpc_log_debug_level() { + static const int level = [] { + const char * env = std::getenv("GGML_RPC_DEBUG"); + if (env == nullptr) { + return 0; + } + if (env[0] == '\0') { + return 1; + } + int value = 0; + for (const char * p = env; *p != '\0'; p++) { + if (*p < '0' || *p > '9') { + return 1; + } + value = value * 10 + (*p - '0'); + if (value > 3) { + value = 3; + } + } + return value; + }(); + return level; +} + +// Byte counts in binary units, e.g. 8208 -> "8.02 KiB". Sizes below 1 KiB stay in plain bytes +std::string rpc_log_format_size(size_t size); + +// Tensor identity: "blk.0.attn_q.weight [NONE, f16, [4096, 4096, 1, 1]]" +std::string rpc_log_format_tensor(const ggml_tensor * tensor); + +#define LOG_INFO(...) GGML_LOG_INFO(__VA_ARGS__) +#define LOG_WARN(...) GGML_LOG_WARN(__VA_ARGS__) +#define LOG_ERROR(...) GGML_LOG_ERROR(__VA_ARGS__) + +#define LOG_DBG(...) do { if (rpc_log_debug_level() >= 1) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) +#define LOG_DBG2(...) do { if (rpc_log_debug_level() >= 2) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) +#define LOG_DBG3(...) do { if (rpc_log_debug_level() >= 3) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) diff --git a/ggml/src/ggml-rpc/transport-apple.cpp b/ggml/src/ggml-rpc/transport-apple.cpp index bb24a5d4c..18c4e7efc 100644 --- a/ggml/src/ggml-rpc/transport-apple.cpp +++ b/ggml/src/ggml-rpc/transport-apple.cpp @@ -1,6 +1,6 @@ #include "transport-apple.h" #include "transport.h" -#include "ggml-impl.h" +#include "log.h" #include <infiniband/verbs.h> @@ -199,6 +199,7 @@ static bool rdma_library_present() { // address, i.e. the one cabled to the peer. std::unique_ptr<apple_rdma> apple_rdma::probe(int fd, const uint8_t * target_gid, uint8_t * caps) { if (!rdma_library_present()) { + LOG_DBG2("[RDMA(Apple)] librdma.dylib not present, continuing with TCP\n"); return nullptr; } int ndev = 0; @@ -224,7 +225,10 @@ std::unique_ptr<apple_rdma> apple_rdma::probe(int fd, const uint8_t * target_gid break; } ibv_free_device_list(devs); - if (!ctx) return nullptr; + if (!ctx) { + LOG_DBG2("[RDMA(Apple)] no RDMA device matched the connection address, continuing with TCP\n"); + return nullptr; + } std::unique_ptr<impl> c(new impl()); c->fd = fd; @@ -286,7 +290,7 @@ std::unique_ptr<apple_rdma> apple_rdma::probe(int fd, const uint8_t * target_gid memcpy(rc.gid, gid.raw, RDMA_GID_SIZE); memcpy(caps, &rc, sizeof(rc)); - GGML_LOG_INFO("RDMA(Apple/UC) probed: dev=%s port=%u gid=%d qpn=%u lid=%u mtu=%d ring=%d x %zu KiB\n", + LOG_INFO("RDMA(Apple/UC) probed: dev=%s port=%u gid=%d qpn=%u lid=%u mtu=%d ring=%d x %zu KiB\n", matched.c_str(), port, gid_idx, c->qpn, (unsigned)pa.lid, 128 << c->path_mtu, RDMA_NBUF, RDMA_STRIDE / 1024); return std::unique_ptr<apple_rdma>(new apple_rdma(std::move(c))); @@ -317,7 +321,7 @@ bool apple_rdma::activate(const uint8_t * caps) { memcpy(&a.ah_attr.grh.dgid, rc.gid, RDMA_GID_SIZE); if (ibv_modify_qp(c->qp, &a, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN) != 0) { - GGML_LOG_ERROR("RDMA(Apple/UC) RTR failed: %s\n", strerror(errno)); + LOG_ERROR("RDMA(Apple/UC) RTR failed: %s\n", strerror(errno)); ok = false; } } @@ -326,7 +330,7 @@ bool apple_rdma::activate(const uint8_t * caps) { a.qp_state = IBV_QPS_RTS; a.sq_psn = RDMA_PSN; if (ibv_modify_qp(c->qp, &a, IBV_QP_STATE | IBV_QP_SQ_PSN) != 0) { - GGML_LOG_ERROR("RDMA(Apple/UC) RTS failed: %s\n", strerror(errno)); + LOG_ERROR("RDMA(Apple/UC) RTS failed: %s\n", strerror(errno)); ok = false; } } @@ -334,7 +338,7 @@ bool apple_rdma::activate(const uint8_t * caps) { // Recvs are posted only now: the controller starts processing them at RTR. for (int i = 0; ok && i < RDMA_NBUF; i++) { if (!c->post_recv(i)) { - GGML_LOG_ERROR("RDMA(Apple/UC) post_recv %d/%d failed\n", i, RDMA_NBUF); + LOG_ERROR("RDMA(Apple/UC) post_recv %d/%d failed\n", i, RDMA_NBUF); ok = false; } } @@ -350,7 +354,7 @@ bool apple_rdma::activate(const uint8_t * caps) { return false; } - GGML_LOG_INFO("RDMA(Apple/UC) activated: qpn=%u->%u mtu=%d rx_depth=%d\n", + LOG_INFO("RDMA(Apple/UC) activated: qpn=%u->%u mtu=%d rx_depth=%d\n", c->qpn, rc.qpn, 128 << c->path_mtu, RDMA_NBUF); return true; } @@ -360,20 +364,20 @@ bool apple_rdma::activate(const uint8_t * caps) { int apple_rdma::impl::progress() { struct ibv_wc wc[RDMA_NBUF * 2]; int n = ibv_poll_cq(cq, RDMA_NBUF * 2, wc); - if (n < 0) { GGML_LOG_ERROR("RDMA(Apple/UC) poll_cq failed\n"); broken = true; return -1; } + if (n < 0) { LOG_ERROR("RDMA(Apple/UC) poll_cq failed\n"); broken = true; return -1; } for (int j = 0; j < n; j++) { uint64_t id = wc[j].wr_id; bool is_recv = (id & RDMA_RECV_WR) != 0; if (wc[j].status != IBV_WC_SUCCESS) { - GGML_LOG_ERROR("RDMA(Apple/UC) %s wc error: status=%d\n", is_recv ? "recv" : "send", wc[j].status); + LOG_ERROR("RDMA(Apple/UC) %s wc error: status=%d\n", is_recv ? "recv" : "send", wc[j].status); broken = true; return -1; } if (is_recv) { int b = (int)(id & RDMA_WR_IDX_MASK); const rdma_seg_hdr * h = (const rdma_seg_hdr *)(recv_mem + (size_t)b * RDMA_STRIDE); - if (h->magic != RDMA_SEG_MAGIC) { GGML_LOG_ERROR("RDMA(Apple/UC) bad frame magic\n"); broken = true; return -1; } - if (h->len > RDMA_PAYLOAD) { GGML_LOG_ERROR("RDMA(Apple/UC) frame len %u exceeds payload\n", h->len); broken = true; return -1; } + if (h->magic != RDMA_SEG_MAGIC) { LOG_ERROR("RDMA(Apple/UC) bad frame magic\n"); broken = true; return -1; } + if (h->len > RDMA_PAYLOAD) { LOG_ERROR("RDMA(Apple/UC) frame len %u exceeds payload\n", h->len); broken = true; return -1; } int slot = (inq_head + inq_count) % RDMA_NBUF; inq[slot].buf = b; inq[slot].off = 0; diff --git a/ggml/src/ggml-rpc/transport.cpp b/ggml/src/ggml-rpc/transport.cpp index b28d16605..c7d9f5321 100644 --- a/ggml/src/ggml-rpc/transport.cpp +++ b/ggml/src/ggml-rpc/transport.cpp @@ -1,5 +1,5 @@ #include "transport.h" -#include "ggml-impl.h" +#include "log.h" #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -43,11 +43,6 @@ using ssize_t = __int64; typedef int sockfd_t; #endif -static const char * RPC_DEBUG = std::getenv("GGML_RPC_DEBUG"); - -#define LOG_DBG(...) \ - do { if (RPC_DEBUG) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) - #ifdef GGML_RPC_RDMA static constexpr size_t RDMA_GID_SIZE = 16; // RoCE GID / IB GID is always 16 bytes using rdma_gid_t = std::array<uint8_t, RDMA_GID_SIZE>; @@ -76,6 +71,11 @@ struct rdma_conn { struct ibv_mr * rx_mr = nullptr; int rx_head = 0; + // received slot that is only partly consumed, so recv sizes need not match send sizes + int rx_cur = -1; + size_t rx_off = 0; + size_t rx_len = 0; + uint32_t max_inline = 0; uint8_t * rx_slot(int i) const { @@ -342,7 +342,7 @@ bool socket_t::impl::rdma_probe() { } else if (gid_version == IBV_GID_TYPE_ROCE_V1) { ver_str = " RoCEv1"; } - GGML_LOG_INFO("RDMA probed: dev=%s gid=%d%s qpn=%u inline=%u\n", + LOG_INFO("RDMA probed: dev=%s gid=%d%s qpn=%u inline=%u\n", matched_dev, gid_idx, ver_str, rdma_local.qpn, rdma->max_inline); return true; } @@ -407,7 +407,7 @@ bool socket_t::impl::rdma_activate(uint32_t remote_qpn, uint32_t remote_psn, con rdma->last_active = std::chrono::steady_clock::now(); - GGML_LOG_INFO("RDMA activated: qpn=%u->%u mtu=%d rx_depth=%d\n", + LOG_INFO("RDMA activated: qpn=%u->%u mtu=%d rx_depth=%d\n", rdma_local.qpn, remote_qpn, 128 << rdma_local.path_mtu, RDMA_RX_DEPTH); return true; } @@ -445,7 +445,7 @@ bool socket_t::impl::rdma_poll(struct ibv_cq * cq, struct ibv_wc * wc) { if (n > 0) { c->last_active = std::chrono::steady_clock::now(); if (wc->status != IBV_WC_SUCCESS) { - GGML_LOG_ERROR("RDMA CQ wc error: status=%d (%s) vendor_err=0x%x\n", + LOG_ERROR("RDMA CQ wc error: status=%d (%s) vendor_err=0x%x\n", wc->status, ibv_wc_status_str(wc->status), wc->vendor_err); } return wc->status == IBV_WC_SUCCESS; @@ -517,14 +517,24 @@ bool socket_t::impl::rdma_recv(void * data, size_t size) { uint8_t * dst = (uint8_t *)data; size_t rem = size; while (rem > 0) { - struct ibv_wc wc; - if (!rdma_poll(c->rcq, &wc)) return false; + if (c->rx_cur < 0) { + struct ibv_wc wc; + if (!rdma_poll(c->rcq, &wc)) return false; - int slot = (int)wc.wr_id; - size_t got = wc.byte_len; - memcpy(dst, c->rx_slot(slot), got); + c->rx_cur = (int)wc.wr_id; + c->rx_off = 0; + c->rx_len = wc.byte_len; + } - if (!c->post_rx(slot)) return false; + size_t got = std::min(rem, c->rx_len - c->rx_off); + memcpy(dst, c->rx_slot(c->rx_cur) + c->rx_off, got); + c->rx_off += got; + + // repost the slot only once it is fully consumed + if (c->rx_off == c->rx_len) { + if (!c->post_rx(c->rx_cur)) return false; + c->rx_cur = -1; + } dst += got; rem -= got; @@ -537,6 +547,7 @@ bool socket_t::impl::rdma_recv(void * data, size_t size) { #endif // GGML_RPC_RDMA bool socket_t::impl::send_data(const void * data, size_t size) { + LOG_DBG3("[%s] transport: %s, size: %s\n", __func__, use_rdma ? "RDMA" : "TCP", rpc_log_format_size(size).c_str()); #ifdef GGML_RPC_RDMA_APPLE if (use_rdma) { return rdma->send(data, size); @@ -551,7 +562,7 @@ bool socket_t::impl::send_data(const void * data, size_t size) { size_t size_to_send = std::min(size - bytes_sent, MAX_CHUNK_SIZE); ssize_t n = send(fd, (const char *)data + bytes_sent, size_to_send, 0); if (n < 0) { - GGML_LOG_ERROR("send failed (bytes_sent=%zu, size_to_send=%zu)\n", + LOG_ERROR("send failed (bytes_sent=%zu, size_to_send=%zu)\n", bytes_sent, size_to_send); return false; } @@ -561,6 +572,7 @@ bool socket_t::impl::send_data(const void * data, size_t size) { } bool socket_t::impl::recv_data(void * data, size_t size) { + LOG_DBG3("[%s] transport: %s, size: %s\n", __func__, use_rdma ? "RDMA" : "TCP", rpc_log_format_size(size).c_str()); #ifdef GGML_RPC_RDMA_APPLE if (use_rdma) { return rdma->recv(data, size); @@ -575,12 +587,12 @@ bool socket_t::impl::recv_data(void * data, size_t size) { size_t size_to_recv = std::min(size - bytes_recv, MAX_CHUNK_SIZE); ssize_t n = recv(fd, (char *)data + bytes_recv, size_to_recv, 0); if (n < 0) { - GGML_LOG_ERROR("recv failed (bytes_recv=%zu, size_to_recv=%zu)\n", + LOG_ERROR("recv failed (bytes_recv=%zu, size_to_recv=%zu)\n", bytes_recv, size_to_recv); return false; } if (n == 0) { - LOG_DBG("recv returned 0 (peer closed?)\n"); + LOG_WARN("recv returned 0 (peer closed?)\n"); return false; } bytes_recv += (size_t)n; @@ -599,6 +611,9 @@ void socket_t::impl::get_caps(uint8_t * local_caps) { if (target_gid) { rdma = apple_rdma::probe(fd, target_gid->data(), local_caps); } + if (!rdma) { + LOG_DBG2("[%s] RDMA probe failed, continuing with TCP\n", __func__); + } # else rdma_local = {}; if (rdma_probe()) { @@ -609,6 +624,7 @@ void socket_t::impl::get_caps(uint8_t * local_caps) { memcpy(local_caps, &rc, sizeof(rc)); } else { rdma.reset(); + LOG_DBG2("[%s] RDMA probe failed, continuing with TCP\n", __func__); } # endif #endif // GGML_RPC_RDMA @@ -623,6 +639,9 @@ void socket_t::impl::update_caps(const uint8_t * remote_caps) { remote_rdma |= remote_caps[i] != 0; } if (!rdma || !remote_rdma) { + if (rdma && !remote_rdma) { + LOG_WARN("[%s] peer does not support RDMA, continuing with TCP\n", __func__); + } rdma.reset(); return; } @@ -636,7 +655,7 @@ void socket_t::impl::update_caps(const uint8_t * remote_caps) { if (activated) { use_rdma = true; } else { - GGML_LOG_ERROR("RDMA activate failed, staying on TCP\n"); + LOG_WARN("RDMA activate failed, staying on TCP\n"); rdma.reset(); } #else @@ -679,6 +698,10 @@ void socket_t::update_caps(const uint8_t * remote_caps) { return pimpl->update_caps(remote_caps); } +const char * socket_t::transport_name() const { + return pimpl->use_rdma ? "RDMA" : "TCP"; +} + static bool is_valid_fd(sockfd_t sockfd) { #ifdef _WIN32 return sockfd != INVALID_SOCKET; @@ -706,9 +729,10 @@ socket_ptr socket_t::accept() { return nullptr; } if (!set_no_delay(client_socket_fd)) { - GGML_LOG_ERROR("Failed to set TCP_NODELAY\n"); + LOG_ERROR("Failed to set TCP_NODELAY\n"); return nullptr; } + LOG_DBG("[%s] accepted client connection\n", __func__); return socket_ptr(new socket_t(std::make_unique<impl>(client_socket_fd))); } @@ -718,11 +742,11 @@ socket_ptr socket_t::create_server(const char * host, int port) { return nullptr; } if (!set_reuse_addr(sockfd)) { - GGML_LOG_ERROR("Failed to set SO_REUSEADDR\n"); + LOG_ERROR("Failed to set SO_REUSEADDR\n"); return nullptr; } if (inet_addr(host) == INADDR_NONE) { - GGML_LOG_ERROR("Invalid host address: %s\n", host); + LOG_ERROR("Invalid host address: %s\n", host); return nullptr; } struct sockaddr_in serv_addr; @@ -736,6 +760,7 @@ socket_ptr socket_t::create_server(const char * host, int port) { if (listen(sockfd, 1) < 0) { return nullptr; } + LOG_DBG("[%s] listening on %s:%d\n", __func__, host, port); return socket_ptr(new socket_t(std::make_unique<impl>(sockfd))); } @@ -745,7 +770,7 @@ socket_ptr socket_t::connect(const char * host, int port) { return nullptr; } if (!set_no_delay(sockfd)) { - GGML_LOG_ERROR("Failed to set TCP_NODELAY\n"); + LOG_ERROR("Failed to set TCP_NODELAY\n"); return nullptr; } struct sockaddr_in addr; @@ -753,13 +778,14 @@ socket_ptr socket_t::connect(const char * host, int port) { addr.sin_port = htons(port); struct hostent * server = gethostbyname(host); if (server == NULL) { - GGML_LOG_ERROR("Cannot resolve host '%s'\n", host); + LOG_ERROR("Cannot resolve host '%s'\n", host); return nullptr; } memcpy(&addr.sin_addr.s_addr, server->h_addr, server->h_length); if (::connect(sockfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { return nullptr; } + LOG_DBG("[%s] connected to %s:%d\n", __func__, host, port); return socket_ptr(new socket_t(std::make_unique<impl>(sockfd))); } diff --git a/ggml/src/ggml-rpc/transport.h b/ggml/src/ggml-rpc/transport.h index 3f747ecff..59b8845c0 100644 --- a/ggml/src/ggml-rpc/transport.h +++ b/ggml/src/ggml-rpc/transport.h @@ -24,6 +24,7 @@ struct socket_t { void get_caps(uint8_t * local_caps); void update_caps(const uint8_t * remote_caps); + const char * transport_name() const; static socket_ptr create_server(const char * host, int port); static socket_ptr connect(const char * host, int port); diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index d2196f74d..8312a8305 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -44,7 +44,7 @@ if (WIN32) if(DEFINED ENV{LEVEL_ZERO_V1_SDK_PATH}) set(LEVEL_ZERO_V1_SDK_PATH $ENV{LEVEL_ZERO_V1_SDK_PATH}) if(EXISTS "${LEVEL_ZERO_V1_SDK_PATH}") - target_include_directories(ggml-sycl PRIVATE "${LEVEL_ZERO_V1_SDK_PATH}/include") + target_include_directories(ggml-sycl SYSTEM PRIVATE "${LEVEL_ZERO_V1_SDK_PATH}/include") set(LEVEL_ZERO_V1_SDK_LIB_PATH "${LEVEL_ZERO_V1_SDK_PATH}/lib") else() message(WARNING "LEVEL_ZERO_V1_SDK_PATH set but folder not found: ${LEVEL_ZERO_V1_SDK_PATH}") @@ -107,57 +107,101 @@ target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing") message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}") if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API) - # Link against Level Zero loader for direct device memory allocation. - # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging - # in the xe kernel driver during multi-GPU inference. - find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) - find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH) - if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR) - target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB}) - target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) - message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}") - message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}") + find_package(PkgConfig) + if (PkgConfig_FOUND) + pkg_check_modules(LEVEL_ZERO level-zero) + if (LEVEL_ZERO_FOUND) + target_include_directories(ggml-sycl PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) + target_link_libraries(ggml-sycl PRIVATE ${LEVEL_ZERO_LINK_LIBRARIES}) + target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + message(STATUS "Level Zero loader found: ${LEVEL_ZERO_LINK_LIBRARIES}") + message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_INCLUDE_DIRS}") + endif() else() - message(WARNING "Level Zero loader or development headers not found, " - "Level Zero API support disabled. " - "Please install the Level Zero SDK/development package " - "to support Level Zero API features. " - "Level Zero API is not mandatory for SYCL backend, " - "but it is required by the special features for better " - "function & performance on Intel GPUs.") + # Link against Level Zero loader for direct device memory allocation. + # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging + # in the xe kernel driver during multi-GPU inference. + find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) + find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH) + if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR) + target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB}) + target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}") + message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}") + else() + message(WARNING "Level Zero loader or development headers not found, " + "Level Zero API support disabled. " + "Please install the Level Zero SDK/development package " + "to support Level Zero API features. " + "Level Zero API is not mandatory for SYCL backend, " + "but it is required by the special features for better " + "function & performance on Intel GPUs.") + endif() endif() endif() # Link against oneDNN set(GGML_SYCL_DNNL 0) if(GGML_SYCL_DNN) - find_package(DNNL) - if(DNNL_FOUND) - if (NOT DEFINED DNNL_GPU_VENDOR) - # default to intel target - set(DNNL_GPU_VENDOR "INTEL") - if(NOT "${GGML_SYCL_TARGET}" STREQUAL "INTEL") - message(WARNING "oneDNN builds bundled with oneapi release only support INTEL target") - endif() + # oneAPI installs dnnl-config.cmake under a versioned path CMake does not + # search by default. When the user did not point CMake at it explicitly + # (DNNL_DIR), auto-detect the oneAPI-bundled GPU build via the DNNLROOT + # environment variable or the standard oneAPI install prefixes. A shell + # with setvars.sh sourced already exports CMAKE_PREFIX_PATH and finds it + # without any of this. + if(NOT DEFINED DNNL_DIR) + if(DEFINED ENV{DNNLROOT} AND EXISTS "$ENV{DNNLROOT}/lib/cmake/dnnl/dnnl-config.cmake") + set(DNNL_DIR "$ENV{DNNLROOT}/lib/cmake/dnnl") + elseif(EXISTS "/opt/intel/oneapi/dnnl/latest/lib/cmake/dnnl/dnnl-config.cmake") + set(DNNL_DIR "/opt/intel/oneapi/dnnl/latest/lib/cmake/dnnl") + elseif(EXISTS "/usr/intel/oneapi/dnnl/latest/lib/cmake/dnnl/dnnl-config.cmake") + set(DNNL_DIR "/usr/intel/oneapi/dnnl/latest/lib/cmake/dnnl") endif() - - # Verify oneDNN was compiled for the same target as llama - if("${GGML_SYCL_TARGET}" STREQUAL "${DNNL_GPU_VENDOR}") - target_link_libraries(ggml-sycl PRIVATE DNNL::dnnl) - set(GGML_SYCL_DNNL 1) - get_target_property(CONFIGS DNNL::dnnl IMPORTED_CONFIGURATIONS) - foreach(CONFIG ${CONFIGS}) - get_target_property(DNNL_LIB DNNL::dnnl IMPORTED_LOCATION_${CONFIG}) - message(STATUS "Found oneDNN: ${DNNL_LIB}") - endforeach() - else() - message(WARNING - "oneDNN must be compiled for the same target as llama.cpp. - llama.cpp: ${GGML_SYCL_TARGET}, oneDNN: ${DNNL_GPU_VENDOR}. - Disabling oneDNN support.") + if(DEFINED DNNL_DIR) + message(STATUS "oneDNN: auto-detected config at ${DNNL_DIR}") endif() + endif() + find_package(DNNL) + if(NOT DNNL_FOUND) + # GGML_SYCL_DNN defaults to ON, so a plain build used to continue + # silently without oneDNN: every flash-attention then runs the much + # slower MKL-FA path (20-40% prefill loss on long prompts). Fail + # loudly instead of producing a degraded binary. + message(FATAL_ERROR + "oneDNN (DNNL) was requested (GGML_SYCL_DNN=ON) but not found. " + "The SYCL backend needs the oneAPI-bundled GPU build: install " + "intel-oneapi-dnnl and intel-oneapi-dnnl-devel from the oneAPI " + "repo, source setvars.sh, or pass " + "-DDNNL_DIR=<dnnl-prefix>/lib/cmake/dnnl (or export DNNLROOT). " + "To build without oneDNN on purpose, pass -DGGML_SYCL_DNN=OFF.") + endif() + if(NOT DEFINED DNNL_GPU_VENDOR) + # A DNNL package that does not declare DNNL_GPU_VENDOR has no GPU/SYCL + # support compiled in (e.g. Debian/Ubuntu's CPU-only libdnnl-dev). + # The old code defaulted the vendor to INTEL here, which fake-passed + # the target check below and silently linked a CPU-only library into + # the SYCL backend. + message(FATAL_ERROR + "Found oneDNN at ${DNNL_DIR} but it does not declare " + "DNNL_GPU_VENDOR - it is a CPU-only build (Debian/Ubuntu " + "libdnnl-dev?). Install the oneAPI-bundled GPU build " + "(intel-oneapi-dnnl-devel) instead.") + endif() + + # Verify oneDNN was compiled for the same target as llama + if("${GGML_SYCL_TARGET}" STREQUAL "${DNNL_GPU_VENDOR}") + target_link_libraries(ggml-sycl PRIVATE DNNL::dnnl) + set(GGML_SYCL_DNNL 1) + get_target_property(CONFIGS DNNL::dnnl IMPORTED_CONFIGURATIONS) + foreach(CONFIG ${CONFIGS}) + get_target_property(DNNL_LIB DNNL::dnnl IMPORTED_LOCATION_${CONFIG}) + message(STATUS "Found oneDNN: ${DNNL_LIB}") + endforeach() else() - message(STATUS "oneDNN not found, disabling oneDNN support") + message(FATAL_ERROR + "oneDNN must be compiled for the same target as llama.cpp: " + "llama.cpp: ${GGML_SYCL_TARGET}, oneDNN: ${DNNL_GPU_VENDOR}. " + "Pass -DGGML_SYCL_DNN=OFF to build without oneDNN.") endif() else() message(STATUS "oneDNN support disabled by the user") @@ -168,12 +212,16 @@ if (GGML_SYCL_F16) add_compile_definitions(GGML_SYCL_F16) endif() +if (GGML_SYCL_SUPPORT_XMX) + add_compile_definitions(GGML_SYCL_SUPPORT_XMX) +endif() + if (GGML_SYCL_TARGET STREQUAL "INTEL") add_compile_definitions(GGML_SYCL_WARP_SIZE=16) if (NOT GGML_SYCL_DEVICE_ARCH) target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required) else() - message(STATUS "Skipping -ze-intel-greater-than-4GB-buffer-required for spir64_gen AOT") + message(STATUS "spir64_gen AOT: -ze-intel-greater-than-4GB-buffer-required passed to ocloc") endif() # Link against Intel oneMKL @@ -203,7 +251,7 @@ if (GGML_SYCL_DEVICE_ARCH) target_compile_options( ggml-sycl PRIVATE -fsycl-targets=spir64_gen - "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\"" + "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH} -options -ze-intel-greater-than-4GB-buffer-required\"" ) # Pass through parallel job (process) count for parallelising the @@ -218,7 +266,30 @@ if (GGML_SYCL_DEVICE_ARCH) target_link_options( ggml-sycl PRIVATE -fsycl-targets=spir64_gen - "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\"" + "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH} -options -ze-intel-greater-than-4GB-buffer-required\"" -fsycl-max-parallel-link-jobs=${GGML_SYCL_MAX_PARALLEL_LINK_JOBS} ) + + # The XMX dequant-GEMM tiles need the sub-group size of the target: 8 on Xe-HPG (DG2, ARL-H), + # 16 on Xe-HPC and Xe2 or newer. ocloc fails on the other size, so build only the one that fits. + # 0 (unknown name, mixed list, or no XMX) builds no XMX tile and the path stays off. + set(_ggml_sycl_xmx_sg "") + string(TOLOWER "${GGML_SYCL_DEVICE_ARCH}" _ggml_sycl_archs) + string(REPLACE "," ";" _ggml_sycl_archs "${_ggml_sycl_archs}") + foreach(_arch IN LISTS _ggml_sycl_archs) + if (_arch MATCHES "^(dg2|acm|ats-m|arl-h|xe-hpg|12\\.5[567]\\.|12\\.74\\.)") + set(_sg 8) + elseif (_arch MATCHES "^(pvc|bmg|lnl|ptl|wcl|nvl|cri|xe2|xe3|xe-hpc|12\\.60\\.|20\\.|30\\.)") + set(_sg 16) + else() + set(_sg 0) + endif() + if (_ggml_sycl_xmx_sg STREQUAL "" OR _ggml_sycl_xmx_sg EQUAL _sg) + set(_ggml_sycl_xmx_sg ${_sg}) + else() + set(_ggml_sycl_xmx_sg 0) + endif() + endforeach() + message(STATUS "GGML_SYCL_DEVICE_ARCH: XMX dequant-GEMM sub-group size ${_ggml_sycl_xmx_sg} (0 = off)") + target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_XMX_AOT_SG=${_ggml_sycl_xmx_sg}) endif() diff --git a/ggml/src/ggml-sycl/add-id.cpp b/ggml/src/ggml-sycl/add-id.cpp index e0adc4fe4..a7901859a 100644 --- a/ggml/src/ggml-sycl/add-id.cpp +++ b/ggml/src/ggml-sycl/add-id.cpp @@ -26,6 +26,15 @@ static void add_id_kernel( float* dst_row = (float*)((char*)dst + i1 * nb1 + i2 * nb2); const float* src0_row = (const float*)((const char*)src0 + i1 * nb01 + i2 * nb02); + + if (i11 == -1) { // skipped slot, add nothing + for (int64_t i0 = item_ct1.get_local_id(2); i0 < ne0; + i0 += item_ct1.get_local_range(2)) { + dst_row[i0] = src0_row[i0]; + } + return; + } + const float* src1_row = (const float*)((const char*)src1 + i11 * nb11); for (int64_t i0 = item_ct1.get_local_id(2); i0 < ne0; diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp index 894006949..44cfede0b 100644 --- a/ggml/src/ggml-sycl/common.cpp +++ b/ggml/src/ggml-sycl/common.cpp @@ -11,6 +11,8 @@ // #include "common.hpp" +#include <mutex> +#include <unordered_map> #include <sycl/backend.hpp> #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API #include <level_zero/ze_api.h> @@ -19,8 +21,14 @@ #include "ggml-backend-impl.h" #include "ggml-impl.h" +thread_local int g_sycl_current_device_id = -1; + int get_current_device_id() { - return dpct::dev_mgr::instance().current_device_id(); + if (g_sycl_current_device_id >= 0) { + return g_sycl_current_device_id; + } + g_sycl_current_device_id = static_cast<int>(dpct::dev_mgr::instance().current_device_id()); + return g_sycl_current_device_id; } void* ggml_sycl_host_malloc(size_t size) try { @@ -92,9 +100,16 @@ static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) { } #endif +// Device bytes allocated through ggml_sycl_malloc_device, per device and per pointer +static std::mutex g_ggml_sycl_dev_alloc_mutex; +static std::unordered_map<void *, std::pair<int, size_t>> g_ggml_sycl_dev_allocs; +static size_t g_ggml_sycl_dev_alloc_bytes[GGML_SYCL_MAX_DEVICES] = {}; +// free memory reported when we had nothing allocated (0 = not taken yet) +static size_t g_ggml_sycl_dev_free_baseline[GGML_SYCL_MAX_DEVICES] = {}; + // Use Level Zero zeMemAllocDevice to avoid sycl::malloc_device triggering // DMA-buf/TTM system RAM staging in the xe kernel driver during multi-GPU inference. -void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) { +static void * ggml_sycl_malloc_device_impl(size_t size, sycl::queue &q, ggml_sycl_mem_type type) { #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (ggml_sycl_use_level_zero_device_alloc(q)) { void *ptr = nullptr; @@ -115,7 +130,8 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t #else ze_device_mem_alloc_desc_t alloc_desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0}; #endif - ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, 64, ze_dev, &ptr); + // request SYCL_BUFFER_ALIGNMENT so tensor offsets aligned to it stay aligned in absolute terms + ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, SYCL_BUFFER_ALIGNMENT, ze_dev, &ptr); if (r == ZE_RESULT_SUCCESS && ptr) { ggml_sycl_memtrace_add(type, ptr, size); return ptr; @@ -133,8 +149,78 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t return ptr; } +// caller holds g_ggml_sycl_dev_alloc_mutex +static bool ggml_sycl_free_mem_enough(int device, size_t size, ggml_sycl_mem_type type) { + const size_t allocated = g_ggml_sycl_dev_alloc_bytes[device]; + + // TODO: remove once zeMemAllocDevice refuses allocations beyond VRAM on BMG + // (https://github.com/intel/compute-runtime/issues/1047). Only model/split buffers are checked. + const gpu_arch arch = ggml_sycl_info().devices[device].hw_info.arch; + const bool is_bmg = arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31; + const bool check = g_ggml_sycl_bmg_vram_check && is_bmg && type == GGML_SYCL_MEM_BUFFER; + const bool trace = type == GGML_SYCL_MEM_BUFFER && ggml_sycl_memtrace_enabled(); + size_t free_mem = 0, total_mem = 0; + if (check || trace) { + sycl_get_mem_info(device, &free_mem, &total_mem); + } + if (trace) { + ggml_sycl_memtrace_report_device("before buffer alloc", device, free_mem, total_mem); + } + if (check) { + size_t & baseline = g_ggml_sycl_dev_free_baseline[device]; + if (allocated == 0) { + baseline = free_mem; + } + // the drop in free memory since the baseline is ours once written, the rest of ours is not counted yet + const size_t written = baseline > free_mem ? baseline - free_mem : 0; + const size_t unwritten = baseline && allocated > written ? allocated - written : 0; + const size_t usable = free_mem > unwritten ? free_mem - unwritten : 0; + if (allocated + size > total_mem || size > usable) { + GGML_LOG_ERROR("%s: SYCL%d: cannot allocate %zu MiB: %zu MiB already allocated, %zu MiB usable " + "(%zu MiB free of %zu MiB), set GGML_SYCL_BMG_VRAM_CHECK=0 to disable this check\n", + __func__, device, size >> 20, allocated >> 20, usable >> 20, free_mem >> 20, total_mem >> 20); + return false; + } + } + return true; +} + +static void ggml_sycl_dev_alloc_record(void * ptr, int device, size_t size) { + g_ggml_sycl_dev_allocs[ptr] = { device, size }; + g_ggml_sycl_dev_alloc_bytes[device] += size; +} + +void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) { + const int device = (int) dpct::dev_mgr::instance().get_device_id(q.get_device()); + GGML_ASSERT(device >= 0 && device < GGML_SYCL_MAX_DEVICES); + + std::lock_guard<std::mutex> lock(g_ggml_sycl_dev_alloc_mutex); + if (!ggml_sycl_free_mem_enough(device, size, type)) { + ggml_sycl_memtrace_fail(type, size); + return nullptr; + } + + void * ptr = ggml_sycl_malloc_device_impl(size, q, type); + if (ptr) { + ggml_sycl_dev_alloc_record(ptr, device, size); + } + return ptr; +} + void ggml_sycl_free_device(void *ptr, sycl::queue &q) { if (!ptr) return; + { + std::lock_guard<std::mutex> lock(g_ggml_sycl_dev_alloc_mutex); + auto it = g_ggml_sycl_dev_allocs.find(ptr); + if (it != g_ggml_sycl_dev_allocs.end()) { + const int device = it->second.first; + g_ggml_sycl_dev_alloc_bytes[device] -= it->second.second; + if (g_ggml_sycl_dev_alloc_bytes[device] == 0) { + g_ggml_sycl_dev_free_baseline[device] = 0; + } + g_ggml_sycl_dev_allocs.erase(it); + } + } ggml_sycl_memtrace_del(ptr); #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (ggml_sycl_use_level_zero_device_alloc(q)) { diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 11fe86cc0..f19b0631d 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -16,7 +16,10 @@ #include <cstddef> #include <fstream> #include <iostream> +#include <optional> #include <string> +#include <type_traits> +#include <unordered_map> #include "base.hpp" #include "dpct/helper.hpp" @@ -26,7 +29,6 @@ #include "presets.hpp" #include "type.hpp" #include "sycl_hw.hpp" -#include "fattn-buffers.hpp" #include "memtrace.hpp" namespace syclexp = sycl::ext::oneapi::experimental; @@ -38,9 +40,14 @@ namespace syclexp = sycl::ext::oneapi::experimental; #endif #endif +#if defined(__INTEL_LLVM_COMPILER) + #define GGML_SYCL_DMMV_HAS_ESIMD +#endif + #if GGML_SYCL_DNNL #include "dnnl.hpp" #include "dnnl_sycl.hpp" +#include "oneapi/dnnl/dnnl_graph.hpp" #endif #define GGML_COMMON_DECL_SYCL @@ -65,14 +72,82 @@ extern int g_ggml_sycl_enable_optimize; extern int g_ggml_sycl_enable_fusion; extern int g_ggml_sycl_enable_esimd; extern int g_ggml_sycl_mmvq_wide; +extern int g_ggml_sycl_enable_xmx_esimd; +extern int g_ggml_sycl_xmx_max_cols; +extern int g_ggml_sycl_xmx_glu_max_cols; extern int g_ggml_sycl_prioritize_dmmv; + +// Which quantized weight formats may take the XMX dequant-GEMM paths. A bitmask rather than one +// flag per path, so a format can be enabled or measured on its own and adding a format is one bit. +enum ggml_sycl_xmx_gather_type { + GGML_SYCL_XMX_GATHER_IQ4_NL = 1 << 0, + GGML_SYCL_XMX_GATHER_IQ3_S = 1 << 1, + GGML_SYCL_XMX_GATHER_IQ4_XS = 1 << 2, + GGML_SYCL_XMX_GATHER_IQ3_XXS = 1 << 3, + GGML_SYCL_XMX_GATHER_IQ2_XXS = 1 << 4, + GGML_SYCL_XMX_GATHER_IQ2_XS = 1 << 5, + GGML_SYCL_XMX_GATHER_IQ2_S = 1 << 6, + GGML_SYCL_XMX_GATHER_IQ1_S = 1 << 7, + GGML_SYCL_XMX_GATHER_IQ1_M = 1 << 8, + GGML_SYCL_XMX_GATHER_Q8_0 = 1 << 9, + GGML_SYCL_XMX_GATHER_Q4_K = 1 << 10, + GGML_SYCL_XMX_GATHER_Q5_K = 1 << 11, + GGML_SYCL_XMX_GATHER_Q6_K = 1 << 12, + GGML_SYCL_XMX_GATHER_Q2_K = 1 << 13, + GGML_SYCL_XMX_GATHER_Q3_K = 1 << 14, +}; +static constexpr int GGML_SYCL_XMX_GATHER_TYPES_DEFAULT = ~0; +extern int g_ggml_sycl_xmx_gather_types; + +enum ggml_sycl_mmid_sched_bit { + GGML_SYCL_MMID_SCHED_DEVICE = 1 << 0, + GGML_SYCL_MMID_SCHED_UNBOUNDED = 1 << 1, +}; +extern int g_ggml_sycl_mmid_sched; +// Which joint_matrix combinations the XMX dequant-GEMM paths may use, one bit each (see fused-gemm.cpp). +// GGML_SYCL_DYNAMIC_PRECISION picks the operand type, this mask the combinations of that type. +static constexpr int GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT = 0xff; +extern int g_ggml_sycl_xmx_gather_shapes; + +// GGML_SYCL_DYNAMIC_PRECISION: operand type of the XMX dequant-GEMM paths. F32 turns them off and +// keeps the library GEMM in f32. A src1 precision request of an op [TAG_GGML_PREC] is always met. +enum ggml_sycl_dynamic_precision { + GGML_SYCL_DYNAMIC_PRECISION_F16, + GGML_SYCL_DYNAMIC_PRECISION_BF16, + GGML_SYCL_DYNAMIC_PRECISION_TF32, + GGML_SYCL_DYNAMIC_PRECISION_F32, +}; +#ifdef GGML_SYCL_F16 +static constexpr int GGML_SYCL_DYNAMIC_PRECISION_DEFAULT = GGML_SYCL_DYNAMIC_PRECISION_F16; +#else +static constexpr int GGML_SYCL_DYNAMIC_PRECISION_DEFAULT = GGML_SYCL_DYNAMIC_PRECISION_F32; +#endif +extern int g_ggml_sycl_dynamic_precision; +// GGML_SYCL_DYNAMIC_REQUIRED_PRECISION: the XMX type an F32 src1 request may run on instead of f32 +// (TF32, or BF16 which also allows tf32). F32 (default): none. F16: src1 requests are ignored. +extern int g_ggml_sycl_dynamic_required_precision; + +// [TAG_GGML_PREC] src1 precision request of the MUL_MAT/MUL_MAT_ID op dst +static inline int32_t ggml_sycl_src1_prec(const ggml_tensor * dst) { + return g_ggml_sycl_dynamic_required_precision == GGML_SYCL_DYNAMIC_PRECISION_F16 ? GGML_PREC_UNDEFINED : + ggml_prec_get_src(dst, 1); +} + +// [TAG_GGML_PREC] the library GEMM and dmmv may convert src1 of the MUL_MAT/MUL_MAT_ID op dst to f16 +static inline bool ggml_sycl_src1_f16_ok(const ggml_tensor * dst) { + const int32_t src1_prec = ggml_sycl_src1_prec(dst); + return g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 && + (src1_prec == GGML_PREC_UNDEFINED || src1_prec >= GGML_PREC_F16); +} extern int g_ggml_sycl_enable_flash_attention; extern int g_ggml_sycl_dev2dev_memcpy; extern int g_ggml_sycl_fa_onednn; extern int g_ggml_sycl_fa_onednn_max_kv; extern int g_ggml_sycl_enable_mkl_fa; +extern int g_ggml_sycl_fa_dpas; extern int g_ggml_sycl_memtrace; extern int g_ggml_sycl_memtrace_step; +extern int g_ggml_sycl_graph_eviction_timeout; #define CHECK_TRY_ERROR(expr) \ @@ -97,12 +172,6 @@ extern int g_ggml_sycl_memtrace_step; #define GGML_SYCL_MAX_NODES 8192 // TODO: adapt to hardwares -// define for XMX in Intel GPU -// TODO: currently, it's not used for XMX really. -#if !defined(GGML_SYCL_FORCE_MMQ) - #define SYCL_USE_XMX -#endif - // max batch size to use MMQ kernels when tensor cores are available #define MMQ_MAX_BATCH_SIZE 32 @@ -169,6 +238,41 @@ typedef sycl::float2 dfloat2; #define MMVQ_MAX_BATCH_SIZE 8 +#if defined(GGML_SYCL_SUPPORT_XMX) && defined(__INTEL_LLVM_COMPILER) && __has_include(<sycl/ext/intel/esimd/xmx/dpas.hpp>) +# define GGML_SYCL_MMVQ_HAS_XMX +#endif // GGML_SYCL_SUPPORT_XMX && __INTEL_LLVM_COMPILER + +// default of the most columns the XMX mul_mat_vec_q handles (GGML_SYCL_XMX_MAX_COLS), wider batches use other kernels +#define GGML_SYCL_XMX_MAX_COLS_DEFAULT 80 + +// most columns the fused gate and up XMX kernel is built for, GGML_SYCL_XMX_GLU_MAX_COLS can only lower this limit +#define GGML_SYCL_XMX_GLU_MAX_COLS 16 + +static inline bool ggml_sycl_xmx_supports_type(ggml_type type) { + switch (type) { + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } +} + +// the DPAS shapes and 2D block loads used by the XMX kernel are those of Xe2 +static inline bool ggml_sycl_xmx_supports_arch(gpu_arch arch) { + return arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31 || + arch == gpu_arch::intel_gpu_lnl_m; +} + +// fewest columns for which the XMX kernel is faster than the other kernels +static inline int ggml_sycl_xmx_min_cols(ggml_type type) { + return type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K ? 1 : 2; +} + static int g_all_sycl_device_count = -1; static bool g_ggml_backend_sycl_buffer_type_initialized = false; @@ -188,6 +292,10 @@ static size_t g_scratch_offset = 0; (void)bad_arch; // suppress unused function warning } +// thread-local copy of dpct's per-thread selected device, -1 until first read. +// Select devices only through ggml_sycl_set_device so the copy stays in sync. +extern thread_local int g_sycl_current_device_id; + int get_current_device_id(); inline int ggml_sycl_get_device() { @@ -195,16 +303,14 @@ inline int ggml_sycl_get_device() { } inline dpct::err0 ggml_sycl_set_device(const int device) try { - int current_device_id; - SYCL_CHECK(CHECK_TRY_ERROR(current_device_id = get_current_device_id())); - - // GGML_SYCL_DEBUG("ggml_sycl_set_device device_id=%d, - // current_device_id=%d\n", device, current_device); - if (device == current_device_id) { + if (device == g_sycl_current_device_id) { return 0; } - - return CHECK_TRY_ERROR(dpct::select_device(device)); + const dpct::err0 err = CHECK_TRY_ERROR(dpct::select_device(device)); + if (err == dpct::success) { + g_sycl_current_device_id = device; + } + return err; } catch (sycl::exception const& exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ << ", line:" << __LINE__ << std::endl; @@ -219,6 +325,7 @@ struct optimize_feature { }; struct sycl_device_info { + bool active; //the device is activated/used in application. int cc; // compute capability int nsm; // number of streaming multiprocessors (CUDA) maps to the maximum // number of compute units on a SYCL device. @@ -235,10 +342,15 @@ struct sycl_device_info { sycl_hw_info hw_info; optimize_feature opt_feature; bool usm_system_support; // support for USM system allocations +#ifdef GGML_SYCL_GRAPH + bool graph_support; // command graphs can be recorded and replayed + bool graph_update_support; // a finalized command graph can be updated +#endif }; struct ggml_sycl_device_info { + int active_device_count; //the number of active devices in the application int device_count; sycl_device_info devices[GGML_SYCL_MAX_DEVICES] = {}; @@ -250,7 +362,11 @@ struct ggml_sycl_device_info { bool ext_oneapi_level_zero = true; // sycl::backend::ext_oneapi_level_zero used by all enumerated GPU devices }; -const ggml_sycl_device_info & ggml_sycl_info(); +ggml_sycl_device_info & ggml_sycl_info(); + +static inline bool ggml_sycl_xmx_enabled(int device) { + return g_ggml_sycl_enable_xmx_esimd && ggml_sycl_xmx_supports_arch(ggml_sycl_info().devices[device].hw_info.arch); +} static constexpr size_t SYCL_BUFFER_ALIGNMENT = 128; @@ -323,6 +439,8 @@ struct ggml_tensor_extra_gpu { }; extern int g_ggml_sycl_use_level_zero_api; +extern int g_ggml_sycl_bmg_vram_check; +bool sycl_get_mem_info(int device, size_t * free, size_t * total); void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type = GGML_SYCL_MEM_DIRECT); void ggml_sycl_free_device(void *ptr, sycl::queue &q); @@ -334,7 +452,49 @@ struct mmid_row_mapping { int32_t i2; }; +struct ggml_sycl_grouped_gemm_tile { + int32_t expert; + int32_t n0; + int32_t n1; +}; + namespace sycl_ex = sycl::ext::oneapi::experimental; + +#ifdef GGML_SYCL_GRAPH +struct ggml_sycl_graph { + // src data/ne/nb are kept next to the node copy: the scheduler can hand back the same src + // pointer with different contents or shape, see https://github.com/ggml-org/llama.cpp/pull/21736 + struct node_properties { + ggml_tensor node; + void * node_src_data_ptrs[GGML_MAX_SRC]; + int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; + size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; + }; + + std::unique_ptr<sycl_ex::command_graph<sycl_ex::graph_state::executable>> exec_graph = nullptr; + std::vector<node_properties> node_props; + bool warmup_complete = false; + uint64_t uid = 0; + int64_t last_used_time = 0; + + // result of check_graph_compatibility() and the uid it was made for + bool compatible = false; + uint64_t compatible_uid = 0; +}; + +static_assert(std::is_trivial<ggml_sycl_graph::node_properties>::value, "node_properties must be trivial"); +#endif + +#if GGML_SYCL_DNNL +struct sdpa_partition { + dnnl::graph::compiled_partition cp; + std::vector<dnnl::graph::logical_tensor> ins; + dnnl::graph::logical_tensor out; + size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0; + bool ok = false; +}; +#endif + struct ggml_backend_sycl_context { int device; std::string name; @@ -403,23 +563,22 @@ struct ggml_backend_sycl_context { dnnl::stream stream_dnnl() { return stream_dnnl(device, 0); } + + std::unordered_map<std::string, sdpa_partition> sdpa_partitions; #endif // pool std::unique_ptr<ggml_sycl_pool> pools[GGML_SYCL_MAX_DEVICES]; - std::unique_ptr<ggml_sycl_fattn_kv_buffers> fattn_bufs[GGML_SYCL_MAX_DEVICES]; - std::unique_ptr<ggml_sycl_pool> host_pools[GGML_SYCL_MAX_DEVICES]; std::vector<mmid_row_mapping> mmid_row_mapping_host; + std::vector<ggml_sycl_grouped_gemm_tile> mmid_tile_schedule_host; static std::unique_ptr<ggml_sycl_pool> new_pool_for_device(queue_ptr qptr, int device); static std::unique_ptr<ggml_sycl_pool> new_pool_for_host(queue_ptr qptr, int device); - static std::unique_ptr<ggml_sycl_fattn_kv_buffers> new_fattn_kv_buffers(queue_ptr qptr, int device); - ggml_sycl_pool & pool(int device) { if (pools[device] == nullptr) { pools[device] = new_pool_for_device(stream(device,0), device); @@ -431,19 +590,37 @@ struct ggml_backend_sycl_context { return pool(device); } - ggml_sycl_fattn_kv_buffers & fattn_buffers(int device) { - if (fattn_bufs[device] == nullptr) { - fattn_bufs[device] = new_fattn_kv_buffers(stream(device, 0), device); +#ifdef GGML_SYCL_GRAPH + // Map from first node pointer to graph - allows multiple graphs per context when the + // computation is split across CPU/GPU (e.g. with --n-cpu-moe) + std::unordered_map<const void *, std::unique_ptr<ggml_sycl_graph>> sycl_graphs; + + int64_t last_graph_eviction_sweep = 0; + std::optional<bool> graph_compatible; + + ggml_sycl_graph * sycl_graph(const void * first_node_ptr) { + const int64_t time_now = ggml_time_us(); + const int64_t eviction_timeout_us = g_ggml_sycl_graph_eviction_timeout * 1'000'000LL; + + // sweep every half the eviction timeout, evicting graphs unused for >= the eviction timeout + if (time_now - last_graph_eviction_sweep >= eviction_timeout_us / 2) { + last_graph_eviction_sweep = time_now; + for (auto it = sycl_graphs.begin(); it != sycl_graphs.end(); ) { + if (time_now - it->second->last_used_time >= eviction_timeout_us) { + it = sycl_graphs.erase(it); + } else { + ++it; + } + } } - return *fattn_bufs[device]; - } - ggml_sycl_fattn_kv_buffers & fattn_buffers() { - return fattn_buffers(device); + auto it = sycl_graphs.find(first_node_ptr); + if (it == sycl_graphs.end()) { + it = sycl_graphs.emplace(first_node_ptr, std::make_unique<ggml_sycl_graph>()).first; + } + it->second->last_used_time = time_now; + return it->second.get(); } - -#ifdef GGML_SYCL_GRAPH - std::unique_ptr<sycl_ex::command_graph<sycl_ex::graph_state::executable>> exec_graph = nullptr; #endif ggml_sycl_pool & host_pool(int device) { diff --git a/ggml/src/ggml-sycl/concat.cpp b/ggml/src/ggml-sycl/concat.cpp index bd5f3b2ce..77a17412f 100644 --- a/ggml/src/ggml-sycl/concat.cpp +++ b/ggml/src/ggml-sycl/concat.cpp @@ -196,7 +196,8 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { } } -static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { +template <typename block_t, int qk, ggml_type qtype> +static void concat_impl_q_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -204,25 +205,25 @@ static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * const int32_t dim = ((int32_t *) dst->op_params)[0]; - GGML_ASSERT(src0->type == GGML_TYPE_Q4_0); - GGML_ASSERT(src1->type == GGML_TYPE_Q4_0); - GGML_ASSERT(dst->type == GGML_TYPE_Q4_0); - GGML_ASSERT(src0->ne[0] % QK4_0 == 0); - GGML_ASSERT(src1->ne[0] % QK4_0 == 0); - GGML_ASSERT(dst->ne[0] % QK4_0 == 0); + GGML_ASSERT(src0->type == qtype); + GGML_ASSERT(src1->type == qtype); + GGML_ASSERT(dst->type == qtype); + GGML_ASSERT(src0->ne[0] % qk == 0); + GGML_ASSERT(src1->ne[0] % qk == 0); + GGML_ASSERT(dst->ne[0] % qk == 0); - const int ne00_blk = src0->ne[0] / QK4_0; - const int ne0_blk = dst->ne[0] / QK4_0; + const int ne00_blk = src0->ne[0] / qk; + const int ne0_blk = dst->ne[0] / qk; if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q4_0 * src0_d = (const block_q4_0 *) src0->data; - const block_q4_0 * src1_d = (const block_q4_0 *) src1->data; - block_q4_0 * dst_d = (block_q4_0 *) dst->data; - const size_t type_size = sizeof(block_q4_0); + const block_t * src0_d = (const block_t *) src0->data; + const block_t * src1_d = (const block_t *) src1->data; + block_t * dst_d = (block_t *) dst->data; + const size_t type_size = sizeof(block_t); if (dim != 3) { for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl<block_q4_0>( + concat_T_sycl<block_t>( src0_d + i3 * (src0->nb[3] / type_size), src1_d + i3 * (src1->nb[3] / type_size), dst_d + i3 * (dst->nb[3] / type_size), @@ -237,223 +238,12 @@ static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); } } else { - concat_T_sycl_non_cont<block_q4_0>( + concat_T_sycl_non_cont<block_t>( stream, (const char *) src0->data, (const char *) src1->data, (char *) dst->data, ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK4_0, src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], - ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); - } -} - -static void concat_impl_q4_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - queue_ptr stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_Q4_1); - GGML_ASSERT(src1->type == GGML_TYPE_Q4_1); - GGML_ASSERT(dst->type == GGML_TYPE_Q4_1); - GGML_ASSERT(src0->ne[0] % QK4_1 == 0); - GGML_ASSERT(src1->ne[0] % QK4_1 == 0); - GGML_ASSERT(dst->ne[0] % QK4_1 == 0); - - const int ne00_blk = src0->ne[0] / QK4_1; - const int ne0_blk = dst->ne[0] / QK4_1; - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q4_1 * src0_d = (const block_q4_1 *) src0->data; - const block_q4_1 * src1_d = (const block_q4_1 *) src1->data; - block_q4_1 * dst_d = (block_q4_1 *) dst->data; - const size_t type_size = sizeof(block_q4_1); - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl<block_q4_1>( - src0_d + i3 * (src0->nb[3] / type_size), - src1_d + i3 * (src1->nb[3] / type_size), - dst_d + i3 * (dst->nb[3] / type_size), - ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, - dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); - } - } else { - concat_T_sycl_non_cont<block_q4_1>( - stream, (const char *) src0->data, (const char *) src1->data, - (char *) dst->data, - ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK4_1, src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], - ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); - } -} - -static void concat_impl_q5_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - queue_ptr stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_Q5_0); - GGML_ASSERT(src1->type == GGML_TYPE_Q5_0); - GGML_ASSERT(dst->type == GGML_TYPE_Q5_0); - GGML_ASSERT(src0->ne[0] % QK5_0 == 0); - GGML_ASSERT(src1->ne[0] % QK5_0 == 0); - GGML_ASSERT(dst->ne[0] % QK5_0 == 0); - - const int ne00_blk = src0->ne[0] / QK5_0; - const int ne0_blk = dst->ne[0] / QK5_0; - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q5_0 * src0_d = (const block_q5_0 *) src0->data; - const block_q5_0 * src1_d = (const block_q5_0 *) src1->data; - block_q5_0 * dst_d = (block_q5_0 *) dst->data; - const size_t type_size = sizeof(block_q5_0); - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl<block_q5_0>( - src0_d + i3 * (src0->nb[3] / type_size), - src1_d + i3 * (src1->nb[3] / type_size), - dst_d + i3 * (dst->nb[3] / type_size), - ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, - dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); - } - } else { - concat_T_sycl_non_cont<block_q5_0>( - stream, (const char *) src0->data, (const char *) src1->data, - (char *) dst->data, - ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK5_0, src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], - ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); - } -} - -static void concat_impl_q5_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - queue_ptr stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_Q5_1); - GGML_ASSERT(src1->type == GGML_TYPE_Q5_1); - GGML_ASSERT(dst->type == GGML_TYPE_Q5_1); - GGML_ASSERT(src0->ne[0] % QK5_1 == 0); - GGML_ASSERT(src1->ne[0] % QK5_1 == 0); - GGML_ASSERT(dst->ne[0] % QK5_1 == 0); - - const int ne00_blk = src0->ne[0] / QK5_1; - const int ne0_blk = dst->ne[0] / QK5_1; - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q5_1 * src0_d = (const block_q5_1 *) src0->data; - const block_q5_1 * src1_d = (const block_q5_1 *) src1->data; - block_q5_1 * dst_d = (block_q5_1 *) dst->data; - const size_t type_size = sizeof(block_q5_1); - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl<block_q5_1>( - src0_d + i3 * (src0->nb[3] / type_size), - src1_d + i3 * (src1->nb[3] / type_size), - dst_d + i3 * (dst->nb[3] / type_size), - ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, - dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); - } - } else { - concat_T_sycl_non_cont<block_q5_1>( - stream, (const char *) src0->data, (const char *) src1->data, - (char *) dst->data, - ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK5_1, src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], - ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); - } -} - -static void concat_impl_q8_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - queue_ptr stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); - GGML_ASSERT(src1->type == GGML_TYPE_Q8_0); - GGML_ASSERT(dst->type == GGML_TYPE_Q8_0); - GGML_ASSERT(src0->ne[0] % QK8_0 == 0); - GGML_ASSERT(src1->ne[0] % QK8_0 == 0); - GGML_ASSERT(dst->ne[0] % QK8_0 == 0); - - const int ne00_blk = src0->ne[0] / QK8_0; - const int ne0_blk = dst->ne[0] / QK8_0; - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q8_0 * src0_d = (const block_q8_0 *) src0->data; - const block_q8_0 * src1_d = (const block_q8_0 *) src1->data; - block_q8_0 * dst_d = (block_q8_0 *) dst->data; - const size_t type_size = sizeof(block_q8_0); - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl<block_q8_0>( - src0_d + i3 * (src0->nb[3] / type_size), - src1_d + i3 * (src1->nb[3] / type_size), - dst_d + i3 * (dst->nb[3] / type_size), - ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, - dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); - } - } else { - concat_T_sycl_non_cont<block_q8_0>( - stream, (const char *) src0->data, (const char *) src1->data, - (char *) dst->data, - ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK8_0, src1->ne[1], src1->ne[2], src1->ne[3], + src1->ne[0] / qk, src1->ne[1], src1->ne[2], src1->ne[3], src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); @@ -487,23 +277,22 @@ void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { concat_impl_sycl<int8_t>(ctx, dst); break; case GGML_TYPE_Q4_0: - concat_impl_q4_0_sycl(ctx, dst); + concat_impl_q_sycl<block_q4_0, QK4_0, GGML_TYPE_Q4_0>(ctx, dst); break; case GGML_TYPE_Q4_1: - concat_impl_q4_1_sycl(ctx, dst); + concat_impl_q_sycl<block_q4_1, QK4_1, GGML_TYPE_Q4_1>(ctx, dst); break; case GGML_TYPE_Q5_0: - concat_impl_q5_0_sycl(ctx, dst); + concat_impl_q_sycl<block_q5_0, QK5_0, GGML_TYPE_Q5_0>(ctx, dst); break; case GGML_TYPE_Q5_1: - concat_impl_q5_1_sycl(ctx, dst); + concat_impl_q_sycl<block_q5_1, QK5_1, GGML_TYPE_Q5_1>(ctx, dst); break; case GGML_TYPE_Q8_0: - concat_impl_q8_0_sycl(ctx, dst); + concat_impl_q_sycl<block_q8_0, QK8_0, GGML_TYPE_Q8_0>(ctx, dst); break; default: fprintf(stderr, "%s: unsupported types: dst: %s\n", __func__, ggml_type_name(dst->type)); GGML_ASSERT(false); - break; } } diff --git a/ggml/src/ggml-sycl/conv2d.cpp b/ggml/src/ggml-sycl/conv2d.cpp index 3b3b49d05..d13ba9723 100644 --- a/ggml/src/ggml-sycl/conv2d.cpp +++ b/ggml/src/ggml-sycl/conv2d.cpp @@ -53,7 +53,16 @@ static inline int64_t whcn_output_index(int64_t n, int64_t c, int64_t y, int64_t return n * (P.OC * P.OW * P.OH) + c * P.OW * P.OH + y * P.OW + x; } -template <typename T> +// cwhn layout helpers: buffers are [N][H][W][C] input and [KH][KW][IC][OC] kernel +static inline int64_t cwhn_input_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) { + return ((n * P.IH + y) * P.IW + x) * P.IC + c; +} + +static inline int64_t cwhn_kernel_index(int64_t c_out, int64_t c_in, int64_t ky, int64_t kx, const conv2d_params & P) { + return ((ky * P.KW + kx) * P.IC + c_in) * P.OC + c_out; +} + +template <typename T, bool CWHN> static void conv2d_kernel(const float * input, const T * kernel, float * output, const conv2d_params P, const sycl::nd_item<3> & item_ct1) { const int64_t global_idx = item_ct1.get_local_id(2) + @@ -77,8 +86,8 @@ static void conv2d_kernel(const float * input, const T * kernel, float * output, const int64_t in_y = calculate_input_coord(out_y, ky, P.ST_Y, P.DL_Y, P.PD_Y); for (int64_t kx = bounds.x_min; kx < bounds.x_max; ++kx) { const int64_t in_x = calculate_input_coord(out_x, kx, P.ST_X, P.DL_X, P.PD_X); - const float input_val = input[whcn_input_index(n, c_in, in_y, in_x, P)]; - const T kernel_val = kernel[whcn_kernel_index(c_out, c_in, ky, kx, P)]; + const float input_val = input[CWHN ? cwhn_input_index(n, c_in, in_y, in_x, P) : whcn_input_index(n, c_in, in_y, in_x, P)]; + const T kernel_val = kernel[CWHN ? cwhn_kernel_index(c_out, c_in, ky, kx, P) : whcn_kernel_index(c_out, c_in, ky, kx, P)]; acc += input_val * ggml_sycl_cast<float>(kernel_val); } } @@ -87,7 +96,7 @@ static void conv2d_kernel(const float * input, const T * kernel, float * output, output[whcn_output_index(n, c_out, out_y, out_x, P)] = acc; } -template <typename T> +template <typename T, bool CWHN> static void conv2d_sycl(const float * X_D, const T * K_D, float * Y_D, const conv2d_params P, const queue_ptr & stream) { const int num_blocks = (P.TOTAL + SYCL_CONV2D_BLOCK_SIZE - 1) / SYCL_CONV2D_BLOCK_SIZE; @@ -95,7 +104,7 @@ static void conv2d_sycl(const float * X_D, const T * K_D, float * Y_D, const sycl::range<3> block_nums(1, 1, num_blocks); stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) { - conv2d_kernel<T>(X_D, K_D, Y_D, P, item_ct1); + conv2d_kernel<T, CWHN>(X_D, K_D, Y_D, P, item_ct1); }); } @@ -108,7 +117,14 @@ void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const float * X_D = (const float *) input->data; float * Y_D = (float *) dst->data; - GGML_ASSERT(ggml_is_contiguous(kernel)); + // WHCN: contiguous [KW,KH,IC,OC] kernel and [W,H,C,N] input + // CWHN: channel-fastest buffer [N][H][W][C] input and [KH][KW][IC][OC] kernel (OC fastest) + const bool whcn = ggml_is_contiguous(kernel) && ggml_is_contiguous(input); + const bool cwhn = input->nb[2] == ggml_type_size(input->type) && + kernel->nb[3] == ggml_type_size(kernel->type); + GGML_ASSERT(whcn || cwhn); + const bool cwhn_layout = !whcn && cwhn; // prefer whcn when both match (degenerate size-1 dims) + GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); GGML_ASSERT(input->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); @@ -126,9 +142,6 @@ void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const int DL_X = p[4]; const int DL_Y = p[5]; - // no cwhn layout support - GGML_ASSERT(p[6] == 0); - const int IW = input->ne[0]; const int IH = input->ne[1]; const int OW = dst->ne[0]; @@ -142,9 +155,17 @@ void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const int64_t total = (int64_t) B * OC * OH * OW; const conv2d_params params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total }; - if (kernel->type == GGML_TYPE_F16) { - conv2d_sycl<sycl::half>(X_D, (const sycl::half *) K_D, Y_D, params, stream); + if (cwhn_layout) { + if (kernel->type == GGML_TYPE_F16) { + conv2d_sycl<sycl::half, true>(X_D, (const sycl::half *) K_D, Y_D, params, stream); + } else { + conv2d_sycl<float, true>(X_D, K_D, Y_D, params, stream); + } } else { - conv2d_sycl<float>(X_D, K_D, Y_D, params, stream); + if (kernel->type == GGML_TYPE_F16) { + conv2d_sycl<sycl::half, false>(X_D, (const sycl::half *) K_D, Y_D, params, stream); + } else { + conv2d_sycl<float, false>(X_D, K_D, Y_D, params, stream); + } } } diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index b660b56ab..1e7edf3b8 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -85,7 +85,7 @@ static void dequantize_row_q2_K_sycl_reorder(const void *vx, dst_t *y, const int stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q2_K_reorder(vx, y, item_ct1, nb); + dequantize_block_q2_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb); }); } @@ -129,7 +129,7 @@ static void dequantize_row_q3_K_sycl_reorder(const void *vx, dst_t *y, const int stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q3_K_reorder(vx, y, item_ct1, nb); + dequantize_block_q3_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb); }); } @@ -158,16 +158,13 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); - int constexpr WARP_K = WARP_SIZE * QK4_0; - const int n_warp = (k + WARP_K - 1) / WARP_K; - GGML_ASSERT(k % 2 == 0); - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * - sycl::range<3>(1, 1, WARP_SIZE), - sycl::range<3>(1, 1, WARP_SIZE)), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ - dequantize_block_q4_0_reorder(vx, y, k, item_ct1); + // flat range<1>, four qs bytes (8 outputs) per item: two vectorized stores. + // k % QK4_0 == 0 gives whole 32-element blocks, so the k/8 launch covers every row exactly. + GGML_ASSERT(k % QK4_0 == 0); + stream->parallel_for(sycl::range<1>((size_t) (k / 8)), + [=](sycl::id<1> idx) { + dequantize_block_q4_0_reorder_wide(vx, y, k, idx); }); - } template <typename dst_t> @@ -177,16 +174,13 @@ static void dequantize_row_q8_0_sycl_reorder(const void *vx, dst_t *y, const int dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); - int constexpr WARP_K = WARP_SIZE * QK8_0; - const int n_warp = (k + WARP_K - 1) / WARP_K; + // flat range<1>, two pairs (4 elements) per item: no nd_range/group math, one vectorized store. + // k % QK8_0 == 0 implies k % 4 == 0, so the wide launch covers the row exactly (no tail). GGML_ASSERT(k % QK8_0 == 0); - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * - sycl::range<3>(1, 1, WARP_SIZE), - sycl::range<3>(1, 1, WARP_SIZE)), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ - dequantize_block_q8_0_reorder(vx, y, k, item_ct1); + stream->parallel_for(sycl::range<1>((size_t) (k / 4)), + [=](sycl::id<1> idx) { + dequantize_block_q8_0_reorder_wide(vx, y, k, idx); }); - } template <typename dst_t> @@ -212,20 +206,10 @@ template <typename dst_t> static void dequantize_row_q4_K_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { const int64_t nb = k / QK_K; - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<uint8_t, 1> scale_local_acc(sycl::range<1>(12), cgh); - cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 32), - sycl::range<3>(1, 1, 32)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q4_K(vx, y, get_pointer(scale_local_acc), item_ct1); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) nb * SYCL_DEQK_WI_PER_BLOCK), + sycl::range<1>((size_t) SYCL_DEQK_WI_PER_BLOCK)), + [=](sycl::nd_item<1> it) { dequantize_block_q4_K_wide(vx, y, nb, it); }); } template <typename dst_t> @@ -241,7 +225,7 @@ static void dequantize_row_q4_K_sycl_reorder(const void * vx, dst_t * y, const i cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(global_size), sycl::range<1>(local_size)), [=](sycl::nd_item<1> item_ct1) { - dequantize_block_q4_K_reorder(vx, y, get_pointer(scale_local_acc), item_ct1, nb); + dequantize_block_q4_K_reorder(vx, y + item_ct1.get_group(0) * QK_K, item_ct1.get_group(0), get_pointer(scale_local_acc), item_ct1, nb); }); }); } @@ -250,32 +234,10 @@ template <typename dst_t> static void dequantize_row_q5_K_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { const int64_t nb = k / QK_K; -#if QK_K == 256 - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 64), - sycl::range<3>(1, 1, 64)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q5_K(vx, y, item_ct1); - }); - } -#else - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 32), - sycl::range<3>(1, 1, 32)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q5_K(vx, y, item_ct1); - }); - } - -#endif + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) nb * SYCL_DEQK_WI_PER_BLOCK), + sycl::range<1>((size_t) SYCL_DEQK_WI_PER_BLOCK)), + [=](sycl::nd_item<1> it) { dequantize_block_q5_K_wide(vx, y, nb, it); }); } template <typename dst_t> @@ -288,9 +250,9 @@ static void dequantize_row_q5_K_sycl_reorder(const void * vx, dst_t * y, const i sycl::local_accessor<uint8_t, 1> scale_local_acc(sycl::range<1>(K_SCALE_SIZE), cgh); cgh.parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)), [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q5_K_reorder(vx, y, get_pointer(scale_local_acc), item_ct1, nb); + dequantize_block_q5_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), get_pointer(scale_local_acc), item_ct1, nb); }); }); } @@ -335,7 +297,7 @@ static void dequantize_row_q6_K_sycl_reorder(const void * vx, dst_t * y, const i stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), - [=](sycl::nd_item<3> item_ct1) { dequantize_block_q6_K_reorder(vx, y, item_ct1, nb); }); + [=](sycl::nd_item<3> item_ct1) { dequantize_block_q6_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb); }); } template <typename dst_t> @@ -463,6 +425,20 @@ static void dequantize_row_iq3_xxs_sycl(const void *vx, dst_t *y, const int64_t } } +template <typename dst_t> +static void dequantize_row_iq3_xxs_sycl_reorder(const void * vx, dst_t * y, const int64_t k, + dpct::queue_ptr stream) { + const int64_t nb = k / QK_K; + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq3_xxs_reorder( + vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb, + iq3xxs_grid, ksigns_iq2xs, kmask_iq2xs); + }); +} + template <typename dst_t> static void dequantize_row_iq3_s_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -483,6 +459,19 @@ static void dequantize_row_iq3_s_sycl(const void *vx, dst_t *y, const int64_t k, } } +template <typename dst_t> +static void dequantize_row_iq3_s_sycl_reorder(const void * vx, dst_t * y, const int64_t k, + dpct::queue_ptr stream) { + const int64_t nb = k / QK_K; + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq3_s_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), + item_ct1, nb, kmask_iq2xs, iq3s_grid); + }); +} + template <typename dst_t> static void dequantize_row_iq4_xs_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -537,6 +526,18 @@ static void dequantize_row_mxfp4_sycl(const void * vx, dst_t * y, const int64_t }); } +template <typename dst_t> +static void dequantize_row_mxfp4_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { + // flat range<1>, four qs bytes (8 outputs) per item: two vectorized stores. + // Each 32-element block holds QK_MXFP4/2 = 16 qs bytes = 4 items, so the + // k/8 launch covers the row exactly (k % QK_MXFP4 == 0 asserted). + GGML_ASSERT(k % QK_MXFP4 == 0); + stream->parallel_for(sycl::range<1>((size_t) (k / 8)), + [=](sycl::id<1> idx) { + dequantize_block_mxfp4_reorder_wide(vx, y, k, idx); + }); +} + template <typename dst_t> static void dequantize_row_nvfp4_sycl(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { GGML_ASSERT(k % QK_NVFP4 == 0); @@ -653,58 +654,68 @@ static void convert_unary_sycl(const void * vx, dst_t * y, const int64_t k, dpct } -to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { +static bool is_reordered_src0(const ggml_tensor * dst) { + return dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder; +} + +template <typename dst_t> +static to_t_sycl_t<dst_t> ggml_get_dequant_sycl(ggml_type type, ggml_tensor * dst) { switch (type) { case GGML_TYPE_Q1_0: - return dequantize_block_sycl<QK1_0, QR1_0, dequantize_q1_0>; + return dequantize_block_sycl<QK1_0, QR1_0, dequantize_q1_0, dst_t>; case GGML_TYPE_Q2_0: - return dequantize_block_sycl<QK2_0, QR2_0, dequantize_q2_0>; + return dequantize_block_sycl<QK2_0, QR2_0, dequantize_q2_0, dst_t>; case GGML_TYPE_Q4_0: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q4_0_sycl_reorder; + } else if constexpr (std::is_same_v<dst_t, float>) { + return dequantize_row_q4_0_sycl; } else { - return dequantize_block_sycl<QK4_0, QR4_0, dequantize_q4_0>; + return dequantize_block_sycl<QK4_0, QR4_0, dequantize_q4_0, dst_t>; } case GGML_TYPE_Q4_1: - return dequantize_block_sycl<QK4_1, QR4_1, dequantize_q4_1>; + if constexpr (std::is_same_v<dst_t, float>) { + return dequantize_row_q4_1_sycl; + } else { + return dequantize_block_sycl<QK4_1, QR4_1, dequantize_q4_1, dst_t>; + } case GGML_TYPE_Q5_0: - return dequantize_block_sycl<QK5_0, QR5_0, dequantize_q5_0>; + return dequantize_block_sycl<QK5_0, QR5_0, dequantize_q5_0, dst_t>; case GGML_TYPE_Q5_1: - return dequantize_block_sycl<QK5_1, QR5_1, dequantize_q5_1>; + return dequantize_block_sycl<QK5_1, QR5_1, dequantize_q5_1, dst_t>; case GGML_TYPE_Q8_0: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q8_0_sycl_reorder; } else { - return dequantize_block_sycl<QK8_0, QR8_0, dequantize_q8_0>; + return dequantize_block_sycl<QK8_0, QR8_0, dequantize_q8_0, dst_t>; } case GGML_TYPE_Q2_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q2_K_sycl_reorder; } else { return dequantize_row_q2_K_sycl; } case GGML_TYPE_Q3_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q3_K_sycl_reorder; } else { return dequantize_row_q3_K_sycl; } case GGML_TYPE_Q4_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q4_K_sycl_reorder; } else { return dequantize_row_q4_K_sycl; } case GGML_TYPE_Q5_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q5_K_sycl_reorder; } else { return dequantize_row_q5_K_sycl; } case GGML_TYPE_Q6_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q6_K_sycl_reorder; } else { return dequantize_row_q6_K_sycl; @@ -720,118 +731,54 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { case GGML_TYPE_IQ2_S: return dequantize_row_iq2_s_sycl; case GGML_TYPE_IQ3_XXS: - return dequantize_row_iq3_xxs_sycl; - case GGML_TYPE_IQ3_S: - return dequantize_row_iq3_s_sycl; - case GGML_TYPE_IQ4_XS: - return dequantize_row_iq4_xs_sycl; - case GGML_TYPE_IQ4_NL: - return dequantize_row_iq4_nl_sycl; - case GGML_TYPE_MXFP4: - return dequantize_row_mxfp4_sycl; - case GGML_TYPE_NVFP4: - return dequantize_row_nvfp4_sycl; - case GGML_TYPE_F32: - return convert_unary_sycl<float>; -#ifdef GGML_SYCL_HAS_BF16 - case GGML_TYPE_BF16: - return convert_unary_sycl<sycl::ext::oneapi::bfloat16>; -#endif - default: - GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type)); - return nullptr; - } -} - -to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { - switch (type) { - case GGML_TYPE_Q1_0: - return dequantize_block_sycl<QK1_0, QR1_0, dequantize_q1_0>; - case GGML_TYPE_Q2_0: - return dequantize_block_sycl<QK2_0, QR2_0, dequantize_q2_0>; - case GGML_TYPE_Q4_0: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q4_0_sycl_reorder; + if (is_reordered_src0(dst)) { + return dequantize_row_iq3_xxs_sycl_reorder; } else { - return dequantize_row_q4_0_sycl; + return dequantize_row_iq3_xxs_sycl; } - case GGML_TYPE_Q4_1: - return dequantize_row_q4_1_sycl; - case GGML_TYPE_Q5_0: - return dequantize_block_sycl<QK5_0, QR5_0, dequantize_q5_0>; - case GGML_TYPE_Q5_1: - return dequantize_block_sycl<QK5_1, QR5_1, dequantize_q5_1>; - case GGML_TYPE_Q8_0: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q8_0_sycl_reorder; - } else { - return dequantize_block_sycl<QK8_0, QR8_0, dequantize_q8_0>; - } - case GGML_TYPE_Q2_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q2_K_sycl_reorder; - } else { - return dequantize_row_q2_K_sycl; - } - case GGML_TYPE_Q3_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q3_K_sycl_reorder; - } else { - return dequantize_row_q3_K_sycl; - } - case GGML_TYPE_Q4_K: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q4_K_sycl_reorder; - } else { - return dequantize_row_q4_K_sycl; - } - case GGML_TYPE_Q5_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q5_K_sycl_reorder; - } else { - return dequantize_row_q5_K_sycl; - } - case GGML_TYPE_Q6_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q6_K_sycl_reorder; + case GGML_TYPE_IQ3_S: + if (is_reordered_src0(dst)) { + return dequantize_row_iq3_s_sycl_reorder; } else { - return dequantize_row_q6_K_sycl; + return dequantize_row_iq3_s_sycl; } - case GGML_TYPE_IQ1_S: - return dequantize_row_iq1_s_sycl; - case GGML_TYPE_IQ1_M: - return dequantize_row_iq1_m_sycl; - case GGML_TYPE_IQ2_XXS: - return dequantize_row_iq2_xxs_sycl; - case GGML_TYPE_IQ2_XS: - return dequantize_row_iq2_xs_sycl; - case GGML_TYPE_IQ2_S: - return dequantize_row_iq2_s_sycl; - case GGML_TYPE_IQ3_XXS: - return dequantize_row_iq3_xxs_sycl; - case GGML_TYPE_IQ3_S: - return dequantize_row_iq3_s_sycl; case GGML_TYPE_IQ4_XS: return dequantize_row_iq4_xs_sycl; case GGML_TYPE_IQ4_NL: return dequantize_row_iq4_nl_sycl; case GGML_TYPE_MXFP4: + if (is_reordered_src0(dst)) { + return dequantize_row_mxfp4_sycl_reorder; + } return dequantize_row_mxfp4_sycl; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_sycl; + case GGML_TYPE_F32: + if constexpr (std::is_same_v<dst_t, sycl::half>) { + return convert_unary_sycl<float>; + } + break; case GGML_TYPE_F16: - return convert_unary_sycl<sycl::half>; + if constexpr (std::is_same_v<dst_t, float>) { + return convert_unary_sycl<sycl::half>; + } + break; #ifdef GGML_SYCL_HAS_BF16 case GGML_TYPE_BF16: return convert_unary_sycl<sycl::ext::oneapi::bfloat16>; #endif default: - GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type)); - return nullptr; + break; } + GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type)); +} + +to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { + return ggml_get_dequant_sycl<sycl::half>(type, dst); +} + +to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { + return ggml_get_dequant_sycl<float>(type, dst); } @@ -846,7 +793,6 @@ to_bf16_sycl_t ggml_get_to_bf16_sycl(ggml_type type, ggml_tensor * /*dst*/) { return convert_unary_sycl<sycl::ext::oneapi::bfloat16>; default: GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type)); - return nullptr; } } #endif diff --git a/ggml/src/ggml-sycl/count-equal.cpp b/ggml/src/ggml-sycl/count-equal.cpp index 4580354cd..20126652b 100644 --- a/ggml/src/ggml-sycl/count-equal.cpp +++ b/ggml/src/ggml-sycl/count-equal.cpp @@ -74,6 +74,5 @@ void ggml_sycl_count_equal(ggml_backend_sycl_context &ctx, ggml_tensor *dst) { } break; default: GGML_ASSERT(false); - break; } } diff --git a/ggml/src/ggml-sycl/cpy.cpp b/ggml/src/ggml-sycl/cpy.cpp index ef7413abd..8e1f1d50a 100644 --- a/ggml/src/ggml-sycl/cpy.cpp +++ b/ggml/src/ggml-sycl/cpy.cpp @@ -1255,6 +1255,8 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co scope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str("\tsrc0", src0)); const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); + // the cpy kernels index in 32-bit + GGML_ASSERT(ne <= INT32_MAX); GGML_TENSOR_BINARY_OP_LOCALS01; diff --git a/ggml/src/ggml-sycl/cpy.hpp b/ggml/src/ggml-sycl/cpy.hpp index 34bae1b2d..d0e20f46c 100644 --- a/ggml/src/ggml-sycl/cpy.hpp +++ b/ggml/src/ggml-sycl/cpy.hpp @@ -25,6 +25,24 @@ __dpct_inline__ int best_index_int8(int n, const int8_t * val, float x) { return x - val[mu - 1] < val[mu] - x ? mu - 1 : mu; } +// exact roundf (half away from zero); x + 0.5f carries for x = 0.5 - 2^-25, and sycl::round is not exact under fast fp +inline int round_half_away_int(float x) { + const float t = sycl::trunc(x); + return (int) (sycl::fabs(x - t) >= 0.5f ? t + sycl::copysign(1.0f, x) : t); +} + +// correctly rounded a / b to match the CPU: fast fp gives an approximate reciprocal (~12% off by an ulp on B580), +// so pick the approximation or a neighbour by smallest exact FMA residual +inline float div_correctly_rounded(float a, float b) { + const float y = a / b; + const float lo = sycl::nextafter(y, -INFINITY); + const float hi = sycl::nextafter(y, INFINITY); + const float e = sycl::fabs(sycl::fma(-b, y, a)); + const float el = sycl::fabs(sycl::fma(-b, lo, a)); + const float eh = sycl::fabs(sycl::fma(-b, hi, a)); + return el < e && el <= eh ? lo : (eh < e ? hi : y); +} + inline void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) { const float * xi = (const float *) cxi; block_q8_0 * dsti = (block_q8_0 *) cdsti; @@ -36,15 +54,16 @@ inline void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) { amax = sycl::fmax(amax, sycl::fabs((float) v)); } - const float d = amax / ((1 << 7) - 1); - const float id = d ? 1.0f / d : 0.0f; + // matches the x86 AVX/AVX2 CPU path (ggml-cpu/arch/x86/quants.c): id = 127/amax, round half to even + const float d = div_correctly_rounded(amax, 127.0f); + const float id = amax != 0.0f ? div_correctly_rounded(127.0f, amax) : 0.0f; dsti->d = d; for (int j = 0; j < QK8_0; ++j) { const float x0 = xi[j] * id; - dsti->qs[j] = sycl::round((float) x0); + dsti->qs[j] = (int) sycl::rint(x0); } } @@ -70,10 +89,6 @@ inline void cpy_blck_f32_q1_0(const char * cxi, char * cdsti) { } } -inline int round_nearest_int(float x) { - return (int)(x >= 0.0f ? x + 0.5f : x - 0.5f); -} - inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) { const float * xi = (const float *) cxi; block_q2_0 * dsti = (block_q2_0 *) cdsti; @@ -85,7 +100,7 @@ inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) { } const float d = amax; - const float id = d > 0.0f ? 1.0f / d : 0.0f; + const float id = d > 0.0f ? div_correctly_rounded(1.0f, d) : 0.0f; dsti->d = d; @@ -94,7 +109,7 @@ inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) { } for (int j = 0; j < QK2_0; ++j) { - int q = round_nearest_int(xi[j] * id) + 1; + int q = round_half_away_int(xi[j] * id) + 1; q = dpct::max(0, dpct::min(3, q)); const int byte_index = j / 4; @@ -378,7 +393,7 @@ inline void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) { } const float d = vmax / -8; - const float id = d ? 1.0f / d : 0.0f; + const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f; dsti->d = d; @@ -386,8 +401,9 @@ inline void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) { const float x0 = xi[0 + j] * id; const float x1 = xi[QK4_0 / 2 + j] * id; - const uint8_t xi0 = dpct::min(15, (int8_t) (x0 + 8.5f)); - const uint8_t xi1 = dpct::min(15, (int8_t) (x1 + 8.5f)); + // no-op fmin stops x * id + 8.5f being fused into an FMA, so x * id is rounded as on the CPU + const uint8_t xi0 = dpct::min(15, (int8_t) (sycl::fmin(x0, 6.5f) + 8.5f)); + const uint8_t xi1 = dpct::min(15, (int8_t) (sycl::fmin(x1, 6.5f) + 8.5f)); dsti->qs[j] = xi0; dsti->qs[j] |= xi1 << 4; @@ -408,8 +424,8 @@ inline void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) { vmax = sycl::max(v, vmax); } - const float d = (vmax - vmin) / ((1 << 4) - 1); - const float id = d ? 1.0f / d : 0.0f; + const float d = div_correctly_rounded(vmax - vmin, (1 << 4) - 1); + const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f; dsti->dm.x() = d; dsti->dm.y() = vmin; @@ -418,8 +434,9 @@ inline void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) { const float x0 = (xi[0 + j] - vmin) * id; const float x1 = (xi[QK4_1 / 2 + j] - vmin) * id; - const uint8_t xi0 = dpct::min(15, (int8_t) (x0 + 0.5f)); - const uint8_t xi1 = dpct::min(15, (int8_t) (x1 + 0.5f)); + // fmin: no FMA, see cpy_blck_f32_q4_0 + const uint8_t xi0 = dpct::min(15, (int8_t) (sycl::fmin(x0, 15.5f) + 0.5f)); + const uint8_t xi1 = dpct::min(15, (int8_t) (sycl::fmin(x1, 15.5f) + 0.5f)); dsti->qs[j] = xi0; dsti->qs[j] |= xi1 << 4; @@ -442,7 +459,7 @@ inline void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) { } const float d = vmax / -16; - const float id = d ? 1.0f / d : 0.0f; + const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f; dsti->d = d; @@ -451,8 +468,9 @@ inline void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) { const float x0 = xi[0 + j] * id; const float x1 = xi[QK5_0 / 2 + j] * id; - const uint8_t xi0 = dpct::min(31, (int8_t) (x0 + 16.5f)); - const uint8_t xi1 = dpct::min(31, (int8_t) (x1 + 16.5f)); + // fmin: no FMA, see cpy_blck_f32_q4_0 + const uint8_t xi0 = dpct::min(31, (int8_t) (sycl::fmin(x0, 14.5f) + 16.5f)); + const uint8_t xi1 = dpct::min(31, (int8_t) (sycl::fmin(x1, 14.5f) + 16.5f)); dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); qh |= ((xi0 & 0x10u) >> 4) << (j + 0); @@ -474,8 +492,8 @@ inline void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) { max = v > max ? v : max; } - const float d = (max - min) / 31; - const float id = d ? 1.0f / d : 0.0f; + const float d = div_correctly_rounded(max - min, 31); + const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f; dsti->dm.x() = d; dsti->dm.y() = min; @@ -485,8 +503,9 @@ inline void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) { const float x0 = (xi[0 + j] - min) * id; const float x1 = (xi[QK5_1 / 2 + j] - min) * id; - const uint8_t xi0 = (uint8_t) (x0 + 0.5f); - const uint8_t xi1 = (uint8_t) (x1 + 0.5f); + // fmin: no FMA, see cpy_blck_f32_q4_0 + const uint8_t xi0 = (uint8_t) (sycl::fmin(x0, 31.5f) + 0.5f); + const uint8_t xi1 = (uint8_t) (sycl::fmin(x1, 31.5f) + 0.5f); dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); qh |= ((xi0 & 0x10u) >> 4) << (j + 0); diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 1b13e0f1a..eb94771d6 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -817,62 +817,73 @@ static void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restri } } +// four qs bytes per item, all inside one 32-element block: two vec<dst_t,4> stores. +// float math, same products as the scalar kernel; nibble order kept (lo at +0, hi at +16). template<typename dst_t> -static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32, - const sycl::nd_item<3> &item_ct1) { - - const int64_t i = item_ct1.get_group(2); - auto k=nb32; - // assume 32 threads - const int64_t tid = item_ct1.get_local_id(2); - const int lane_ib = i * WARP_SIZE + tid; - - if (lane_ib >= k / QK4_0) { - return; - } - - dst_t * y_ptr = yy + lane_ib * QK4_0; - - auto qs = (const uint8_t*)vx + lane_ib * QK4_0 / 2; - auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k / 2) + lane_ib; - - const float d = float(*s_ptr); - -#pragma unroll - for (int l = 0; l < QK4_0 / 2; ++l) { - int vq = qs[l]; - y_ptr[l + 0] = d * ((vq & 0xF) - 8); - y_ptr[l + 16] = d * ((vq >> 4) - 8); - } - +static void dequantize_block_q4_0_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, + const sycl::id<1> idx) { + const int64_t i = (int64_t) idx[0]; + const int64_t ib = i / 4; + const int64_t iqs = 4 * (i % 4); + + const uint8_t * qs = (const uint8_t *) vx; + const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k/2); + + const float d = float(d_ptr[ib]); + const uint8_t * q = qs + ib*(QK4_0/2) + iqs; + + sycl::vec<float, 4> lo, hi; + lo[0] = d * ((q[0] & 0xF) - 8); + lo[1] = d * ((q[1] & 0xF) - 8); + lo[2] = d * ((q[2] & 0xF) - 8); + lo[3] = d * ((q[3] & 0xF) - 8); + hi[0] = d * ((q[0] >> 4) - 8); + hi[1] = d * ((q[1] >> 4) - 8); + hi[2] = d * ((q[2] >> 4) - 8); + hi[3] = d * ((q[3] >> 4) - 8); + + dst_t * yb = yy + ib*QK4_0 + iqs; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb) = lo.convert<dst_t, sycl::rounding_mode::automatic>(); + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + QK4_0/2) = hi.convert<dst_t, sycl::rounding_mode::automatic>(); } // Dequantize Q8_0 from reorder layout: [all qs (k bytes)][all d values] -// Each thread handles one block of QK8_0 elements. template<typename dst_t> static void dequantize_block_q8_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, const sycl::nd_item<3> &item_ct1) { - - const int64_t i = item_ct1.get_group(2); - const int64_t tid = item_ct1.get_local_id(2); - const int lane_ib = i * WARP_SIZE + tid; - - if (lane_ib >= k / QK8_0) { - return; + // two values per thread, like dequantize_block<dequantize_q8_0>; the reorder layout keeps all qs contiguous + // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX + // float math, not dequantize_q8_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output + const int8_t * qs = (const int8_t *) vx; + const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k); + const int64_t stride = 2 * item_ct1.get_local_range(2) * item_ct1.get_group_range(2); + + for (int64_t i = 2 * item_ct1.get_global_id(2); i < k; i += stride) { + const float d = float(d_ptr[i / QK8_0]); + + yy[i + 0] = d * qs[i + 0]; + yy[i + 1] = d * qs[i + 1]; } +} - dst_t * y_ptr = yy + lane_ib * QK8_0; +template<typename dst_t> +static void dequantize_block_q8_0_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, + const sycl::id<1> idx) { + const int64_t i = 4 * (int64_t) idx[0]; - auto qs = (const int8_t*)vx + lane_ib * QK8_0; - auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k) + lane_ib; + const int8_t * qs = (const int8_t *) vx; + const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k); - const float d = float(*s_ptr); + const float d0 = float(d_ptr[i / QK8_0]); + const float d1 = float(d_ptr[(i + 2) / QK8_0]); -#pragma unroll - for (int l = 0; l < QK8_0; ++l) { - y_ptr[l] = d * qs[l]; - } + sycl::vec<float, 4> p; + p[0] = d0 * (float) qs[i + 0]; + p[1] = d0 * (float) qs[i + 1]; + p[2] = d1 * (float) qs[i + 2]; + p[3] = d1 * (float) qs[i + 3]; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yy + i) = p.convert<dst_t, sycl::rounding_mode::automatic>(); } template<typename dst_t> @@ -944,10 +955,9 @@ static void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restri } template<typename dst_t> -static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, +static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { #if QK_K == 256 - const int64_t i = item_ct1.get_group(2); if (i >= n_blocks) { return; } @@ -962,22 +972,29 @@ static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t * const ggml_half2 * dm = reinterpret_cast<const ggml_half2 *>(base + dm_offset); const int64_t tid = item_ct1.get_local_id(2); - const int64_t n = tid / 32; - const int64_t l = tid - 32 * n; - const int64_t is = 8 * n + l / 16; + const int64_t n = tid / 32; // 128-element chunk: 0 or 1 + const int64_t eh0 = 4 * (tid - 32 * n); // base element within the chunk - const uint8_t q = qs[32 * n + l]; - dst_t * y = yy + i * QK_K + 128 * n; + const int shift = 2 * (eh0 / 32); // 2-bit lane: 0/2/4/6 + const uint8_t * q4 = qs + 32 * n + (eh0 & 31); + const uint8_t sc0 = scales[8 * n + eh0 / 16]; const float dall = (*dm)[0]; const float dmin = (*dm)[1]; - y[l+ 0] = dall * (scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (scales[is+0] >> 4); - y[l+32] = dall * (scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (scales[is+2] >> 4); - y[l+64] = dall * (scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (scales[is+4] >> 4); - y[l+96] = dall * (scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (scales[is+6] >> 4); + + // remapped decomposition: one aligned vec<dst_t,4> per thread instead of four + // scalars strided 32 apart; a 4-run shares one scale byte and one 2-bit lane, + // so the per-element math and output are identical to the scalar kernel. + sycl::vec<dst_t, 4> v; +#pragma unroll + for (int j = 0; j < 4; ++j) { + v[j] = dall * (sc0 & 0xF) * ((q4[j] >> shift) & 3) - dmin * (sc0 >> 4); + } + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yy + 128 * n + eh0) = v; #else GGML_UNUSED(vx); GGML_UNUSED(yy); + GGML_UNUSED(i); GGML_UNUSED(item_ct1); GGML_UNUSED(n_blocks); GGML_ABORT("Q2_K reorder dequantize not supported for QK_K != 256"); @@ -1040,10 +1057,9 @@ static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restri } template<typename dst_t> -static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, +static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { #if QK_K == 256 - const int64_t i = item_ct1.get_group(2); if (i >= n_blocks) { return; } @@ -1080,16 +1096,20 @@ static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t * const float dl = d_all * (us - 32); - dst_t * y = yy + i * QK_K + 128 * n + 32 * j; + dst_t * y = yy + 128 * n + 32 * j; const uint8_t * q = qs + 32 * n; const uint8_t * hm = hmask; - for (int l = l0; l < l0 + 4; ++l) { - y[l] = dl * ((int8_t) ((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)); + sycl::vec<dst_t, 4> v; +#pragma unroll + for (int jj = 0; jj < 4; ++jj) { + v[jj] = dl * ((int8_t) ((q[l0 + jj] >> shift) & 3) - ((hm[l0 + jj] & m) ? 0 : 4)); } + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + l0) = v; #else GGML_UNUSED(vx); GGML_UNUSED(yy); + GGML_UNUSED(i); GGML_UNUSED(item_ct1); GGML_UNUSED(n_blocks); GGML_ABORT("Q3_K reorder dequantize not supported for QK_K != 256"); @@ -1108,6 +1128,53 @@ static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8 } #endif +// Work-items that cover one 256-weight K-quant block: four 64-element chunks, 8 work-items +// per chunk, 4 qs bytes per work-item. +static constexpr int SYCL_DEQK_WI_PER_BLOCK = 32; + +template <typename dst_t> +static inline void dequantize_q4_K_wide_one(dst_t * __restrict__ y, const uint8_t * __restrict__ qs, + const uint8_t * __restrict__ scales, const float dall, + const float dmin, const int tid) { + const int il = tid >> 3; // 64-element chunk, 0..3 + const int ir = (tid & 7) * 4; // 4 qs bytes within the chunk + + uint8_t sc, m; + get_scale_min_k4(2 * il + 0, scales, sc, m); + const float d1 = dall * sc; + const float m1 = dmin * m; + get_scale_min_k4(2 * il + 1, scales, sc, m); + const float d2 = dall * sc; + const float m2 = dmin * m; + + const sycl::vec<uint8_t, 4> q = vec_aligned_load<uint8_t, 4>(qs + 32 * il + ir); + + sycl::vec<dst_t, 4> lo, hi; +#pragma unroll + for (int l = 0; l < 4; ++l) { + lo[l] = d1 * (q[l] & 0xF) - m1; + hi[l] = d2 * (q[l] >> 4) - m2; + } + dst_t * yb = y + 64 * il + ir; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + 0) = lo; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + 32) = hi; +} + +template <typename dst_t> +static void dequantize_block_q4_K_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb, + const sycl::nd_item<1> & it) { + static_assert(QK_K == 256, "wide K-quant dequant assumes QK_K == 256"); + const int64_t g = it.get_global_id(0); + const int64_t b = g / SYCL_DEQK_WI_PER_BLOCK; + const int tid = (int) (g % SYCL_DEQK_WI_PER_BLOCK); + if (b >= nb) { + return; + } + const block_q4_K * x = (const block_q4_K *) vx + b; + dequantize_q4_K_wide_one(yy + b * QK_K, x->qs, x->scales, (float) x->dm[0], (float) x->dm[1], tid); +} + + template <typename dst_t> inline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __restrict__ qs_ptr, const float dall, const float dmin, uint8_t * __restrict__ scales_local, int il, int ir) { @@ -1124,56 +1191,24 @@ inline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __res const float m2 = dmin * m; sycl::vec<uint8_t, n> q_vec = vec_aligned_load<uint8_t, n>(qs_ptr + 32 * il + n * ir); - for (int l = 0; l < n; ++l) { - y[l + 0] = d1 * (q_vec[l] & 0xF) - m1; - y[l + 32] = d2 * (q_vec[l] >> 4) - m2; - } -} - -template<typename dst_t> -static void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy, - uint8_t* scales_local, const sycl::nd_item<3> &item_ct1) { - const block_q4_K * x = (const block_q4_K *) vx; - const int64_t i = item_ct1.get_group(2); - -#if QK_K == 256 - const int64_t tid = item_ct1.get_local_id(2); - const int64_t il = tid / 8; - const int64_t ir = tid % 8; - - dst_t * y = yy + i * QK_K + 64 * il + 4 * ir; - - const sycl::half2 dm = x[i].dm; - const float dall = dm[0]; - const float dmin = dm[1]; - - if (tid < 12) { - scales_local[tid] = x[i].scales[tid]; + sycl::vec<dst_t, n> lo, hi; + for (int l = 0; l < n; ++l) { + lo[l] = d1 * (q_vec[l] & 0xF) - m1; + hi[l] = d2 * (q_vec[l] >> 4) - m2; } - - item_ct1.barrier(sycl::access::fence_space::local_space); - dequantize_q4_K_common(y, x[i].qs, dall, dmin, scales_local, il, ir); -#else - const int64_t tid = item_ct1.get_local_id(2); - const uint8_t * q = x[i].qs; - dst_t * y = yy + i*QK_K; - const float d = (float)x[i].dm[0]; - const float m = (float)x[i].dm[1]; - y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4); - y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4); -#endif + *reinterpret_cast<sycl::vec<dst_t, n> *>(y + 0) = lo; + *reinterpret_cast<sycl::vec<dst_t, n> *>(y + 32) = hi; } template <typename dst_t> -static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, uint8_t * scales_local, - const sycl::nd_item<1> & item_ct1, int64_t nb) { - const int64_t i = item_ct1.get_group(0); // block index +static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, + uint8_t * scales_local, const sycl::nd_item<1> & item_ct1, int64_t nb) { const int64_t tid = item_ct1.get_local_id(0); // thread index within block const int64_t il = tid / 8; const int64_t ir = tid % 8; - dst_t * y = yy + i * QK_K + 64 * il + 4 * ir; + dst_t * y = yy + 64 * il + 4 * ir; const uint8_t * base = static_cast<const uint8_t *>(vx); const size_t qs_offset = i * (QK_K / 2); @@ -1195,67 +1230,67 @@ static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * dequantize_q4_K_common(y, qs_ptr, dall, dmin, scales_local, il, ir); } -template<typename dst_t> -static void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy, - const sycl::nd_item<3> &item_ct1) { - const block_q5_K * x = (const block_q5_K *) vx; - - const int64_t i = item_ct1.get_group(2); - -#if QK_K == 256 - // assume 64 threads - this is very slightly better than the one below - const int64_t tid = item_ct1.get_local_id(2); - const int64_t il = tid/16; // il is in 0...3 - const int64_t ir = tid%16; // ir is in 0...15 - const int64_t is = 2*il; // is is in 0...6 +template <typename dst_t> +static inline void dequantize_q5_K_wide_one(dst_t * __restrict__ y, const uint8_t * __restrict__ qs, + const uint8_t * __restrict__ qh_ptr, + const uint8_t * __restrict__ scales, const float dall, + const float dmin, const int tid) { + const int il = tid >> 3; // 64-element chunk, 0..3 + const int ir = (tid & 7) * 4; // 4 qs bytes within the chunk; qh holds the 5th bit - dst_t * y = yy + i*QK_K + 64*il + 2*ir; + uint8_t sc, m; + get_scale_min_k4(2 * il + 0, scales, sc, m); + const float d1 = dall * sc; + const float m1 = dmin * m; + get_scale_min_k4(2 * il + 1, scales, sc, m); + const float d2 = dall * sc; + const float m2 = dmin * m; - const float dall = x[i].dm[0]; - const float dmin = x[i].dm[1]; + const sycl::vec<uint8_t, 4> q = vec_aligned_load<uint8_t, 4>(qs + 32 * il + ir); + const sycl::vec<uint8_t, 4> h = vec_aligned_load<uint8_t, 4>(qh_ptr + ir); - const uint8_t * ql = x[i].qs + 32*il + 2*ir; - const uint8_t * qh = x[i].qh + 2*ir; + const uint8_t hm1 = 1 << (2 * il); + const uint8_t hm2 = hm1 << 1; - uint8_t sc, m; - get_scale_min_k4(is + 0, x[i].scales, sc, m); - const float d1 = dall * sc; const float m1 = dmin * m; - get_scale_min_k4(is + 1, x[i].scales, sc, m); - const float d2 = dall * sc; const float m2 = dmin * m; + sycl::vec<dst_t, 4> lo, hi; +#pragma unroll + for (int l = 0; l < 4; ++l) { + lo[l] = d1 * ((q[l] & 0xF) + ((h[l] & hm1) ? 16 : 0)) - m1; + hi[l] = d2 * ((q[l] >> 4) + ((h[l] & hm2) ? 16 : 0)) - m2; + } + dst_t * yb = y + 64 * il + ir; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + 0) = lo; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + 32) = hi; +} - uint8_t hm = 1 << (2*il); - y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1; - y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1; - hm <<= 1; - y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2; - y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2; -#else - const int64_t tid = item_ct1.get_local_id(2); - const uint8_t q = x[i].qs[tid]; - const int64_t im = tid/8; // 0...3 - const int64_t in = tid%8; // 0...7 - const int64_t is = tid/16; // 0 or 1 - const uint8_t h = x[i].qh[in] >> im; - const float d = x[i].d; - dst_t * y = yy + i*QK_K + tid; - y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16)); - y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16)); -#endif +template <typename dst_t> +static void dequantize_block_q5_K_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb, + const sycl::nd_item<1> & it) { + static_assert(QK_K == 256, "wide K-quant dequant assumes QK_K == 256"); + const int64_t g = it.get_global_id(0); + const int64_t b = g / SYCL_DEQK_WI_PER_BLOCK; + const int tid = (int) (g % SYCL_DEQK_WI_PER_BLOCK); + if (b >= nb) { + return; + } + const block_q5_K * x = (const block_q5_K *) vx + b; + dequantize_q5_K_wide_one(yy + b * QK_K, x->qs, x->qh, x->scales, (float) x->dm[0], (float) x->dm[1], tid); } template <typename dst_t> -static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, +static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { - const int64_t ib = item_ct1.get_group(2); #if QK_K == 256 - // assume 64 threads + // remapped decomposition: 32 threads, each emitting two aligned vec<dst_t,4> + // from four consecutive ql/qh bytes; a 4-element run shares one scale pair, + // so the per-element math and output are identical to the 64-thread kernel. const int64_t tid = item_ct1.get_local_id(2); - const int64_t il = tid / 16; // 0...3 - const int64_t ir = tid % 16; // 0...15 + const int64_t il = tid / 8; // 0...3 + const int64_t ir = tid % 8; // 0...7 const int64_t is = 2 * il; - dst_t * y = yy + ib * QK_K + 64 * il + 2 * ir; + dst_t * y = yy + 64 * il + 4 * ir; const uint8_t * base = static_cast<const uint8_t *>(vx); @@ -1273,8 +1308,8 @@ static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t * const float dall = dm_values.x(); const float dmin = dm_values.y(); - const uint8_t * ql = qs_ptr + 32 * il + 2 * ir; - const uint8_t * qh = qh_ptr + 2 * ir; + const uint8_t * ql = qs_ptr + 32 * il + 4 * ir; // four consecutive bytes + const uint8_t * qh = qh_ptr + 4 * ir; // four consecutive bytes if (tid < K_SCALE_SIZE) { scales_local[tid] = scales_ptr[tid]; @@ -1288,12 +1323,18 @@ static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t * get_scale_min_k4(is + 1, scales_local, sc, m); const float d2 = dall * sc; const float m2 = dmin * m; - uint8_t hm = 1 << (2 * il); - y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1; - y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1; - hm <<= 1; - y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2; - y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2; + const uint8_t hm_lo = 1 << (2 * il); + const uint8_t hm_hi = 1 << (2 * il + 1); + + sycl::vec<dst_t, 4> lo, hi; +#pragma unroll + for (int j = 0; j < 4; ++j) { + lo[j] = d1 * ((ql[j] & 0xF) + (qh[j] & hm_lo ? 16 : 0)) - m1; + hi[j] = d2 * ((ql[j] >> 4) + (qh[j] & hm_hi ? 16 : 0)) - m2; + } + + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 0) = lo; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 32) = hi; #else GGML_UNUSED(ib); GGML_UNUSED(tid); GGML_UNUSED(yy); GGML_UNUSED(scales_local); GGML_UNUSED(n_blocks); GGML_ABORT("Q5_K reorder dequantize not supported for QK_K != 256"); @@ -1347,14 +1388,15 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri } template <typename dst_t> -static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, +static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t ib, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { - const int64_t ib = item_ct1.get_group(2); - + // remapped decomposition: each of the 64 threads emits one aligned vec<dst_t,4> + // instead of four scalars strided 32 apart. The four elements of a run share one + // 16-element scale group, one ql nibble source and one qh bit pair, so the + // per-element math and the output are identical to the scalar kernel. const int64_t tid = item_ct1.get_local_id(2); - const int64_t ip = tid / 32; // ip is 0 or 1 - const int64_t il = tid - 32 * ip; // 0...32 - const int64_t is = 8 * ip + il / 16; + const int64_t ip = tid / 32; // 128-element half: 0 or 1 + const int64_t eh0 = 4 * (tid - 32 * ip); // base element within the half const uint8_t * base_ptr = static_cast<const uint8_t *>(vx); const auto ql_offset = ib * (QK_K / 2); @@ -1366,16 +1408,18 @@ static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * const uint8_t * scales_ptr = base_ptr + base_scales_offset; const ggml_half * d = (const ggml_half *) (base_ptr + base_d_offset) + ib; - dst_t * y = yy + ib * QK_K + 128 * ip + il; + const int shift = eh0 >= 64 ? 4 : 0; // low or high ql nibble + const int qb = 2 * (eh0 / 32); // qh bit pair: 0/2/4/6 + const int8_t sc0 = reinterpret_cast<const int8_t *>(scales_ptr + 8 * ip + eh0 / 16)[0]; + const uint8_t * ql4 = ql_ptr + 64 * ip + (eh0 & 63); + const uint8_t * qh4 = qh_ptr + 32 * ip + (eh0 & 31); - const uint8_t * ql = ql_ptr + 64 * ip + il; - const uint8_t qh = *(qh_ptr + 32 * ip + il); - const int8_t * sc = reinterpret_cast<const int8_t *>(scales_ptr + is); - - y[0] = *d * sc[0] * ((int8_t) ((ql[0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32); - y[32] = *d * sc[2] * ((int8_t) ((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32); - y[64] = *d * sc[4] * ((int8_t) ((ql[0] >> 4) | (((qh >> 4) & 3) << 4)) - 32); - y[96] = *d * sc[6] * ((int8_t) ((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32); + sycl::vec<dst_t, 4> v; +#pragma unroll + for (int j = 0; j < 4; ++j) { + v[j] = *d * sc0 * ((int8_t) (((ql4[j] >> shift) & 0xF) | ((((qh4[j] >> qb) & 3)) << 4)) - 32); + } + *reinterpret_cast<sycl::vec<dst_t, 4> *>(yy + 128 * ip + eh0) = v; } template<typename dst_t> @@ -1490,6 +1534,48 @@ static void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __res } +template<typename dst_t> +static void dequantize_block_iq3_xxs_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, + const sycl::nd_item<3> & item_ct1, const int64_t nb, + const uint32_t * iq3xxs_grid, + const uint8_t * ksigns_iq2xs, + const uint8_t * kmask_iq2xs) { + const int64_t tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int64_t il = tid / 8; + const int64_t ib = tid % 8; + const uint8_t * base = static_cast<const uint8_t *>(vx); + const uint8_t * qs = base + i * (3 * QK_K / 8); + const ggml_half * d = reinterpret_cast<const ggml_half *>(base + nb * (3 * QK_K / 8)); + const uint8_t * q3 = qs + 8 * ib; + const uint16_t * gas = reinterpret_cast<const uint16_t *>(qs + QK_K / 4) + 2 * ib; + const uint8_t * grid1 = reinterpret_cast<const uint8_t *>(iq3xxs_grid + q3[2*il+0]); + const uint8_t * grid2 = reinterpret_cast<const uint8_t *>(iq3xxs_grid + q3[2*il+1]); + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float dl = (float) d[i] * (0.5f + (aux32 >> 28)) * 0.5f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127]; + dst_t * y = yy + 32 * ib + 8 * il; + sycl::vec<dst_t, 4> lo, hi; +#pragma unroll + for (int j = 0; j < 4; ++j) { + lo[j] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + hi[j] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 0) = lo; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 4) = hi; +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(i); + GGML_UNUSED(item_ct1); + GGML_UNUSED(nb); + GGML_UNUSED(iq3xxs_grid); + GGML_UNUSED(ksigns_iq2xs); + GGML_UNUSED(kmask_iq2xs); + GGML_ABORT("IQ3_XXS reorder dequantization requires QK_K == 256"); +#endif +} + template <typename dst_t> __dpct_inline__ static void dequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy, @@ -1520,6 +1606,51 @@ dequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy, } +template <typename dst_t> +__dpct_inline__ static void +dequantize_block_iq3_s_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, + const sycl::nd_item<3> & item_ct1, const int64_t nb, + const uint8_t * kmask_iq2xs, const uint32_t * iq3s_grid) { + const int64_t tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int64_t il = tid / 8; + const int64_t ib = tid % 8; + const uint8_t * base = static_cast<const uint8_t *>(vx); + const uint8_t * qs = base + i * (QK_K / 4); + const uint8_t * qh = base + nb * (QK_K / 4) + i * (QK_K / 32); + const uint8_t * signs_base = base + nb * (QK_K / 4 + QK_K / 32) + i * (QK_K / 8); + const uint8_t * metadata = base + nb * (QK_K / 4 + QK_K / 32 + QK_K / 8) + + i * (sizeof(ggml_half) + IQ3S_N_SCALE); + const ggml_half d = *reinterpret_cast<const ggml_half *>(metadata); + const uint8_t * scales = metadata + sizeof(ggml_half); + const uint8_t * qs_ib = qs + 8 * ib; + const uint8_t * grid1 = reinterpret_cast<const uint8_t *>( + iq3s_grid + (qs_ib[2*il+0] | ((qh[ib] << (8 - 2*il)) & 256))); + const uint8_t * grid2 = reinterpret_cast<const uint8_t *>( + iq3s_grid + (qs_ib[2*il+1] | ((qh[ib] << (7 - 2*il)) & 256))); + const float dl = (float) d * (1 + 2 * ((scales[ib/2] >> (4 * (ib%2))) & 0xf)); + const uint8_t signs = signs_base[4 * ib + il]; + dst_t * y = yy + 32 * ib + 8 * il; + sycl::vec<dst_t, 4> lo, hi; +#pragma unroll + for (int j = 0; j < 4; ++j) { + lo[j] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + hi[j] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 0) = lo; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 4) = hi; +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(i); + GGML_UNUSED(item_ct1); + GGML_UNUSED(nb); + GGML_UNUSED(kmask_iq2xs); + GGML_UNUSED(iq3s_grid); + GGML_ABORT("IQ3_S reorder dequantization requires QK_K == 256"); +#endif +} + template <typename dst_t> __dpct_inline__ static void dequantize_block_iq1_s(const void *__restrict__ vx, dst_t *__restrict__ yy, @@ -1646,6 +1777,30 @@ static void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restr } } +// Reordered MXFP4 ([qs...][e...], see ggml_sycl_reordered::block_q_t<MXFP4>). +// four qs bytes per item, all inside one 32-element block: two vec<dst_t,4> stores. +// same LUT lookups and per-block scale; bit-identical output. +template<typename dst_t> +static void dequantize_block_mxfp4_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, + const sycl::id<1> idx) { + const int64_t i = (int64_t) idx[0]; + const int64_t ib = i / 4; + const int64_t iq = 4 * (i % 4); // first of four qs bytes in the block + + const uint8_t * qs = (const uint8_t *) vx + ib * (QK_MXFP4 / 2); + const float d = ggml_sycl_e8m0_to_fp32(((const uint8_t *) vx)[k / 2 + ib]) * 0.5f; + dst_t * y = yy + ib * QK_MXFP4 + iq; + + sycl::vec<dst_t, 4> lo, hi; +#pragma unroll + for (int j = 0; j < 4; ++j) { + lo[j] = d * kvalues_mxfp4[qs[iq + j] & 0xf]; + hi[j] = d * kvalues_mxfp4[qs[iq + j] >> 4]; + } + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y) = lo; + *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + QK_MXFP4 / 2) = hi; +} + template <typename dst_t> static void dequantize_block_nvfp4( diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index 4f1f97030..19af72c0e 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -10,7 +10,6 @@ #endif #include <sycl/ext/intel/esimd.hpp> #include "esimd.hpp" - #define GGML_SYCL_DMMV_HAS_ESIMD #endif static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){ @@ -66,6 +65,9 @@ static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * for (int i = 0; i < ncols; i += iter_stride) { const int col = i + vals_per_iter*tid; + if (col >= ncols) { + continue; + } const int ib = (row*ncols + col)/qk; // x block index const int iqs = (col%qk)/qr; // x quant index const int iybs = col - col%qk; // y block start index @@ -110,11 +112,12 @@ static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * } } -template <int qk, int qr, dequantize_kernel_t_reorder dequantize_kernel_reorder> +template <int qk, int qr, int qbytes, dequantize_kernel_t_reorder dequantize_kernel_reorder> static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows, const sycl::nd_item<3> &item_ct1) { // qk = quantized weights per x block // qr = number of quantized weights per data value in x block + // qbytes = bytes of the quants region per block (not qk/qr for bit-packed types) const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); @@ -125,7 +128,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d const int tid = item_ct1.get_local_id(2); - const int ncols_left = ncols % (QK4_0*WARP_SIZE); + const int ncols_left = ncols % (qk*WARP_SIZE); const int ncols_align = ncols - ncols_left; const int iter_stride = 8*2*GGML_SYCL_DMMV_X; const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter //64/16=4, 512/16/2= 16 @@ -137,7 +140,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d #else float tmp = 0.0f; #endif // GGML_SYCL_F16 - const char *d_ptr = (const char*)vx+ncols*nrows/2; + const char *d_ptr = (const char*)vx + (size_t) ncols * nrows / qk * qbytes; int i=0; for (i = 0; i < ncols_align; i += iter_stride) { const int col = i + vals_per_iter*tid; @@ -153,7 +156,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d // dequantize // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val dfloat2 v; - dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v); + dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * (qk/qr) +iqs+j/qr, v); // matrix multiplication // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2 @@ -170,7 +173,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d } for (; i < ncols; i += iter_stride) { - if (tid>=ncols_left/QK4_0) continue; + if (tid>=ncols_left/qk) continue; const int col = i + vals_per_iter*tid; const int ib = (row*ncols + col)/qk; // x block index const int iqs = (col%qk)/qr; // x quant index @@ -184,7 +187,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d // dequantize // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val dfloat2 v; - dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v); + dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * (qk/qr) +iqs+j/qr, v); // matrix multiplication // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2 @@ -1485,55 +1488,10 @@ static void dequantize_mul_mat_vec_q6_k_reorder(const void * __restrict__ vx, co } } -static void dequantize_mul_mat_vec_q4_0_sycl_reorder(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_reorder<QK4_0, QR4_0, dequantize_q4_0_reorder>( - vx, y, dst, ncols, nrows, item_ct1); - }); - } -} - - -static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>( - vx, y, dst, ncols, nrows, item_ct1); - }); - } -} - -static void dequantize_mul_mat_vec_q1_0_sycl_reorder(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { +template <int qk, int qr, dequantize_kernel_t dequantize_kernel> +static void launch_dequantize_mul_mat_vec(const void *vx, const dfloat *y, float *dst, + const int ncols, const int nrows, + dpct::queue_ptr stream) { GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead @@ -1546,16 +1504,16 @@ static void dequantize_mul_mat_vec_q1_0_sycl_reorder(const void *vx, const dfloa stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_reorder<QK1_0, QR1_0, dequantize_q1_0_reorder>( + dequantize_mul_mat_vec<qk, qr, dequantize_kernel>( vx, y, dst, ncols, nrows, item_ct1); }); } } -static void dequantize_mul_mat_vec_q1_0_sycl(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { +template <int qk, int qr, int qbytes, dequantize_kernel_t_reorder dequantize_kernel> +static void launch_dequantize_mul_mat_vec_reorder(const void *vx, const dfloat *y, float *dst, + const int ncols, const int nrows, + dpct::queue_ptr stream) { GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead @@ -1568,313 +1526,91 @@ static void dequantize_mul_mat_vec_q1_0_sycl(const void *vx, const dfloat *y, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec<QK1_0, QR1_0, dequantize_q1_0>( - vx, y, dst, ncols, nrows, item_ct1); - }); - } -} - -static void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>( - vx, y, dst, ncols, nrows, item_ct1); - }); - } -} - -static void dequantize_mul_mat_vec_q5_0_sycl(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>( - vx, y, dst, ncols, nrows, item_ct1); - }); - } -} - -static void dequantize_mul_mat_vec_q5_1_sycl(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>( - vx, y, dst, ncols, nrows, item_ct1); - }); - } -} - -static void dequantize_mul_mat_vec_q8_0_sycl_reorder(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - // Q8_0 reorder layout: [all qs (ncols*nrows bytes)][all d values] - // Cannot reuse dequantize_mul_mat_vec_reorder template because it has - // Q4_0-specific constants hardcoded (d_ptr offset and qs stride). - const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + - item_ct1.get_local_id(1); - if (row >= nrows) return; - - const int tid = item_ct1.get_local_id(2); - const int iter_stride = 8*2*GGML_SYCL_DMMV_X; - const int vals_per_iter = iter_stride / WARP_SIZE; - const int ncols_left = ncols % (QK8_0*WARP_SIZE); - const int ncols_align = ncols - ncols_left; - -#ifdef GGML_SYCL_F16 - sycl::half2 tmp = {0.0f, 0.0f}; -#else - float tmp = 0.0f; -#endif - const char *d_ptr = (const char*)vx + ncols*nrows; // d after all qs - - int i = 0; - for (i = 0; i < ncols_align; i += iter_stride) { - const int col = i + vals_per_iter*tid; - const int ib = (row*ncols + col)/QK8_0; - const int iqs = col % QK8_0; - -#pragma unroll - for (int j = 0; j < vals_per_iter; j += 2) { - dfloat2 v; - dequantize_q8_0_reorder((const void *)d_ptr, ib, (const void *)vx, - ib * QK8_0 + iqs + j, v); - -#ifdef GGML_SYCL_F16 - dfloat2 t1{y[col + j + 0], y[col + j + 1]}; - tmp += v * t1; -#else - tmp += v.x() * y[col + j + 0]; - tmp += v.y() * y[col + j + 1]; -#endif - } - } - - // handle remaining columns - for (; i < ncols; i += iter_stride) { - if (tid >= ncols_left/QK8_0) continue; - const int col = i + vals_per_iter*tid; - const int ib = (row*ncols + col)/QK8_0; - const int iqs = col % QK8_0; - -#pragma unroll - for (int j = 0; j < vals_per_iter; j += 2) { - dfloat2 v; - dequantize_q8_0_reorder((const void *)d_ptr, ib, (const void *)vx, - ib * QK8_0 + iqs + j, v); - -#ifdef GGML_SYCL_F16 - dfloat2 t1{y[col + j + 0], y[col + j + 1]}; - tmp += v * t1; -#else - tmp += v.x() * y[col + j + 0]; - tmp += v.y() * y[col + j + 1]; -#endif - } - } - - // reduce - const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2; - for (int mask = mask_start; mask > 0; mask >>= 1) { - tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); - } - - if (tid == 0) { -#ifdef GGML_SYCL_F16 - dst[row] = tmp.x() + tmp.y(); -#else - dst[row] = tmp; -#endif - } - }); - } -} - -static void dequantize_mul_mat_vec_q8_0_sycl(const void *vx, const dfloat *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>( + dequantize_mul_mat_vec_reorder<qk, qr, qbytes, dequantize_kernel>( vx, y, dst, ncols, nrows, item_ct1); }); } } -static void dequantize_mul_mat_vec_q2_K_sycl(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { +template <int ny, auto kernel> +static void launch_dequantize_mul_mat_vec_k(const void *vx, const float *y, float *dst, + const int ncols, const int nrows, + dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2 const int block_num_y = (nrows + ny - 1) / ny; const sycl::range<3> block_nums(1, 1, block_num_y); const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q2_k(vx, y, dst, ncols, nrows, item_ct1); + kernel(vx, y, dst, ncols, nrows, item_ct1); }); } -static void dequantize_mul_mat_vec_q2_K_sycl_reorder(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q2_k_reorder(vx, y, dst, ncols, nrows, item_ct1); - }); -} +#ifdef GGML_SYCL_DMMV_HAS_ESIMD +using ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE; -static void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q3_k(vx, y, dst, ncols, nrows, item_ct1); - }); -} +// policies for dequantize_mul_mat_vec_reorder_esimd: row mapping and what to do with the two sums +// plain mat-vec: a work-group owns two consecutive rows of one matrix +struct esimd_dmmv_row_pair_policy { + const void * vx; + float * dst; -static void dequantize_mul_mat_vec_q3_K_sycl_reorder(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q3_k_reorder(vx, y, dst, ncols, nrows, item_ct1); - }); -} + ESIMD_INLINE const void * w_a() const { return vx; } + ESIMD_INLINE const void * w_b() const { return vx; } -static void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q4_k(vx, y, dst, ncols, nrows, item_ct1); - }); -} + ESIMD_INLINE int row_a(int g) const { return g * 2; } + ESIMD_INLINE int row_b(int g) const { return g * 2 + 1; } + ESIMD_INLINE bool has_b(int g, int nrows) const { return row_b(g) < nrows; } -static void dequantize_mul_mat_vec_q5_K_sycl(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, nrows, item_ct1); - }); -} + ESIMD_INLINE void epilogue(int g, int nrows, float sum_a, float sum_b) const { + dst[row_a(g)] = sum_a; + if (has_b(g, nrows)) { + dst[row_b(g)] = sum_b; + } + } +}; -static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q6_k(vx, y, dst, ncols, nrows, item_ct1); - }); -} +// fused gate+up+GLU: a work-group owns one row of both matrices; a is the gate, b the up +struct esimd_dmmv_glu_policy { + const void * vgate; + const void * vup; + float * dst; + ggml_glu_op glu_op; -#ifdef GGML_SYCL_DMMV_HAS_ESIMD -using ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE; + ESIMD_INLINE const void * w_a() const { return vgate; } + ESIMD_INLINE const void * w_b() const { return vup; } -// generic reordered dequantize-matvec: each work-group owns a pair of -// consecutive output rows and updates one 32-wide accumulator per row -template <ggml_type T> + ESIMD_INLINE int row_a(int g) const { return g; } + ESIMD_INLINE int row_b(int g) const { return g; } + ESIMD_INLINE bool has_b(int, int) const { return true; } // up always contributes + + ESIMD_INLINE void epilogue(int g, int, float gate, float up) const { + using namespace sycl::ext::intel::esimd; + + // SwiGLU: silu(gate)*up = gate*sigmoid(gate)*up. GEGLU's tanh form reduces + // to the same shape via tanh(y) = 2*sigmoid(2y) - 1, so only the sigmoid + // argument differs. sigmoid via exp2 (width-1 simd is a degenerate case for + // the EM pipe, so broadcast to 16 lanes); lane 0 holds the result. + const float GELU_COEF_A = 0.044715f; + const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; + const float NEG_LOG2E = -1.44269504088896341f; + + const float arg = (glu_op == GGML_GLU_OP_GEGLU) ? + 2.0f * SQRT_2_OVER_PI * gate * (1.0f + GELU_COEF_A * gate * gate) : + gate; + + simd<float, 16> t = arg * NEG_LOG2E; + simd<float, 16> e = exp2(t); + dst[g] = gate / (1.0f + e[0]) * up; + } +}; + +// generic reordered dequantize-matvec: each work-group owns a pair of rows chosen by the policy +// and updates one 32-wide accumulator per row +template <ggml_type T, typename Policy> ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd( - const void * vx, const float * y, float * dst, + const Policy & policy, const float * y, const int ncols, const int nrows, sycl::local_accessor<float, 1> lmem, const sycl::nd_item<1> & it) { @@ -1883,41 +1619,41 @@ ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd( const int num_blocks_per_row = ncols / QK_K; const size_t nb = (size_t) nrows * num_blocks_per_row; - const auto ps = traits::make_ptrs(vx, nb); + const auto pa = traits::make_ptrs(policy.w_a(), nb); + const auto pb = traits::make_ptrs(policy.w_b(), nb); - const int tid = it.get_local_id(0); - const int row_pair = it.get_group(0); - const int row0 = row_pair * 2; // two consecutive output rows - const bool has_row1 = row0 + 1 < nrows; + const int tid = it.get_local_id(0); + const int g = it.get_group(0); + const int row_a = policy.row_a(g); + const int row_b = policy.row_b(g); + const bool has_b = policy.has_b(g, nrows); - // one 32-wide accumulator per output row (small footprint, no spill) - simd<float, 32> acc0 = 0.0f; - simd<float, 32> acc1 = 0.0f; + // one 32-wide accumulator per row (small footprint, no spill) + simd<float, 32> acc_a = 0.0f; + simd<float, 32> acc_b = 0.0f; for (int ib = tid; ib < num_blocks_per_row; ib += GGML_SYCL_DMMV_ESIMD_WG_SIZE) { simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) ib * QK_K); - const size_t bi0 = (size_t) (row0 + 0) * num_blocks_per_row + ib; - const size_t bi1 = (size_t) (row0 + 1) * num_blocks_per_row + ib; + const size_t bi_a = (size_t) row_a * num_blocks_per_row + ib; + const size_t bi_b = (size_t) row_b * num_blocks_per_row + ib; - traits::mac_pair(ps, bi0, ps, bi1, has_row1, y_vec, acc0, acc1); + traits::mac_pair(pa, bi_a, pb, bi_b, has_b, y_vec, acc_a, acc_b); } - lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{}); - lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{}); + lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{}); + lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{}); it.barrier(sycl::access::fence_space::local_space); if (tid == 0) { - float sum0 = 0.0f; - float sum1 = 0.0f; + float sum_a = 0.0f; + float sum_b = 0.0f; for (int p = 0; p < GGML_SYCL_DMMV_ESIMD_WG_SIZE; ++p) { - sum0 += lmem[p * 2 + 0]; - sum1 += lmem[p * 2 + 1]; - } - dst[row0 + 0] = sum0; - if (has_row1) { - dst[row0 + 1] = sum1; + sum_a += lmem[p * 2 + 0]; + sum_b += lmem[p * 2 + 1]; } + + policy.epilogue(g, nrows, sum_a, sum_b); } } @@ -1933,7 +1669,7 @@ static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, const sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q2_K>( - vx, y, dst, ncols, nrows, lmem, it); + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); }); }); } @@ -1950,7 +1686,7 @@ static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q3_K>( - vx, y, dst, ncols, nrows, lmem, it); + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); }); }); } @@ -1967,7 +1703,7 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q4_K>( - vx, y, dst, ncols, nrows, lmem, it); + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); }); }); } @@ -1984,7 +1720,7 @@ static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q5_K>( - vx, y, dst, ncols, nrows, lmem, it); + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); }); }); } @@ -2001,7 +1737,7 @@ static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q6_K>( - vx, y, dst, ncols, nrows, lmem, it); + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); }); }); } @@ -2042,9 +1778,9 @@ ESIMD_INLINE void q8_0_mac_stripe( } } -template <int WG> +template <int WG, typename Policy> ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd( - const void * vx, const float * y, float * dst, + const Policy & policy, const float * y, const int ncols, const int nrows, sycl::local_accessor<float, 1> lmem, const sycl::nd_item<1> & it) { @@ -2054,137 +1790,142 @@ ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd( const int nblk_row = ncols / QK8_0; const size_t nb = (size_t) nrows * nblk_row; - const int8_t * qs = (const int8_t *) vx; - const sycl::half * d = (const sycl::half *) (qs + nb * QK8_0); + const int8_t * qs_a = (const int8_t *) policy.w_a(); + const int8_t * qs_b = (const int8_t *) policy.w_b(); + const sycl::half * d_a = (const sycl::half *) (qs_a + nb * QK8_0); + const sycl::half * d_b = (const sycl::half *) (qs_b + nb * QK8_0); - const int tid = it.get_local_id(0); - const int row_pair = it.get_group(0); - const int row0 = row_pair * 2; - const bool has_row1 = row0 + 1 < nrows; + const int tid = it.get_local_id(0); + const int g = it.get_group(0); + const bool has_b = policy.has_b(g, nrows); - const size_t base0 = (size_t) row0 * nblk_row; - const size_t base1 = has_row1 ? (size_t) (row0 + 1) * nblk_row : base0; + const size_t base_a = (size_t) policy.row_a(g) * nblk_row; + const size_t base_b = (size_t) policy.row_b(g) * nblk_row; - simd<float, 32> acc0 = 0.0f; - simd<float, 32> acc1 = 0.0f; + simd<float, 32> acc_a = 0.0f; + simd<float, 32> acc_b = 0.0f; // Each thread processes one contiguous stripe. int ib = 0; for (; ib + WG * STRIPE <= nblk_row; ib += WG * STRIPE) { const int b = ib + tid * STRIPE; simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) b * QK8_0); - q8_0_mac_stripe<STRIPE>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0, - d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1); + q8_0_mac_stripe<STRIPE>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0, + d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b); } // Distribute remaining blocks across the work-group. for (int b = ib + tid; b < nblk_row; b += WG) { simd<float, 32> y_vec = block_load<float, 32>(y + (size_t) b * QK8_0); - q8_0_mac_stripe<1>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0, - d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1); + q8_0_mac_stripe<1>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0, + d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b); } - lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{}); - lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{}); + lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{}); + lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{}); it.barrier(sycl::access::fence_space::local_space); if (tid == 0) { - float sum0 = 0.0f; - float sum1 = 0.0f; + float sum_a = 0.0f; + float sum_b = 0.0f; for (int p = 0; p < WG; ++p) { - sum0 += lmem[p * 2 + 0]; - sum1 += lmem[p * 2 + 1]; - } - dst[row0 + 0] = sum0; - if (has_row1) { - dst[row0 + 1] = sum1; + sum_a += lmem[p * 2 + 0]; + sum_b += lmem[p * 2 + 1]; } + + policy.epilogue(g, nrows, sum_a, sum_b); } } -template <int WG> -static void q8_0_esimd_launch(const void * vx, const float * y, float * dst, const int ncols, - const int nrows, dpct::queue_ptr stream) { - const int workgroups = (nrows + 1) / 2; +template <int WG, typename Policy> +static void q8_0_esimd_launch(const Policy & policy, const float * y, const int ncols, const int nrows, + const int workgroups, dpct::queue_ptr stream) { stream->submit([&](sycl::handler & h) { sycl::local_accessor<float, 1> lmem(sycl::range<1>(WG * 2), h); h.parallel_for( sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * WG), sycl::range<1>(WG)), [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { - dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(vx, y, dst, ncols, nrows, lmem, it); + dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(policy, y, ncols, nrows, lmem, it); }); }); } -static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { +// Scale the work-group with the number of blocks per row. +template <typename Policy> +static void q8_0_esimd_launch_scaled(const Policy & policy, const float * y, const int ncols, const int nrows, + const int workgroups, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK8_0 == 0); - // Scale the work-group with the number of blocks per row. const int nblk_row = ncols / QK8_0; if (nblk_row >= 64) { - q8_0_esimd_launch<8>(vx, y, dst, ncols, nrows, stream); + q8_0_esimd_launch<8>(policy, y, ncols, nrows, workgroups, stream); } else if (nblk_row >= 32) { - q8_0_esimd_launch<4>(vx, y, dst, ncols, nrows, stream); + q8_0_esimd_launch<4>(policy, y, ncols, nrows, workgroups, stream); } else if (nblk_row >= 16) { - q8_0_esimd_launch<2>(vx, y, dst, ncols, nrows, stream); + q8_0_esimd_launch<2>(policy, y, ncols, nrows, workgroups, stream); } else { - q8_0_esimd_launch<1>(vx, y, dst, ncols, nrows, stream); + q8_0_esimd_launch<1>(policy, y, ncols, nrows, workgroups, stream); } } -#endif // GGML_SYCL_DMMV_HAS_ESIMD - -static void dequantize_mul_mat_vec_q4_K_sycl_reorder(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q4_k_reorder(vx, y, dst, ncols, nrows, item_ct1); - }); +static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + q8_0_esimd_launch_scaled(esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, (nrows + 1) / 2, stream); } -static void dequantize_mul_mat_vec_q5_K_sycl_reorder(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, +// ESIMD fused gate+up+GLU on reordered K-quant SOA weights: one output row (gate +// and up) per work-group, two independent 32-wide accumulators per super-block. +// Reuses the DMMV kernels' dequant+MAC primitive (esimd_reorder_q_traits<T>::mac_pair). +template <ggml_type T> +static void dequantize_mul_mat_vec_glu_reorder_esimd(const void * vx, const void * vgate, + const float * y, float * dst, + const int ncols, const int nrows, + const ggml_glu_op glu_op, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q5_k_reorder(vx, y, dst, ncols, nrows, item_ct1); - }); + const int workgroups = nrows; // one output row (gate+up) per work-group + stream->submit([&](sycl::handler & h) { + sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd<T>( + esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, lmem, it); + }); + }); } -static void dequantize_mul_mat_vec_q6_K_sycl_reorder(const void *vx, const float *y, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int ny = 2 / K_QUANTS_PER_ITERATION; - const int block_num_y = (nrows + ny - 1) / ny; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - dequantize_mul_mat_vec_q6_k_reorder(vx, y, dst, ncols, nrows, item_ct1); - }); +bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(enum ggml_type src0_type, enum ggml_glu_op glu_op, + const void * vx, const void * vgate, const float * y, + float * dst, int ncols, int nrows, dpct::queue_ptr stream) { + switch (src0_type) { + case GGML_TYPE_Q2_K: + dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q2_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q3_K: + dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q3_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q4_K: + dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q4_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q5_K: + dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q5_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q6_K: + dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q6_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q8_0: + q8_0_esimd_launch_scaled(esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, nrows, stream); + return true; + default: + return false; + } } +#endif // GGML_SYCL_DMMV_HAS_ESIMD + void ggml_sycl_op_dequantize_mul_mat_vec( ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, @@ -2233,27 +1974,27 @@ void ggml_sycl_op_dequantize_mul_mat_vec( case GGML_TYPE_Q1_0: if ((ggml_tensor_extra_gpu*)dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { - dequantize_mul_mat_vec_q1_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_reorder<QK1_0, QR1_0, QK1_0/8, dequantize_q1_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); } else { - dequantize_mul_mat_vec_q1_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec<QK1_0, QR1_0, dequantize_q1_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q4_0: if ((ggml_tensor_extra_gpu*)dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { - dequantize_mul_mat_vec_q4_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_reorder<QK4_0, QR4_0, QK4_0/2, dequantize_q4_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); } else { - dequantize_mul_mat_vec_q4_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q4_1: - dequantize_mul_mat_vec_q4_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); break; case GGML_TYPE_Q5_0: - dequantize_mul_mat_vec_q5_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); break; case GGML_TYPE_Q5_1: - dequantize_mul_mat_vec_q5_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); break; case GGML_TYPE_Q8_0: if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && @@ -2264,10 +2005,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec( } else #endif { - dequantize_mul_mat_vec_q8_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_reorder<QK8_0, QR8_0, QK8_0, dequantize_q8_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); } } else { - dequantize_mul_mat_vec_q8_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q2_K: @@ -2280,10 +2021,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec( else #endif { - dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q2_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } } else { - dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2, dequantize_mul_mat_vec_q2_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q3_K: @@ -2296,10 +2037,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec( else #endif { - dequantize_mul_mat_vec_q3_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q3_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } } else { - dequantize_mul_mat_vec_q3_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q3_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q4_K: @@ -2312,10 +2053,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec( else #endif { - dequantize_mul_mat_vec_q4_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q4_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } } else { - dequantize_mul_mat_vec_q4_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q4_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q5_K: @@ -2328,10 +2069,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec( else #endif { - dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q5_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } } else { - dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q5_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q6_K: @@ -2344,10 +2085,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec( else #endif { - dequantize_mul_mat_vec_q6_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q6_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } } else { - dequantize_mul_mat_vec_q6_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q6_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_F16: diff --git a/ggml/src/ggml-sycl/dmmv.hpp b/ggml/src/ggml-sycl/dmmv.hpp index bd8373564..318e3489d 100644 --- a/ggml/src/ggml-sycl/dmmv.hpp +++ b/ggml/src/ggml-sycl/dmmv.hpp @@ -24,4 +24,13 @@ void ggml_sycl_op_dequantize_mul_mat_vec( const int64_t src1_ncols, const int64_t src1_padded_row_size, const dpct::queue_ptr &stream); +// Fused dense-FFN mat-vec: writes glu(gate . y, up . y) for same-type reordered +// weights (Q2_K..Q6_K, Q8_0) and a single F32 activation column, using the ESIMD DMMV kernels. +// vx is the up weight, vgate the gate weight. Only defined when GGML_SYCL_DMMV_HAS_ESIMD; +// returns false if the type is unhandled. +bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd( + enum ggml_type src0_type, enum ggml_glu_op glu_op, + const void * vx, const void * vgate, const float * y, + float * dst, int ncols, int nrows, dpct::queue_ptr stream); + #endif // GGML_SYCL_DMMV_HPP diff --git a/ggml/src/ggml-sycl/dpct/helper.hpp b/ggml/src/ggml-sycl/dpct/helper.hpp index 85af4cab6..fd3b9655a 100644 --- a/ggml/src/ggml-sycl/dpct/helper.hpp +++ b/ggml/src/ggml-sycl/dpct/helper.hpp @@ -3159,7 +3159,7 @@ namespace dpct _local_mem_size = local_mem_size; - }; + } static inline std::mutex kernel_function_ptr_map_mutex; public: diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp index 2e926abea..247590f7c 100644 --- a/ggml/src/ggml-sycl/element_wise.cpp +++ b/ggml/src/ggml-sycl/element_wise.cpp @@ -361,13 +361,6 @@ static void unary_op_sqr_kernel(const T * x, T * dst, const int k, const sycl::n } } -template<typename T> -static void unary_op_clamp_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1, float min_val, float max_val) { - SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = op_clamp(x[i], min_val, max_val); - } -} - template<typename T> static void unary_op_ceil_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { @@ -375,14 +368,6 @@ static void unary_op_ceil_kernel(const T * x, T * dst, const int k, const sycl:: } } -template<typename T> -static void clamp(const T * x, T * dst, const float min, const float max, const int k, - const sycl::nd_item<1> &item_ct1) { - SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = x[i] < static_cast<T>(min) ? static_cast<T>(min) : (x[i] > static_cast<T>(max) ? static_cast<T>(max) : x[i]); - } -} - template<typename T, typename F> static void unary_gated_op_flat_kernel(const T * x, const T * g, T * dst, const uint64_t k, const sycl::nd_item<1> & item_ct1, F func) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { @@ -452,6 +437,46 @@ static void unary_mul_sycl(const T * x, const T * g, T * dst, const int64_t k, c }); } +// ADD(bias) + UNARY + MUL(scale) with both broadcast over dim 0, the delta-net alpha gate: +// dst[i] = op(a[i] + bias[i % ne0]) * scale[i % ne0]. k == ne0 makes that the flat index. +template<typename F> +static void add_unary_mul_flat_kernel(const float * a, const float * bias, const float * scale, float * dst, + const int64_t k, const sycl::nd_item<1> &item_ct1, F op) { + SYCL_GLOBAL_ID_LOOP(k, item_ct1) { + dst[i] = op(a[i] + bias[i]) * scale[i]; + } +} + +template<typename F> +static void add_unary_mul_bcast_kernel(const float * a, const float * bias, const float * scale, float * dst, + const int64_t k, const sycl::uint3 ne0_fd, const sycl::nd_item<1> &item_ct1, F op) { + SYCL_GLOBAL_ID_LOOP(k, item_ct1) { + const uint32_t h = fastmodulo((uint32_t) i, ne0_fd); + dst[i] = op(a[i] + bias[h]) * scale[h]; + } +} + +template<typename F> +static void add_unary_mul_sycl(const float * a, const float * bias, const float * scale, float * dst, + const int64_t k, const int64_t ne0, queue_ptr main_stream, F op) { + const size_t num_blocks = ceil_div((size_t) k, (size_t) SYCL_GLU_BLOCK_SIZE); + const sycl::nd_range<1> range(num_blocks * sycl::range<1>(SYCL_GLU_BLOCK_SIZE), sycl::range<1>(SYCL_GLU_BLOCK_SIZE)); + + if (k == ne0) { + main_stream->parallel_for(range, [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + add_unary_mul_flat_kernel(a, bias, scale, dst, k, item_ct1, op); + }); + return; + } + + // 32-bit fastdiv, exact only below 2^31; ggml_sycl_can_fuse() already declined past that + GGML_ASSERT(k < ((int64_t) 1 << 31)); + const sycl::uint3 ne0_fd = init_fastdiv_values((uint32_t) ne0); + main_stream->parallel_for(range, [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + add_unary_mul_bcast_kernel(a, bias, scale, dst, k, ne0_fd, item_ct1, op); + }); +} + namespace ggml_sycl_detail { static void acc_f32_sycl(const char *x, const char *y, float *dst, const int64_t n_elements, @@ -889,20 +914,11 @@ static inline void ggml_sycl_op_sqr(ggml_backend_sycl_context & ctx, ggml_tensor } static inline void ggml_sycl_op_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - float min_val; - float max_val; - memcpy(&min_val, dst->op_params, sizeof(float)); - memcpy(&max_val, (float *) dst->op_params + 1, sizeof(float)); - ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst, - [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream, float min_arg, float max_arg) { - const int num_blocks = ceil_div(k_elements, SYCL_CLAMP_BLOCK_SIZE); - stream->parallel_for( - sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE), - sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - clamp(src, dst_ptr, min_arg, max_arg, k_elements, item_ct1); - }); - }, min_val, max_val); + const float min_val = ggml_get_op_params_f32(dst, 0); + const float max_val = ggml_get_op_params_f32(dst, 1); + ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [min_val, max_val](auto x) { + return op_clamp(x, min_val, max_val); + }); } static inline void ggml_sycl_op_xielu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -995,6 +1011,19 @@ static inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_ten }); } +// Hands `launch` the functor for the unary op of a fused unary chain. Anything else +// ggml_sycl_can_fuse() has already declined, so the default is a dispatcher bug. +template<typename F> +static void dispatch_fused_unary_op(ggml_unary_op uop, F && launch) { + switch (uop) { + case GGML_UNARY_OP_SILU: launch([](float v) { return op_silu(v); }); break; + case GGML_UNARY_OP_SIGMOID: launch([](float v) { return op_sigmoid(v); }); break; + case GGML_UNARY_OP_SOFTPLUS: launch([](float v) { return op_softplus(v); }); break; + default: + GGML_ABORT("fused unary chain: unsupported unary op %s", ggml_unary_op_name(uop)); + } +} + // dst = op(unary_node->src[0]) * other, written straight to the MUL output, saving the // standalone unary launch. Preconditions come from ggml_sycl_can_fuse(); re-asserted here. void ggml_sycl_op_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node) { @@ -1032,13 +1061,41 @@ void ggml_sycl_op_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * } }; - switch (ggml_get_unary_op(unary_node)) { - case GGML_UNARY_OP_SILU: dispatch_type([](float v) { return op_silu(v); }); break; - case GGML_UNARY_OP_SIGMOID: dispatch_type([](float v) { return op_sigmoid(v); }); break; - case GGML_UNARY_OP_SOFTPLUS: dispatch_type([](float v) { return op_softplus(v); }); break; - default: - GGML_ABORT("fused unary+mul: unsupported unary op %s", ggml_unary_op_name(ggml_get_unary_op(unary_node))); - } + dispatch_fused_unary_op(ggml_get_unary_op(unary_node), dispatch_type); +} + +// dst = op(a + bias) * scale for an ADD + UNARY + MUL chain whose bias and scale broadcast +// over dim 0. Preconditions come from ggml_sycl_can_fuse(); re-asserted here. +void ggml_sycl_op_add_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add_node, + ggml_tensor * unary_node, ggml_tensor * mul_node) { + // the dst-arity convention the other fusions follow; a and bias live on add_node + scope_op_debug_print scope_dbg_print(__func__, mul_node, /*num_src=*/2); + + const ggml_tensor * a = add_node->src[0]; + const ggml_tensor * bias = add_node->src[1]; + const ggml_tensor * scale = (mul_node->src[0] == unary_node) ? mul_node->src[1] : mul_node->src[0]; + + // scale is picked by elimination; ggml_can_fuse()'s single-use rule rules out MUL(unary, unary) + GGML_ASSERT(scale != unary_node); + GGML_ASSERT(a->type == GGML_TYPE_F32 && bias->type == GGML_TYPE_F32); + GGML_ASSERT(scale->type == GGML_TYPE_F32 && mul_node->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_are_same_shape(a, mul_node)); + // a and dst are indexed flat + GGML_ASSERT(ggml_is_contiguous(a) && ggml_is_contiguous(mul_node)); + // bias and scale are one contiguous ne0-length row each, broadcast over the outer dims + GGML_ASSERT(bias->ne[0] == a->ne[0] && scale->ne[0] == a->ne[0]); + GGML_ASSERT(ggml_nrows(bias) == 1 && ggml_nrows(scale) == 1); + GGML_ASSERT(ggml_is_contiguous(bias) && ggml_is_contiguous(scale)); + + queue_ptr main_stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const auto dispatch_op = [&](auto op) { + add_unary_mul_sycl((const float *) a->data, (const float *) bias->data, (const float *) scale->data, + (float *) mul_node->data, ggml_nelements(mul_node), mul_node->ne[0], main_stream, op); + }; + + dispatch_fused_unary_op(ggml_get_unary_op(unary_node), dispatch_op); } __dpct_inline__ float ggml_sycl_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) { @@ -1088,7 +1145,7 @@ static void swiglu_oai_sycl(const T * x, }); } -void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { +static void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; void * src0_d = src0->data; @@ -1102,9 +1159,7 @@ void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) GGML_ASSERT(ggml_is_contiguous_1(src0)); GGML_ASSERT(src0->nb[0] == ggml_element_size(src0)); GGML_ASSERT(ggml_is_contiguous(dst)); - - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); GGML_ASSERT(src0->type == dst->type); GGML_ASSERT(dst->ne[0] == nc); GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0)); @@ -1116,20 +1171,33 @@ void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) GGML_ASSERT(src0->type == src1->type); } - //const int32_t swapped = ((const int32_t *) dst->op_params)[1]; const int32_t swapped = ggml_get_op_params_i32(dst, 1); - const float alpha = ggml_get_op_params_f32(dst, 2); - const float limit = ggml_get_op_params_f32(dst, 3); + const float alpha = ggml_get_op_params_f32(dst, 2); + const float limit = ggml_get_op_params_f32(dst, 3); - float * src0_p = (float *) src0_d; - float * src1_p = (float *) src1_d; + if (src0->type == GGML_TYPE_F16) { + sycl::half * src0_p = (sycl::half *) src0_d; + sycl::half * src1_p = (sycl::half *) src1_d; - if (!src1) { - src0_p += swapped ? nc : 0; - src1_p += swapped ? 0 : nc; - } + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } - swiglu_oai_sycl(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream); + swiglu_oai_sycl(src0_p, src1_p, (sycl::half *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(sycl::half), + src1_o / sizeof(sycl::half), alpha, limit, stream); + } else { + float * src0_p = (float *) src0_d; + float * src1_p = (float *) src1_d; + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + swiglu_oai_sycl(src0_p, src1_p, (float *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), + src1_o / sizeof(float), alpha, limit, stream); + } } template <typename T> diff --git a/ggml/src/ggml-sycl/element_wise.hpp b/ggml/src/ggml-sycl/element_wise.hpp index d280066ef..0a5bfa956 100644 --- a/ggml/src/ggml-sycl/element_wise.hpp +++ b/ggml/src/ggml-sycl/element_wise.hpp @@ -132,4 +132,9 @@ void ggml_sycl_arange(ggml_backend_sycl_context & ctx, ggml_tensor * dst); // fused UNARY(silu|sigmoid|softplus) + MUL; see ggml_sycl_can_fuse() for the accepted shapes void ggml_sycl_op_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node); +// fused f32 ADD + UNARY(silu|sigmoid|softplus) + MUL with the bias and the scale broadcast +// over dim 0; see ggml_sycl_can_fuse() for the accepted shapes +void ggml_sycl_op_add_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add_node, + ggml_tensor * unary_node, ggml_tensor * mul_node); + #endif // GGML_SYCL_ELEMENTWISE_HPP diff --git a/ggml/src/ggml-sycl/esimd.hpp b/ggml/src/ggml-sycl/esimd.hpp index 0485ff0ce..568173e7a 100644 --- a/ggml/src/ggml-sycl/esimd.hpp +++ b/ggml/src/ggml-sycl/esimd.hpp @@ -2,9 +2,14 @@ #define GGML_SYCL_ESIMD_HPP #include <sycl/ext/intel/esimd.hpp> +#include <sycl/ext/intel/experimental/grf_size_properties.hpp> #include "common.hpp" +#ifdef GGML_SYCL_MMVQ_HAS_XMX +#include <sycl/ext/intel/esimd/xmx/dpas.hpp> +#endif + namespace ggml_sycl_esimd { constexpr int GGML_SYCL_DMMV_ESIMD_WG_SIZE = 4; @@ -584,6 +589,818 @@ template <> struct esimd_reorder_q_traits<GGML_TYPE_Q6_K> { } }; + +// +// Multi-column mul_mat_vec for reordered K-quant weights on the XMX engines (int8 DPAS). +// A thread owns 16 weight rows (B operand, one dword is 4 weights of a row), the q8_1 activation +// columns are the A operand. A super-block is 8 groups of 32, run as 2 parts of 4 groups. A quant +// type supplies its loads and unpack as a traits struct. +// +using sycl::ext::intel::esimd::simd; +constexpr int XMX_ROWS = 16; // weight rows per thread (DPAS N) +constexpr int XMX_CHUNK = 64; // widest single launch, more columns run as repeated launches +constexpr int XMX_FUSED_CHUNK = GGML_SYCL_XMX_GLU_MAX_COLS; // widest launch of the fused gate and up kernel + +// threads that split K for one tile, sized so that the shared memory stays small +template <int NC> constexpr int xmx_n_split() { + return NC <= 8 ? 32 : (NC == 16 ? 16 : 8); +} + +inline bool xmx_supported(const void * vx, const int ncols) { + // 2D block loads need a 64 byte aligned base + return ncols % QK_K == 0 && (uintptr_t) vx % 64 == 0; +} + +// transposed 2D block load of 16 rows, dword (kd, n) lands at [kd * 16 + n] +// 2D block loads need rows of at least 64 bytes, shorter rows (SR) are read row by row +template <int W, bool SR> +ESIMD_INLINE simd<uint32_t, W * XMX_ROWS> xmx_load_t(const void * base, unsigned pitch, unsigned h, int x, int y) { + using namespace sycl::ext::intel::esimd; + if constexpr (SR) { + simd<uint32_t, W * XMX_ROWS> r; + for (int n = 0; n < XMX_ROWS; ++n) { + const unsigned row = y + n < (int) h ? y + n : h; + const char * p = (const char *) base + (size_t) row * (pitch + 1); + r.template select<W, XMX_ROWS>(n) = block_load<uint32_t, W>((const uint32_t *) p + x); + } + return r; + } else { + return load_2d<uint32_t, W, XMX_ROWS, 1, true, false>((const uint32_t *) base, pitch, h, pitch, x, y); + } +} + +// per byte (q - zero) as int8 for q in 0..127, no borrow between bytes +ESIMD_INLINE simd<uint32_t, 128> xmx_bytes_sub(simd<uint32_t, 128> x, uint32_t zero_x4) { + return ((x | 0x80808080u) - zero_x4) ^ 0x80808080u; +} + +// How the scales of a quant type enter the DPAS epilogue of a group of 32 (see xmx_group) +enum class xmx_scale_kind { + split, // two scales per group, one per 16 wide half: the weights are masked to one half for each DPAS + min, // one scale and one min per group: a second DPAS sums the activations for the min term + min16, // a scale and a min per 16 wide half: the scales are folded into the DPAS operands + plain, // one scale per group +}; + +// What every quant type states about itself. nc_wide is the tile width from which it takes the large register file. +template <bool SR, int NC_WIDE, xmx_scale_kind KIND> struct xmx_traits_base { + static constexpr bool short_rows = SR; + static constexpr int nc_wide = NC_WIDE; + static constexpr bool has_min = KIND == xmx_scale_kind::min; + static constexpr bool plain = KIND == xmx_scale_kind::plain; + static constexpr bool min16 = KIND == xmx_scale_kind::min16; +}; + +// The part of a context every quant type shares: the last row (the clamp of 2D loads), this thread's first row and +// the number of super-blocks per row. A type adds its array pointers and 2D load pitches. +struct xmx_ctx_base { + unsigned h; + int row0; + int bpr; +}; + +ESIMD_INLINE void xmx_ctx_init(xmx_ctx_base & c, int nrows, int bpr, int row0) { + c.h = nrows - 1; + c.row0 = row0; + c.bpr = bpr; +} + +// pitch of a 2D load over an array with `bytes` bytes per super-block +constexpr unsigned xmx_pitch(int bpr, int bytes) { + return bpr * bytes - 1; +} + +// per group row scales of the 16 rows, one vector per group +template <int N> struct xmx_row_scales { + simd<float, 16> v[N]; + + ESIMD_INLINE simd<float, 16> get(int i) { return v[i]; } +}; + +// the 12 packed scale bytes of a super-block (Q3_K, Q4_K, Q5_K) of the 16 rows as three dwords +ESIMD_INLINE void xmx_load_sc12(const uint8_t * sc, + const simd<uint32_t, 16> & bi, + simd<uint32_t, 16> & w0, + simd<uint32_t, 16> & w1, + simd<uint32_t, 16> & w2) { + using namespace sycl::ext::intel::esimd; + w0 = gather<uint32_t, 16>((const uint32_t *) sc, bi * 12); + w1 = gather<uint32_t, 16>((const uint32_t *) sc, bi * 12 + 4); + w2 = gather<uint32_t, 16>((const uint32_t *) sc, bi * 12 + 8); +} + +// d and dmin (a half2 per super-block, Q2_K, Q4_K, Q5_K) of the 16 rows +ESIMD_INLINE void xmx_load_dm(const sycl::half * dm, + const simd<uint32_t, 16> & bi, + simd<float, 16> & d, + simd<float, 16> & dmin) { + using namespace sycl::ext::intel::esimd; + d = convert<float>(gather<sycl::half, 16>(dm, bi * (2 * (uint32_t) sizeof(sycl::half)))); + dmin = convert<float>( + gather<sycl::half, 16>(dm, bi * (2 * (uint32_t) sizeof(sycl::half)) + (uint32_t) sizeof(sycl::half))); +} + +// Q6_K reorder layout: [ql: nb*128] [qh: nb*64] [scales int8: nb*16] [d: nb*2] +template <bool SR> struct xmx_traits_q6_k : xmx_traits_base<SR, 7, xmx_scale_kind::split> { + struct ctx : xmx_ctx_base { + const uint8_t * ql; + const uint8_t * qh; + const int8_t * scales; + const sycl::half * dh; + unsigned pl, ph, ps; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + const size_t nb = (size_t) nrows * bpr; + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.ql = (const uint8_t *) vx; + c.qh = c.ql + nb * (QK_K / 2); + c.scales = (const int8_t *) (c.qh + nb * (QK_K / 4)); + c.dh = (const sycl::half *) (c.scales + nb * (QK_K / 16)); + c.pl = xmx_pitch(bpr, QK_K / 2); + c.ph = xmx_pitch(bpr, QK_K / 4); + c.ps = xmx_pitch(bpr, QK_K / 16); + return c; + } + + // row scales of the 16 scale groups of a super-block + struct scales_t { + simd<int8_t, 256> sc8; + simd<float, 16> dv; + + ESIMD_INLINE simd<float, 16> get(int gr) { + simd<int8_t, 16> s = sc8.template select<16, 4>(64 * (gr / 4) + (gr % 4)); + return sycl::ext::intel::esimd::convert<float>(s) * dv; + } + }; + + // rows: clamped row index of each of the 16 lanes + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + s.dv = convert<float>(gather<sycl::half, 16>(c.dh, (rows * c.bpr + sb) * (uint32_t) sizeof(sycl::half))); + simd<uint32_t, 64> raw = xmx_load_t<4, SR>(c.scales, c.ps, c.h, sb * 4, c.row0); + s.sc8 = raw.template bit_cast_view<int8_t>(); + return s; + } + + struct part_t { + simd<uint32_t, 128> a, b, h; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; + p.a = xmx_load_t<8, SR>(c.ql, c.pl, c.h, sb * 32 + part * 16, c.row0); + p.b = xmx_load_t<8, SR>(c.ql, c.pl, c.h, sb * 32 + part * 16 + 8, c.row0); + p.h = xmx_load_t<8, SR>(c.qh, c.ph, c.h, sb * 16 + part * 8, c.row0); + return p; + } + + static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int /*part*/) { + simd<uint32_t, 128> qs = (g & 1) ? p.b : p.a; + simd<uint32_t, 128> x; + if (g == 0) { + x = (qs & 0x0F0F0F0Fu) | ((p.h << 4) & 0x30303030u); + } else if (g == 1) { + x = (qs & 0x0F0F0F0Fu) | ((p.h << 2) & 0x30303030u); + } else if (g == 2) { + x = ((qs >> 4) & 0x0F0F0F0Fu) | (p.h & 0x30303030u); + } else { + x = ((qs >> 4) & 0x0F0F0F0Fu) | ((p.h >> 2) & 0x30303030u); + } + return xmx_bytes_sub(x, 0x20202020u); + } +}; + +// Q3_K reorder layout: [qs: nb*64] [hmask: nb*32] [scales: nb*12] [d: nb*2] +// chunk s of 32: 2 bits from qs byte 32*(s/4)+l at shift 2*(s%4), high bit is hmask[l] bit s, value is q - 4 +template <bool SR> struct xmx_traits_q3_k : xmx_traits_base<SR, 16, xmx_scale_kind::split> { + struct ctx : xmx_ctx_base { + const uint8_t * qs; + const uint8_t * hm; + const uint8_t * sc; + const sycl::half * dh; + unsigned pq, ph; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + const size_t nb = (size_t) nrows * bpr; + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.qs = (const uint8_t *) vx; + c.hm = c.qs + nb * (QK_K / 4); + c.sc = c.hm + nb * (QK_K / 8); + c.dh = (const sycl::half *) (c.sc + nb * 12); + c.pq = xmx_pitch(bpr, QK_K / 4); + c.ph = xmx_pitch(bpr, QK_K / 8); + return c; + } + + using scales_t = xmx_row_scales<16>; + + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + simd<uint32_t, 16> bi = rows * c.bpr + sb; + simd<uint32_t, 16> w0, w1, w2; + xmx_load_sc12(c.sc, bi, w0, w1, w2); + simd<float, 16> dv = convert<float>(gather<sycl::half, 16>(c.dh, bi * (uint32_t) sizeof(sycl::half))); +#pragma unroll + for (int i = 0; i < 16; ++i) { + const simd<uint32_t, 16> lo = (i % 8) < 4 ? w0 : w1; + const int bl = i % 4; + simd<uint32_t, 16> nib = (lo >> (8 * bl + (i < 8 ? 0 : 4))) & 0xFu; + simd<uint32_t, 16> hi = ((w2 >> (8 * bl + 2 * (i / 4))) & 3u) << 4; + simd<int, 16> code = nib | hi; + s.v[i] = convert<float>(code - 32) * dv; + } + return s; + } + + struct part_t { + simd<uint32_t, 128> q, h; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; + p.q = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 16 + part * 8, c.row0); + p.h = xmx_load_t<8, SR>(c.hm, c.ph, c.h, sb * 8, c.row0); + return p; + } + + static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int part) { + const int s = 4 * part + g; + simd<uint32_t, 128> x = ((p.q >> (2 * g)) & 0x03030303u) | (((p.h >> s) & 0x01010101u) << 2); + return xmx_bytes_sub(x, 0x04040404u); + } +}; + +// Q4_K and Q5_K (QH) reorder layout: [qs: nb*128] ([qh: nb*32]) [scales: nb*12] [dm: nb*4] +// group j of 32: nibble of qs byte 32*(j/2)+l (low for even j), Q5_K adds 16 * bit j of qh[l] +// value is dall * sc_j * q - dmin * m_j, one scale and one min per group +template <bool QH, bool SR> struct xmx_traits_q45_k : xmx_traits_base<SR, QH ? 16 : 7, xmx_scale_kind::min> { + struct ctx : xmx_ctx_base { + const uint8_t * qs; + const uint8_t * qh; + const uint8_t * sc; + const sycl::half * dm; + unsigned pq, ph; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + const size_t nb = (size_t) nrows * bpr; + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.qs = (const uint8_t *) vx; + c.qh = c.qs + nb * (QK_K / 2); + c.sc = QH ? c.qh + nb * (QK_K / 8) : c.qh; + c.dm = (const sycl::half *) (c.sc + nb * 12); + c.pq = xmx_pitch(bpr, QK_K / 2); + c.ph = xmx_pitch(bpr, QK_K / 8); + return c; + } + + struct scales_t { + xmx_row_scales<8> d; // dall * scale of each group + xmx_row_scales<8> m; // dmin * min of each group + }; + + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + simd<uint32_t, 16> bi = rows * c.bpr + sb; + simd<uint32_t, 16> w0, w1, w2; + xmx_load_sc12(c.sc, bi, w0, w1, w2); + simd<float, 16> d, dmin; + xmx_load_dm(c.dm, bi, d, dmin); +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int jj = j % 4; + simd<uint32_t, 16> sc, m; + if (j < 4) { + sc = (w0 >> (8 * jj)) & 63u; + m = (w1 >> (8 * jj)) & 63u; + } else { + sc = ((w2 >> (8 * jj)) & 0xFu) | (((w0 >> (8 * jj + 6)) & 3u) << 4); + m = ((w2 >> (8 * jj + 4)) & 0xFu) | (((w1 >> (8 * jj + 6)) & 3u) << 4); + } + s.d.v[j] = convert<float>(simd<int, 16>(sc)) * d; + s.m.v[j] = convert<float>(simd<int, 16>(m)) * dmin; + } + return s; + } + + struct part_t { + simd<uint32_t, 128> a, b, h; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; + p.a = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 32 + part * 16, c.row0); + p.b = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 32 + part * 16 + 8, c.row0); + if constexpr (QH) { + p.h = xmx_load_t<8, SR>(c.qh, c.ph, c.h, sb * 8, c.row0); + } + return p; + } + + static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int part) { + simd<uint32_t, 128> qs = (g < 2) ? p.a : p.b; + simd<uint32_t, 128> x = (qs >> (4 * (g & 1))) & 0x0F0F0F0Fu; + if constexpr (QH) { + x |= ((p.h >> (4 * part + g)) & 0x01010101u) << 4; + } + return x; + } +}; + +// Q2_K reorder layout: [qs: nb*64] [scales: nb*16] [dm: nb*4] +// chunk s of 32: 2 bits from qs byte 32*(s/4)+l at shift 2*(s%4), each 16 wide half has a 4 bit scale and min +// value is d * (code & 15) * q - dmin * (code >> 4) +template <bool SR> struct xmx_traits_q2_k : xmx_traits_base<SR, 16, xmx_scale_kind::min16> { + struct ctx : xmx_ctx_base { + const uint8_t * qs; + const uint8_t * sc; + const sycl::half * dm; + unsigned pq, ps; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + const size_t nb = (size_t) nrows * bpr; + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.qs = (const uint8_t *) vx; + c.sc = c.qs + nb * (QK_K / 4); + c.dm = (const sycl::half *) (c.sc + nb * (QK_K / 16)); + c.pq = xmx_pitch(bpr, QK_K / 4); + c.ps = xmx_pitch(bpr, QK_K / 16); + return c; + } + + struct scales_t { + simd<uint8_t, 256> code; // 4 bit scale (low) and min (high) of each 16 wide half + simd<float, 16> d, dmin; + + ESIMD_INLINE simd<uint8_t, 16> byte(int gr) { return code.template select<16, 4>(64 * (gr / 4) + (gr % 4)); } + }; + + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + simd<uint32_t, 16> bi = rows * c.bpr + sb; + simd<uint32_t, 64> raw = xmx_load_t<4, SR>(c.sc, c.ps, c.h, sb * 4, c.row0); + s.code = raw.template bit_cast_view<uint8_t>(); + xmx_load_dm(c.dm, bi, s.d, s.dmin); + return s; + } + + struct part_t { + simd<uint32_t, 128> q; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; + p.q = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 16 + part * 8, c.row0); + return p; + } + + static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int /*part*/) { + return (p.q >> (2 * g)) & 0x03030303u; + } +}; + +// Q8_0 reorder layout: [qs: nrows*K bytes] [d: one half per 32 weights] +// the int8 weights feed DPAS as they are, a group is one block with one scale +template <bool SR> struct xmx_traits_q8_0 : xmx_traits_base<SR, 16, xmx_scale_kind::plain> { + struct ctx : xmx_ctx_base { + const uint8_t * qs; + const sycl::half * dh; + unsigned pq; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.qs = (const uint8_t *) vx; + c.dh = (const sycl::half *) (c.qs + (size_t) nrows * bpr * QK_K); + c.pq = xmx_pitch(bpr, QK_K); + return c; + } + + struct scales_t { + xmx_row_scales<8> d; // scale of each of the 8 blocks of a super-block + }; + + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + simd<uint32_t, 16> blk = (rows * c.bpr + sb) * (QK_K / QK8_0); +#pragma unroll + for (int j = 0; j < 8; ++j) { + s.d.v[j] = convert<float>(gather<sycl::half, 16>(c.dh, (blk + j) * (uint32_t) sizeof(sycl::half))); + } + return s; + } + + struct part_t { + simd<uint32_t, 128> q[4]; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; +#pragma unroll + for (int g = 0; g < 4; ++g) { + p.q[g] = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 64 + 8 * (4 * part + g), c.row0); + } + return p; + } + + static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int /*part*/) { return p.q[g]; } +}; + +template <bool SR> using xmx_traits_q4_k = xmx_traits_q45_k<false, SR>; +template <bool SR> using xmx_traits_q5_k = xmx_traits_q45_k<true, SR>; + +// registers per thread, the large file halves the resident threads but avoids spills in wide tiles, the fused +// kernel keeps two sets of weights and scales and always takes it +template <typename T, int NC, bool FUSED> constexpr int xmx_grf() { + return FUSED || NC >= T::nc_wide ? 256 : 128; +} + +#ifdef GGML_SYCL_MMVQ_HAS_XMX +// One DPAS of a block of RC columns: the int32 products of the weights B and the activations A, as float and scaled +// per weight row. Every quant type sums one or two of these per group of 32. +template <int RC, int NT> +ESIMD_INLINE simd<float, RC * NT> xmx_term(const simd<int8_t, 512> & B, + const simd<int8_t, RC * 32> & A, + const simd<float, NT> & s) { + using namespace sycl::ext::intel::esimd; + namespace xmx = sycl::ext::intel::esimd::xmx; + + simd<int, RC * NT> C = xmx::dpas<8, RC, int>(B, A); + return convert<float>(C) * s.template replicate<RC>(); +} + +// one group of 32 of one weight set, accumulated into Cf for all the columns of the tile +template <typename T, int NC, typename A2T, typename D8T> +ESIMD_INLINE void xmx_group(simd<float, NC * XMX_ROWS> & Cf, + const typename T::part_t & pd, + typename T::scales_t & sc, + A2T & A2, + D8T & d8f, + simd<int8_t, 512> & Bl, + simd<int8_t, 512> & Bh, + const simd<int8_t, 512> & Ol, + const int g, + const int part) { + using namespace sycl::ext::intel::esimd; + + constexpr int NT = XMX_ROWS; + constexpr int RC = NC < 8 ? NC : 8; + constexpr int NB = NC / RC; + constexpr bool split = !T::has_min && !T::plain && !T::min16; // two scales per group of 32 + constexpr bool pair = (RC * 32) % 64 == 0; // a block of RC rows fills whole registers + + const int gi = 4 * part + g; + + simd<uint32_t, 128> x = T::unpack(pd, g, part); + simd<int8_t, 512> B = x.template bit_cast_view<int8_t>(); + if constexpr (split) { + Bl.template select<256, 1>(0) = B.template select<256, 1>(0); + Bh.template select<256, 1>(256) = B.template select<256, 1>(256); + } + + simd<float, NT> rd; + simd<float, NT> rm; + simd<float, NT> rs0; + simd<float, NT> rs1; + + // Q2_K: scale and min of each 16 wide half are 4 bit codes, the weight is d * scl * q - dmin * mc. scl * q + // is at most 45, so a dword multiply keeps the bytes apart and the scaled weights go to DPAS as int8. The + // min term sums a * mc over K, the codes mc take the place of the weights in a second DPAS. + simd<int8_t, 512> Bs; + simd<int8_t, 512> Bm; + if constexpr (T::min16) { + const simd<uint8_t, NT> c0 = sc.byte(2 * gi); + const simd<uint8_t, NT> c1 = sc.byte(2 * gi + 1); + simd<uint32_t, NT> sl0 = convert<uint32_t>(simd<uint8_t, NT>(c0 & (uint8_t) 15)); + simd<uint32_t, NT> sl1 = convert<uint32_t>(simd<uint8_t, NT>(c1 & (uint8_t) 15)); + simd<uint32_t, NT> mc0 = convert<uint32_t>(simd<uint8_t, NT>(c0 >> (uint8_t) 4)) * 0x01010101u; + simd<uint32_t, NT> mc1 = convert<uint32_t>(simd<uint8_t, NT>(c1 >> (uint8_t) 4)) * 0x01010101u; + simd<uint32_t, 128> ws; + simd<uint32_t, 128> wm; + ws.template select<64, 1>(0) = x.template select<64, 1>(0) * sl0.template replicate<4>(); + ws.template select<64, 1>(64) = x.template select<64, 1>(64) * sl1.template replicate<4>(); + wm.template select<64, 1>(0) = mc0.template replicate<4>(); + wm.template select<64, 1>(64) = mc1.template replicate<4>(); + Bs = ws.template bit_cast_view<int8_t>(); + Bm = wm.template bit_cast_view<int8_t>(); + } + if constexpr (T::plain) { + rd = sc.d.get(gi); + } else if constexpr (T::has_min) { + rd = sc.d.get(gi); + rm = sc.m.get(gi); + } else if constexpr (T::min16) { + // scales are applied through Bs and Bm + } else { + rs0 = sc.get(2 * gi); + rs1 = sc.get(2 * gi + 1); + } + +#pragma unroll + for (int b = 0; b < NB; ++b) { + simd<int8_t, RC * 32> A = + A2[b].template select<RC * 8, 1>(pair ? (g % 2) * RC * 8 : 0).template bit_cast_view<int8_t>(); + simd<float, RC * NT> d8r = + d8f[b].template select<RC, RC == 1 ? 1 : 8>(gi).template replicate_vs_w_hs<RC, 1, NT, 0>(0); + auto Cb = Cf.template select<RC * NT, 1>(b * RC * NT); + + simd<float, RC * NT> z; + if constexpr (T::plain) { + z = xmx_term<RC, NT>(B, A, rd); + } else if constexpr (T::has_min) { + // the second DPAS (all ones against the activations) is the exact integer sum of the activations + z = xmx_term<RC, NT>(B, A, rd) - xmx_term<RC, NT>(Ol, A, rm); + } else if constexpr (T::min16) { + z = xmx_term<RC, NT>(Bs, A, sc.d) - xmx_term<RC, NT>(Bm, A, sc.dmin); + } else { + z = xmx_term<RC, NT>(Bl, A, rs0) + xmx_term<RC, NT>(Bh, A, rs1); + } + Cb += z * d8r; + } +} + +// activation of the GLU applied to the gate, uniform across the launch +template <int N> ESIMD_INLINE simd<float, N> xmx_glu_act(const simd<float, N> & gate, const int glu_op) { + using namespace sycl::ext::intel::esimd; + if (glu_op == GGML_GLU_OP_SWIGLU) { + return gate / (1.0f + exp(-gate)); + } + // GEGLU, the tanh is written with exp + simd<float, N> a = 0.79788456f * gate * (1.0f + 0.044715f * gate * gate); + simd<float, N> t = 1.0f - 2.0f / (exp(2.0f * a) + 1.0f); + return 0.5f * gate * (1.0f + t); +} + +// activations come in through 2D block loads, one for the (d, s) pairs of a super-block (s is unused) and one per two +// groups for the quants, the DPAS A operand needs no repacking. Groups with two scales (16 wide) mask the +// weights instead, zeros in B drop the other half of K. +// KS is the most threads that split K for one tile, the launch may use fewer. +// FUSED computes up * act(gate) from the weights vx (up) and vg (gate), reading the activations once. +template <typename T, int NC, int KS, bool FUSED> +ESIMD_INLINE void xmx_mul_mat(const void * vx, + const void * vg, + const char * vy, + float * dst, + const int ncols, + const int nrows, + const int ncv, + const int stride_y_bytes, + const int ldd, + const int glu_op, + const int tile, + const int lid, + const int ks) { + using namespace sycl::ext::intel::esimd; + + constexpr int NT = XMX_ROWS; + constexpr int RC = NC < 8 ? NC : 8; + constexpr int NB = NC / RC; + constexpr int N_ACC = FUSED ? 2 : 1; + constexpr bool pair = (RC * 32) % 64 == 0; // a block of RC rows fills whole registers + + if constexpr (KS > 1) { + slm_init<N_ACC * KS * NC * NT * sizeof(float)>(); + } + + const int bpr = ncols / QK_K; + const int row0 = tile * NT; + const auto c = T::make(vx, nrows, bpr, row0); + const auto cg = T::make(FUSED ? vg : vx, nrows, bpr, row0); + + simd<uint32_t, NT> rows; +#pragma unroll + for (int n = 0; n < NT; ++n) { + rows[n] = row0 + n < nrows ? row0 + n : nrows - 1; + } + + const uint32_t * yq = (const uint32_t *) vy; + const uint32_t * yd = (const uint32_t *) (vy + ncols); + const unsigned yqw = ncols - 1; + const unsigned ydw = ncols / QK8_1 * sizeof(sycl::half2) - 1; + const unsigned yh = ncv - 1; + const unsigned yp = stride_y_bytes - 1; + + simd<float, NC * NT> Cf = 0.0f; + simd<float, NC * NT> Cfg = 0.0f; // gate, only used when fused + simd<int8_t, 512> Bl = int8_t(0); + simd<int8_t, 512> Bh = int8_t(0); + simd<int8_t, 512> Ol = int8_t(0); + if constexpr (T::has_min) { + Ol = int8_t(1); + } + + for (int sb = lid; sb < bpr; sb += ks) { + auto sc = T::load_scales(c, sb, rows); + typename T::scales_t scg; + if constexpr (FUSED) { + scg = T::load_scales(cg, sb, rows); + } + + // d of the 8 groups of the super-block, [column * 8 + group] + simd<float, 8 * RC> d8f[NB]; +#pragma unroll + for (int b = 0; b < NB; ++b) { + simd<uint32_t, 8 * RC> ds = load_2d<uint32_t, 8, RC, 1, false, false>(yd, ydw, yh, yp, sb * 8, b * RC); + simd<sycl::half, 16 * RC> dh = ds.template bit_cast_view<sycl::half>(); + d8f[b] = convert<float>(simd<sycl::half, 8 * RC>(dh.template select<8 * RC, 2>(0))); + } + +#pragma unroll + for (int part = 0; part < 2; ++part) { + const auto pd = T::load_part(c, sb, part); + typename T::part_t pdg; + if constexpr (FUSED) { + pdg = T::load_part(cg, sb, part); + } + + simd<uint32_t, 16 * RC> A2[NB]; // quants of two groups + +#pragma unroll + for (int g = 0; g < 4; ++g) { + const int gi = 4 * part + g; + if constexpr (pair) { + if (g % 2 == 0) { +#pragma unroll + for (int b = 0; b < NB; ++b) { + A2[b] = + load_2d<uint32_t, 8, RC, 2, false, false>(yq, yqw, yh, yp, sb * 64 + gi * 8, b * RC); + } + } + } else { +#pragma unroll + for (int b = 0; b < NB; ++b) { + A2[b].template select<RC * 8, 1>(0) = + load_2d<uint32_t, 8, RC, 1, false, false>(yq, yqw, yh, yp, sb * 64 + gi * 8, b * RC); + } + } + + xmx_group<T, NC>(Cf, pd, sc, A2, d8f, Bl, Bh, Ol, g, part); + if constexpr (FUSED) { + xmx_group<T, NC>(Cfg, pdg, scg, A2, d8f, Bl, Bh, Ol, g, part); + } + } + } + } + + if constexpr (KS > 1) { + slm_block_store<float, NC * NT>(lid * NC * NT * sizeof(float), Cf); + if constexpr (FUSED) { + slm_block_store<float, NC * NT>((KS + lid) * NC * NT * sizeof(float), Cfg); + } + barrier(); + if (lid != 0) { + return; + } + Cf = 0.0f; + Cfg = 0.0f; +#pragma unroll 4 + for (int j = 0; j < ks; ++j) { + Cf += slm_block_load<float, NC * NT>(j * NC * NT * sizeof(float)); + if constexpr (FUSED) { + Cfg += slm_block_load<float, NC * NT>((KS + j) * NC * NT * sizeof(float)); + } + } + } + + if constexpr (FUSED) { + Cf *= xmx_glu_act<NC * NT>(Cfg, glu_op); + } + + if (row0 + NT <= nrows && ldd % 4 == 0 && (uintptr_t) dst % 16 == 0) { +#pragma unroll + for (int m = 0; m < NC; ++m) { + if (NC <= 8 || m < ncv) { + block_store<float, NT>(dst + (size_t) m * ldd + row0, Cf.template select<NT, 1>(m * NT)); + } + } + } else { + // tail tile or unaligned output + simd<uint32_t, NT> lane(0, 1); + simd_mask<NT> in_rows = (lane + row0) < (uint32_t) nrows; +#pragma unroll + for (int m = 0; m < NC; ++m) { + if (NC <= 8 || m < ncv) { + scatter<float, NT>(dst + (size_t) m * ldd + row0, lane * (uint32_t) sizeof(float), + Cf.template select<NT, 1>(m * NT), in_rows); + } + } + } +} + +// KS_MAX bounds the threads that split K for one tile, the launch uses as few as keep every thread to the same +// number of super-blocks as the busiest one +template <typename T, int NC, int KS_MAX, bool FUSED> +static void xmx_mul_mat_launch_ks(const void * vx, + const void * vg, + const char * vy, + float * dst, + const int ncols, + const int nrows, + const int ncv, + const int stride_y_bytes, + const int ldd, + const int glu_op, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const size_t n_tiles = (nrows + XMX_ROWS - 1) / XMX_ROWS; + const int bpr = ncols / QK_K; + const int rounds = (bpr + KS_MAX - 1) / KS_MAX; + const int ks = (bpr + rounds - 1) / rounds; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(n_tiles * ks), sycl::range<1>(ks)), + sycl::ext::oneapi::experimental::properties{ + sycl::ext::intel::experimental::grf_size<xmx_grf<T, NC, FUSED>()> }, + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + xmx_mul_mat<T, NC, KS_MAX, FUSED>(vx, vg, vy, dst, ncols, nrows, ncv, stride_y_bytes, ldd, + glu_op, (int) it.get_group(0), (int) it.get_local_id(0), + ks); + }); + }); +} + +template <typename T, int NC, bool FUSED> +static void xmx_mul_mat_launch(const void * vx, + const void * vg, + const char * vy, + float * dst, + const int ncols, + const int nrows, + const int ncv, + const int stride_y_bytes, + const int ldd, + const int glu_op, + dpct::queue_ptr stream) { + // the fused kernel keeps two sets of accumulators in shared memory, half the threads for the same size + constexpr int ks_max = xmx_n_split<NC>() / (FUSED ? 2 : 1); + xmx_mul_mat_launch_ks<T, NC, ks_max, FUSED>(vx, vg, vy, dst, ncols, nrows, ncv, stride_y_bytes, ldd, glu_op, + stream); +} + +// FUSED: vx and vg are the up and gate weights, dst gets up * act(gate) with glu_op a ggml_glu_op +template <typename T, bool FUSED = false> +static void xmx_mul_mat_ncols(const void * vx, + const void * vg, + const char * vy, + float * dst, + const int ncols, + const int nrows, + const int ncols_dst, + const int stride_y_bytes, + const int ldd, + const int glu_op, + dpct::queue_ptr stream) { + constexpr int chunk_max = FUSED ? XMX_FUSED_CHUNK : XMX_CHUNK; + for (int c0 = 0; c0 < ncols_dst; c0 += chunk_max) { + const int n = std::min(chunk_max, ncols_dst - c0); + const char * y = vy + (size_t) c0 * stride_y_bytes; + float * d = dst + (size_t) c0 * ldd; + const int nc = n > 32 ? 64 : (n > 16 ? 32 : (n > 8 ? 16 : n)); + switch (nc) { +#define XMX_CASE(N) \ + case N: \ + xmx_mul_mat_launch<T, N, FUSED>(vx, vg, y, d, ncols, nrows, n, stride_y_bytes, ldd, glu_op, stream); \ + break + XMX_CASE(1); + XMX_CASE(2); + XMX_CASE(3); + XMX_CASE(4); + XMX_CASE(5); + XMX_CASE(6); + XMX_CASE(7); + XMX_CASE(8); +#undef XMX_CASE + // short row variants are for tests, not built for the wide tiles +#define XMX_CASE_WIDE(N) \ + case N: \ + if constexpr (!T::short_rows && (!FUSED || N <= XMX_FUSED_CHUNK)) { \ + xmx_mul_mat_launch<T, N, FUSED>(vx, vg, y, d, ncols, nrows, n, stride_y_bytes, ldd, glu_op, stream); \ + } \ + break + XMX_CASE_WIDE(16); + XMX_CASE_WIDE(32); + XMX_CASE_WIDE(64); +#undef XMX_CASE_WIDE + default: + GGML_ABORT("unsupported ncols_dst=%d for XMX MMV", n); + } + } +} + +#endif // GGML_SYCL_MMVQ_HAS_XMX + } // namespace ggml_sycl_esimd #endif // GGML_SYCL_ESIMD_HPP diff --git a/ggml/src/ggml-sycl/fattn-buffers.cpp b/ggml/src/ggml-sycl/fattn-buffers.cpp deleted file mode 100644 index 78a52d2ab..000000000 --- a/ggml/src/ggml-sycl/fattn-buffers.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// MIT license -// Copyright (C) 2025 Intel Corporation -// SPDX-License-Identifier: MIT -// - -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// - -#include "common.hpp" - -sycl::half * ggml_sycl_fattn_kv_buffers::kv_buffer::ensure_half(size_t n_elems) { - const size_t need_bytes = n_elems * sizeof(sycl::half); - - if (capacity >= need_bytes) { - return ptr; - } - - if (ptr) { - SYCL_CHECK(CHECK_TRY_ERROR(qptr->wait())); - ggml_sycl_memtrace_del(ptr); - SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr))); - ptr = nullptr; - capacity = 0; - } - - size_t cap = 0; - while (cap < need_bytes) { - cap += CHUNK_SIZE; - } - - void * dev_ptr; - SYCL_CHECK( - CHECK_TRY_ERROR(dev_ptr = sycl::malloc_device( - cap, *qptr))); - - if (!dev_ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, cap); - ggml_sycl_memtrace_fail(GGML_SYCL_MEM_FATTN_KV, cap); - GGML_ABORT("fattn buffer alloc failed"); - } - - ptr = static_cast<sycl::half *>(dev_ptr); - capacity = cap; - ggml_sycl_memtrace_add(GGML_SYCL_MEM_FATTN_KV, ptr, cap); - return ptr; -} - -ggml_sycl_fattn_kv_buffers::kv_buffer::~kv_buffer() { -#ifdef DEBUG_SYCL_POOL - GGML_LOG_INFO("ggml_sycl_fattn_kv_buffer[%d]: %.2f MiB\n", device, capacity / 1024.0 / 1024.0); -#endif - if (ptr) { - ggml_sycl_memtrace_del(ptr); - SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr))); - } -} diff --git a/ggml/src/ggml-sycl/fattn-buffers.hpp b/ggml/src/ggml-sycl/fattn-buffers.hpp deleted file mode 100644 index c00461de6..000000000 --- a/ggml/src/ggml-sycl/fattn-buffers.hpp +++ /dev/null @@ -1,63 +0,0 @@ -// -// MIT license -// Copyright (C) 2025 Intel Corporation -// SPDX-License-Identifier: MIT -// - -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// - -#ifndef GGML_SYCL_FATTN_BUFFERS_HPP -#define GGML_SYCL_FATTN_BUFFERS_HPP - -#include <sycl/sycl.hpp> - -typedef sycl::queue *queue_ptr; - -struct ggml_sycl_fattn_kv_buffers { - // buffers grow in chunks of this size - static constexpr size_t CHUNK_SIZE = 16ull << 20; // 16 MiB - - struct kv_buffer { - kv_buffer(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {} - ~kv_buffer(); - - kv_buffer(const kv_buffer &) = delete; - kv_buffer & operator=(const kv_buffer &) = delete; - - sycl::half * ensure_half(size_t n_elems); - - private: - sycl::half * ptr = nullptr; - size_t capacity = 0; - queue_ptr qptr = nullptr; - [[maybe_unused]] int device = 0; - }; - - kv_buffer K; - kv_buffer V; - - ggml_sycl_fattn_kv_buffers(queue_ptr qptr, int device) : K(qptr, device), V(qptr, device) {} - - ggml_sycl_fattn_kv_buffers(const ggml_sycl_fattn_kv_buffers &) = delete; - ggml_sycl_fattn_kv_buffers & operator=(const ggml_sycl_fattn_kv_buffers &) = delete; -}; - -/** - * Imitates `ggml_sycl_pool_alloc` to keep the code calling alloc unchanged. - */ -struct ggml_sycl_fattn_alloc { - ggml_sycl_fattn_kv_buffers::kv_buffer & buf; - sycl::half * ptr = nullptr; - - explicit ggml_sycl_fattn_alloc(ggml_sycl_fattn_kv_buffers::kv_buffer & buf_) : buf(buf_) {} - - sycl::half * alloc(size_t n_elems) { - ptr = buf.ensure_half(n_elems); - return ptr; - } -}; -#endif diff --git a/ggml/src/ggml-sycl/fattn-common.hpp b/ggml/src/ggml-sycl/fattn-common.hpp index 3c2d1a776..c37326336 100644 --- a/ggml/src/ggml-sycl/fattn-common.hpp +++ b/ggml/src/ggml-sycl/fattn-common.hpp @@ -6,7 +6,6 @@ #include "common.hpp" #include "convert.hpp" #include "vecdotq.hpp" -#include "fattn-buffers.hpp" #include "fattn.hpp" #include "ggml.h" @@ -833,6 +832,29 @@ static void flash_attn_combine_results(const float * __restrict__ VKQ_parts, dst[tid] = VKQ_numerator / VKQ_denominator; } +// Carries the sub-group size (and grf_size_automatic when use_large_grf) as +// kernel properties via get(properties_tag), in place of the +// reqd_sub_group_size attribute and the deprecated parallel_for overload. +template <typename KernelFunc, uint32_t Size, bool UseGrf> +struct props_kernel { + KernelFunc kernel; + + void operator()(sycl::nd_item<3> item) const { + kernel(item); + } + + auto get(sycl::ext::oneapi::experimental::properties_tag) const { + if constexpr (UseGrf) { + return sycl::ext::oneapi::experimental::properties{ + sycl::ext::intel::experimental::grf_size_automatic, + sycl::ext::oneapi::experimental::sub_group_size<Size> }; + } else { + return sycl::ext::oneapi::experimental::properties{ + sycl::ext::oneapi::experimental::sub_group_size<Size> }; + } + } +}; + template <fattn_kernel_t fattn_kernel, int warp_size, bool use_large_grf = false> static void lauch_kernel( dpct::dim3 group_range, @@ -882,7 +904,7 @@ static void lauch_kernel( static_cast<sycl::range<3>>(group_range * local_range), static_cast<sycl::range<3>>(local_range)); - const auto kernel = [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] { + const auto kernel = [=](sycl::nd_item<3> item_ct1) { GGML_UNUSED(item_ct1); fattn_kernel(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, @@ -893,14 +915,7 @@ static void lauch_kernel( q->submit([&](sycl::handler &cgh) { // grf_size_automatic lets the compiler use the large register file when the kernel needs it. grf_size<256> would say the same but is undefined on devices that do not have it. - if constexpr (use_large_grf) { - cgh.parallel_for( - rng, - sycl::ext::oneapi::experimental::properties{ sycl::ext::intel::experimental::grf_size_automatic }, - kernel); - } else { - cgh.parallel_for(rng, kernel); - } + cgh.parallel_for(rng, props_kernel<decltype(kernel), warp_size, use_large_grf>{kernel}); }); } @@ -933,13 +948,10 @@ void launch_fattn( GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16); ggml_sycl_pool & pool = ctx.pool(); - ggml_sycl_fattn_kv_buffers & fbuf = ctx.fattn_buffers(); dpct::queue_ptr main_stream = ctx.stream(); const int id = ggml_sycl_get_device(); const int nsm = ggml_sycl_info().devices[id].nsm; - ggml_sycl_fattn_alloc K_f16(fbuf.K); - ggml_sycl_fattn_alloc V_f16(fbuf.V); const ggml_sycl_fattn_extra extra = ggml_sycl_fattn_get_extra(dst); ggml_sycl_pool_alloc<int> KV_max(pool); ggml_sycl_pool_alloc<float> dst_tmp(pool); @@ -959,8 +971,8 @@ void launch_fattn( const size_t bs = ggml_blck_size(K->type); const size_t ts = ggml_type_size(K->type); - sycl::half * K_f16_ptr = extra.K_buffer_ptr ? (sycl::half *) extra.K_buffer_ptr - : K_f16.alloc(ggml_nelements(K)); + GGML_ASSERT(extra.K_buffer_ptr); + sycl::half * K_f16_ptr = (sycl::half *) extra.K_buffer_ptr; if (ggml_is_contiguously_allocated(K)) { to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(K->type, dst); to_fp16(K_data, K_f16_ptr, ggml_nelements(K), main_stream); @@ -993,8 +1005,8 @@ void launch_fattn( const size_t bs = ggml_blck_size(V->type); const size_t ts = ggml_type_size(V->type); - sycl::half * V_f16_ptr = extra.V_buffer_ptr ? (sycl::half *) extra.V_buffer_ptr - : V_f16.alloc(ggml_nelements(V)); + GGML_ASSERT(extra.V_buffer_ptr); + sycl::half * V_f16_ptr = (sycl::half *) extra.V_buffer_ptr; if (ggml_is_contiguously_allocated(V)) { to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(V->type, dst); to_fp16(V_data, V_f16_ptr, ggml_nelements(V), main_stream); diff --git a/ggml/src/ggml-sycl/fattn-dpas.cpp b/ggml/src/ggml-sycl/fattn-dpas.cpp new file mode 100644 index 000000000..fb0b3a816 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-dpas.cpp @@ -0,0 +1,722 @@ +#include <sycl/sycl.hpp> +#include <sycl/ext/intel/experimental/grf_size_properties.hpp> +#include "dpct/helper.hpp" +#include "common.hpp" +#include "fattn-common.hpp" +#include "fattn-dpas.hpp" + +// GGML_SYCL_SUPPORT_XMX=OFF builds skip the DPAS below, as do spir64_gen builds for sub-group 8 +// targets (Xe-HPG) or mixed target lists, which cannot compile it +#if !defined(GGML_SYCL_SUPPORT_XMX) || (defined(GGML_SYCL_XMX_AOT_SG) && GGML_SYCL_XMX_AOT_SG != 16) +#define GGML_SYCL_FA_DPAS_BUILD 0 +#else +#define GGML_SYCL_FA_DPAS_BUILD 1 +#endif + +#if GGML_SYCL_FA_DPAS_BUILD +static constexpr int FA_DPAS_WG = 256; // work-group size +static constexpr int FA_DPAS_NSG = 16; // sub-groups of 16 lanes +static constexpr int FA_DPAS_MAX_Q = 31; // larger batches go to the prefill kernels +static constexpr int FA_DPAS_MAX_G = 32; // query heads per KV head +static constexpr float FA_DPAS_MINF = -1.0e30f; // finite start value of the running max + +static bool fattn_dpas_support_arch(gpu_arch arch) { + return arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31 || + arch == gpu_arch::intel_gpu_lnl_m || arch == gpu_arch::intel_gpu_ptl_h || + arch == gpu_arch::intel_gpu_ptl_u || arch == gpu_arch::intel_gpu_wcl; +} + +// head sizes with a compiled kernel (multiples of the 32-dim q4_0 / q8_0 block) +static bool fattn_dpas_head_ok(int64_t d) { + return d == 64 || d == 96 || d == 128 || d == 256 || d == 512; +} +#endif // GGML_SYCL_FA_DPAS_BUILD + +bool ggml_sycl_flash_attn_ext_dpas_supported(int device, const ggml_tensor * dst) { +#if !GGML_SYCL_FA_DPAS_BUILD + GGML_UNUSED(device); + GGML_UNUSED(dst); + return false; +#else + if (!g_ggml_sycl_fa_dpas || !fattn_dpas_support_arch(ggml_sycl_info().devices[device].hw_info.arch)) { + return false; + } + + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + + if (Q->type != GGML_TYPE_F32 || (K->type != GGML_TYPE_Q4_0 && K->type != GGML_TYPE_Q8_0) || V->type != K->type || + (sinks && sinks->type != GGML_TYPE_F32)) { + return false; + } + if (!fattn_dpas_head_ok(K->ne[0]) || V->ne[0] != K->ne[0] || Q->ne[0] != K->ne[0]) { + return false; + } + if (Q->ne[1] < 1 || Q->ne[1] > FA_DPAS_MAX_Q || K->ne[1] < 1) { + return false; + } + if (K->ne[2] < 1 || V->ne[2] != K->ne[2] || Q->ne[2] % K->ne[2] != 0 || Q->ne[2] / K->ne[2] > FA_DPAS_MAX_G) { + return false; + } + if (Q->ne[3] != K->ne[3] || V->ne[3] != K->ne[3]) { + return false; + } + + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + if (max_bias != 0.0f || logit_softcap != 0.0f) { + return false; + } + + if (mask && (mask->type != GGML_TYPE_F16 || mask->ne[2] != 1 || mask->nb[0] != sizeof(sycl::half) || + mask->ne[0] < K->ne[1] || mask->ne[1] < Q->ne[1])) { + return false; + } + if (sinks && (sinks->nb[0] != sizeof(float) || sinks->ne[0] < Q->ne[2])) { + return false; + } + if (Q->nb[0] != sizeof(float)) { + return false; + } + // q4_0 / q8_0 rows are read as dwords from the dword at or before the row start: rows must start on 2-byte bounds + // (head sizes with an odd block count, e.g. 96, put every other head 2 bytes into a dword) + // so a row's first/last dword may hold 2 bytes outside it; aligned dwords never cross the 128-byte buffer alignment + for (const ggml_tensor * t : { K, V }) { + if (t->nb[0] != ggml_type_size(t->type) || t->nb[1] % 2 != 0 || t->nb[2] % 2 != 0 || t->nb[3] % 2 != 0 || + (uintptr_t) t->data % 2 != 0) { + return false; + } + } + return true; +#endif +} + +#if GGML_SYCL_FA_DPAS_BUILD + +typedef short fa_dpas_short8 __attribute__((ext_vector_type(8))); +typedef int fa_dpas_int4 __attribute__((ext_vector_type(4))); +typedef int fa_dpas_int8 __attribute__((ext_vector_type(8))); +typedef float fa_dpas_float8 __attribute__((ext_vector_type(8))); + +// Xe2 DPAS (SIMD16, systolic depth 8, repeat count 8) as inline vISA: joint_matrix has no s4 type. +// d = a x b. a: 8 rows x 32 s8, 2 per lane per row. b: 32 x 16 s4, 16 bytes per lane, one column per lane. +static inline void fa_dpas_s8_s4(const fa_dpas_short8 & a, const fa_dpas_int4 & b, fa_dpas_int8 & d) { +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl DB v_type=G type=ud num_elts=64 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts=64 align=GRF alias=<%2,0>\n" + "dpas.s4.s8.8.8 (M1, 16) %0.0 %%null.0 DB.0 DA(0,0)\n" + "}\n" + : "=rw"(d) + : "rw"(b), "rw"(a)); +#else + d = 0; + GGML_UNUSED(a); + GGML_UNUSED(b); +#endif +} + +// d = a x b. a: 8 rows x 32 s8, 2 per lane per row. b: 32 x 16 s8, 32 bytes per lane, one column per lane. +static inline void fa_dpas_s8_s8(const fa_dpas_short8 & a, const fa_dpas_int8 & b, fa_dpas_int8 & d) { +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl DB v_type=G type=ud num_elts=128 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts=64 align=GRF alias=<%2,0>\n" + "dpas.s8.s8.8.8 (M1, 16) %0.0 %%null.0 DB.0 DA(0,0)\n" + "}\n" + : "=rw"(d) + : "rw"(b), "rw"(a)); +#else + d = 0; + GGML_UNUSED(a); + GGML_UNUSED(b); +#endif +} + +// acc += a x b. a: 8 rows x 16 f16, one per lane per row. b: 16 x 16 f16 as pairs, 8 dwords per lane, one column per lane. +static inline void fa_dpas_f16(const fa_dpas_short8 & a, const fa_dpas_int8 & b, fa_dpas_float8 & acc) { +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl DB v_type=G type=ud num_elts=128 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts=64 align=GRF alias=<%2,0>\n" + "dpas.hf.hf.8.8 (M1, 16) %0.0 %0.0 DB.0 DA(0,0)\n" + "}\n" + : "+rw"(acc) + : "rw"(b), "rw"(a)); +#else + GGML_UNUSED(a); + GGML_UNUSED(b); + GGML_UNUSED(acc); +#endif +} + +// Block b of a row = f16 scale + NQ dwords of quants (q4_0: 18 bytes, NQ 4; q8_0: 34 bytes, NQ 8), at byte (2 + 4 NQ) b. +// w = the NQ + 1 dwords from that byte / 4. An odd block starts 2 bytes into w[0]. +template <int NQ> static inline void fa_dpas_split_block(const uint32_t * w, bool odd, uint32_t & sc, uint32_t * q) { + if (!odd) { + sc = w[0] & 0xFFFFu; +#pragma unroll + for (int i = 0; i < NQ; ++i) { + q[i] = (w[i] >> 16) | (w[i + 1] << 16); + } + } else { + sc = w[0] >> 16; +#pragma unroll + for (int i = 0; i < NQ; ++i) { + q[i] = w[i + 1]; + } + } +} + +struct fa_dpas_params { + const char * Q; + const char * K; + const char * V; + const char * mask; + float * dst; + float * parts; // per-split partial results, nsplit > 1 only + sycl::float2 * meta; // per-split (max, sum), nsplit > 1 only + float scale; + const float * sinks; // per query head softmax sink logit, or null + int G; // query heads per KV head + int nq; // query tokens per work-group + int nqc; // token chunks + int ne01; + int ne02; + int ne11; + int ne33; + int nsplit; + int chunk; // keys per split + int64_t nb01, nb02, nb03; + int64_t nb11, nb12, nb13; + int64_t nb21, nb22, nb23; + int64_t nb31, nb33; +}; + +// One work-group = one KV head x nq query tokens x one slice of the keys: S = Q K^T by s8 x s4 / s8 x s8 DPAS on the +// raw K blocks, online softmax, O^T += V^T P^T by f16 DPAS. ST: stage K/V tiles in SLM (1: 16-byte, 2: dword loads). +// LG: 256-register file (no spills at 32 rows, same occupancy). Q8: K/V are q8_0, else q4_0. +template <int D, int NR16, int KG, int TC, int ST, bool LG, bool Q8> +static void fattn_dpas_kernel(const fa_dpas_params p, const int nkvh, const int ne03, dpct::queue_ptr stream) { + constexpr int RMAX = 16 * NR16; + constexpr int RTMAX = RMAX / 8; // 8-row tiles in Q.K + constexpr int TK = 16 * KG; // keys per tile + static_assert(QK4_0 == QK8_0, "q4_0 and q8_0 blocks hold the same number of dims"); + constexpr int BB = Q8 ? (int) sizeof(block_q8_0) : (int) sizeof(block_q4_0); // bytes per block + constexpr int NQ = (BB - 2) / 4; // quant dwords per block + constexpr int NB = D / QK4_0; // blocks per row + constexpr int DPS = D <= 128 ? 8 : D <= 256 ? 16 : 32; // head dims per sub-group in P.V, inside one 32-dim block + constexpr int NPV = D / DPS; // sub-groups doing P.V, the rest only reach its barrier + constexpr bool ALLPV = NPV == FA_DPAS_NSG; // every sub-group does P.V: no runtime check in the loops + constexpr int XT = DPS / 8; // 8-dim tiles per sub-group in P.V + constexpr int PS = TK + 16; // f16 row stride of P + constexpr int RW = NB * BB / 4; // dwords per row + constexpr bool ODD = (NB * BB) % 4 != 0; // odd block count: rows may start 2 bytes into a dword + constexpr int SW = RW + 1; // odd SLM row stride: no bank conflicts for lane = key + static_assert(D % QK4_0 == 0 && D % DPS == 0 && NPV <= FA_DPAS_NSG && QK4_0 % DPS == 0, "unsupported head size"); + static_assert(ST != 1 || RW % 4 == 0, "16-byte staging needs rows of 16-byte multiples"); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor<fa_dpas_short8, 1> sQ8(sycl::range<1>(RTMAX * NB * 16), cgh); // Q s8 [tile][block][lane] + sycl::local_accessor<fa_dpas_float8, 1> sDQ(sycl::range<1>(RTMAX * NB), cgh); // Q scales [tile][block] + sycl::local_accessor<float, 1> sS(sycl::range<1>(RMAX * TK), cgh); // S [row][key] + sycl::local_accessor<fa_dpas_int8, 1> sP(sycl::range<1>(RMAX * PS / 16), cgh); // P f16 [row][key] + sycl::local_accessor<float, 1> sM(sycl::range<1>(RMAX), cgh); + sycl::local_accessor<float, 1> sL(sycl::range<1>(RMAX), cgh); + sycl::local_accessor<float, 1> sA(sycl::range<1>(RMAX), cgh); + sycl::local_accessor<uint32_t, 1> sKV(sycl::range<1>(ST ? TK * SW : 1), cgh); + const sycl::nd_range<3> ndr(sycl::range<3>((size_t) ne03 * p.nqc, nkvh, (size_t) p.nsplit * FA_DPAS_WG), + sycl::range<3>(1, 1, FA_DPAS_WG)); + auto kern = [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0) / p.nqc; + const int c0 = (it.get_group(0) % p.nqc) * p.nq; + const int G = p.G; + const int R = p.nq * G; + const int RT = (R + 7) / 8; + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + short * sQs = (short *) &sQ8[0]; + short * sPs = (short *) &sP[0]; + + // Q -> s8, one scale per row and block. Lane l holds the 2 dims of K column elements 2 l, 2 l + 1: + // q4_0 packs dims l and l + 16 into byte l, q8_0 stores dims in order + for (int e = tid; e < RT * 8 * NB; e += FA_DPAS_WG) { + const int r = e / NB; + const int b = e % NB; + const int c = c0 + r / G; + const bool real = r < R && c < p.ne01; + const float * qr = (const float *) (p.Q + seq * p.nb03 + (int64_t) (real ? c : 0) * p.nb01 + + (int64_t) (kvh * G + (real ? r % G : 0)) * p.nb02) + b * QK4_0; + float x[QK4_0]; + float amax = 0.0f; +#pragma unroll + for (int i = 0; i < QK4_0; ++i) { + x[i] = real ? qr[i] * p.scale : 0.0f; + amax = sycl::fmax(amax, sycl::fabs(x[i])); + } + const float dq = amax / 127.0f; + const float iq = amax > 0.0f ? 127.0f / amax : 0.0f; + const int t = r / 8; + const int m = r % 8; + ((float *) &sDQ[0])[(t * NB + b) * 8 + m] = dq; +#pragma unroll + for (int l = 0; l < 16; ++l) { + const int lo = (int) sycl::rint(x[Q8 ? 2 * l : l] * iq); + const int hi = (int) sycl::rint(x[Q8 ? 2 * l + 1 : l + 16] * iq); + sQs[((t * NB + b) * 16 + l) * 8 + m] = (short) ((lo & 0xFF) | ((hi & 0xFF) << 8)); + } + } + // rows without a query keep P = 0 + for (int e = tid; e < RMAX * PS; e += FA_DPAS_WG) { + sPs[e] = 0; + } + if (tid < RMAX) { + // the sink is one more key with logit sinks[head] and no value; the first key slice owns it + const bool sink = p.sinks && split == 0 && tid < R; + sM[tid] = sink ? p.sinks[kvh * G + tid % G] : FA_DPAS_MINF; + sL[tid] = sink ? 1.0f : 0.0f; + sA[tid] = 1.0f; + } + it.barrier(sycl::access::fence_space::local_space); + + const int k_begin = split * p.chunk; + const int k_end = sycl::min(k_begin + p.chunk, p.ne11); + const char * Kh = p.K + seq * p.nb13 + kvh * p.nb12; + const char * Vh = p.V + seq * p.nb23 + kvh * p.nb22; + const sycl::half * mrow = p.mask ? (const sycl::half *) (p.mask + (seq % p.ne33) * p.nb33) : nullptr; + + // O^T: lane = query row, element m = head dim + fa_dpas_float8 o[NR16][XT]; +#pragma unroll + for (int t = 0; t < NR16; ++t) { +#pragma unroll + for (int x = 0; x < XT; ++x) { + o[t][x] = 0.0f; + } + } + // the dims of this sub-group lie in one V block + const int vb = (DPS * w) / QK4_0; + const int vd0 = (DPS * w) % QK4_0; + + // keys past k_end repeat key k_begin, their scores are -inf + auto stage = [&](const char * base, int64_t nb, int t0) { + if constexpr (ST == 2) { + for (int e = tid; e < TK * RW; e += FA_DPAS_WG) { + const int key = e / RW; + const int q = e - key * RW; + const int j = t0 + key < k_end ? t0 + key : k_begin; + sKV[key * SW + q] = ((const uint32_t *) (base + (int64_t) j * nb))[q]; + } + return; + } + constexpr int RV = ST == 1 ? RW / 4 : 1; + constexpr int NV = TK * RV; + constexpr int PER = (NV + FA_DPAS_WG - 1) / FA_DPAS_WG; + sycl::uint4 v[PER]; +#pragma unroll + for (int i = 0; i < PER; ++i) { + const int e = tid + i * FA_DPAS_WG; + if (e < NV) { + const int key = e / RV; + const int q = e - key * RV; + const int j = t0 + key < k_end ? t0 + key : k_begin; + v[i] = ((const sycl::uint4 *) (base + (int64_t) j * nb))[q]; + } + } +#pragma unroll + for (int i = 0; i < PER; ++i) { + const int e = tid + i * FA_DPAS_WG; + if (e < NV) { + const int key = e / RV; + const int q = e - key * RV; + const int o = key * SW + 4 * q; + sKV[o + 0] = v[i][0]; + sKV[o + 1] = v[i][1]; + sKV[o + 2] = v[i][2]; + sKV[o + 3] = v[i][3]; + } + } + }; + + // the NQ + 1 dwords of block b: of tile key kk in SLM, or of a row that starts 2 ph bytes into dword row[0] + auto load_block = [&](const uint32_t * row, int ph, int kk, int b, uint32_t * wv) { + const int wd = (2 * ph + BB * b) / 4; + if constexpr (ST) { + GGML_UNUSED(row); +#pragma unroll + for (int i = 0; i < NQ + 1; ++i) { + wv[i] = sKV[kk * SW + wd + i]; + } + } else { + GGML_UNUSED(kk); +#pragma unroll + for (int i = 0; i < NQ + 1; ++i) { + wv[i] = row[wd + i]; + } + } + }; + + // start loading the rows of keys t0 .. t0 + TK - 1 into L1 while the current phase computes + auto prefetch_rows = [&](const char * base, int64_t nb, int t0) { + constexpr int NL = RW * 4 / 64 + 1; // cache lines per row + for (int e = tid; e < TK * NL; e += FA_DPAS_WG) { + const int j = t0 + e / NL; + if (j < k_end) { + char * pa = const_cast<char *>(base) + (int64_t) j * nb + (e % NL) * 64; + sycl::ext::oneapi::experimental::prefetch(pa, sycl::ext::oneapi::experimental::properties{ sycl::ext::oneapi::experimental::prefetch_hint_L1 }); + } + } + }; + + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + if constexpr (ST) { + stage(Kh, p.nb11, t0); + it.barrier(sycl::access::fence_space::local_space); + } + prefetch_rows(Vh, p.nb21, t0); + + // S = Q K^T, job = (key group, TC row tiles) + const int nch = (RT + TC - 1) / TC; + for (int job = w; job < KG * nch; job += FA_DPAS_NSG) { + const int kg = job % KG; + const int ch = job / KG; + const int j = t0 + kg * 16 + lane; + const bool jv = j < k_end; + const char * kb = Kh + (int64_t) (jv ? j : k_begin) * p.nb11; + const int kp = (ST || !ODD) ? 0 : (int) (((uintptr_t) kb >> 1) & 1); // row starts 2 bytes into a dword + const uint32_t * kr = (const uint32_t *) (kb - 2 * kp); + fa_dpas_float8 s[TC]; +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + s[tt] = 0.0f; + } +#pragma unroll + for (int b = 0; b < NB; ++b) { + uint32_t wv[NQ + 1]; + uint32_t sc; + uint32_t q[NQ]; + load_block(kr, kp, kg * 16 + lane, b, wv); + fa_dpas_split_block<NQ>(wv, (b + kp) % 2 != 0, sc, q); + const float dk = static_cast<float>(sycl::bit_cast<sycl::half>((uint16_t) sc)); +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + const int t = ch * TC + tt; + if (t < RT) { + fa_dpas_int8 ia; + if constexpr (Q8) { + const fa_dpas_int8 bv = { (int) q[0], (int) q[1], (int) q[2], (int) q[3], + (int) q[4], (int) q[5], (int) q[6], (int) q[7] }; + fa_dpas_s8_s8(sQ8[(t * NB + b) * 16 + lane], bv, ia); + } else { + // u4 (n + 8) -> s4 n + const fa_dpas_int4 bv = { (int) (q[0] ^ 0x88888888u), (int) (q[1] ^ 0x88888888u), + (int) (q[2] ^ 0x88888888u), (int) (q[3] ^ 0x88888888u) }; + fa_dpas_s8_s4(sQ8[(t * NB + b) * 16 + lane], bv, ia); + } + const fa_dpas_float8 ds = sDQ[t * NB + b] * dk; + s[tt] += __builtin_convertvector(ia, fa_dpas_float8) * ds; + } + } + } +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + const int t = ch * TC + tt; + if (t < RT) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int r = t * 8 + m; + const int c = c0 + r / G; + if (r < R) { + const bool ok = jv && c < p.ne01; + const float mv = (ok && mrow) ? static_cast<float>(mrow[((int64_t) c * p.nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f; + sS[r * TK + kg * 16 + lane] = ok ? s[tt][m] + mv : -INFINITY; + } + } + } + } + } + it.barrier(sycl::access::fence_space::local_space); + if constexpr (ST) { + stage(Vh, p.nb21, t0); // the softmax does not use sKV + } + + // online softmax per row -> P + for (int r = w; r < R; r += FA_DPAS_NSG) { + float sv[KG]; + float mt = -INFINITY; +#pragma unroll + for (int i = 0; i < KG; ++i) { + sv[i] = sS[r * TK + i * 16 + lane]; + mt = sycl::fmax(mt, sv[i]); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum<float>()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); + float lt = 0.0f; +#pragma unroll + for (int i = 0; i < KG; ++i) { + const float pv = sycl::native::exp(sv[i] - m_new); + sPs[r * PS + i * 16 + lane] = (short) sycl::bit_cast<uint16_t>(sycl::half(pv)); + lt += pv; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus<float>()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); + + prefetch_rows(Kh, p.nb11, t0 + TK); + + // O^T = diag(alpha) O^T + V^T P^T. A = V^T (8 dims x 16 keys, lane = key), B = P^T (16 keys x 16 rows) +#pragma unroll + for (int t = 0; t < NR16; ++t) { + const float a = sA[t * 16 + lane]; +#pragma unroll + for (int x = 0; x < XT; ++x) { + o[t][x] *= a; + } + } +#pragma unroll 1 + for (int kg = 0; kg < ((ALLPV || w < NPV) ? KG : 0); ++kg) { + const int j = t0 + kg * 16 + lane; + const bool jv = j < k_end; + const char * vbp = Vh + (int64_t) (jv ? j : k_begin) * p.nb21; + const int vp = (ST || !ODD) ? 0 : (int) (((uintptr_t) vbp >> 1) & 1); + const uint32_t * vr = (const uint32_t *) (vbp - 2 * vp); + uint32_t wv[NQ + 1]; + uint32_t sc; + uint32_t q[NQ]; + load_block(vr, vp, kg * 16 + lane, vb, wv); + fa_dpas_split_block<NQ>(wv, (vb + vp) % 2 != 0, sc, q); + const float dv = jv ? static_cast<float>(sycl::bit_cast<sycl::half>((uint16_t) sc)) : 0.0f; + fa_dpas_short8 va[XT]; +#pragma unroll + for (int x = 0; x < XT; ++x) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int d = vd0 + x * 8 + m; + int v; + if constexpr (Q8) { + v = (int8_t) ((q[d / 4] >> (8 * (d % 4))) & 0xFF); + } else { + const int byte = d % 16; + v = (int) ((q[byte / 4] >> (8 * (byte % 4) + 4 * (d / 16))) & 0xF) - 8; + } + va[x][m] = (short) sycl::bit_cast<uint16_t>(sycl::half((float) v * dv)); + } + } +#pragma unroll + for (int t = 0; t < NR16; ++t) { + const fa_dpas_int8 pb = sP[((t * 16 + lane) * PS + kg * 16) / 16]; +#pragma unroll + for (int x = 0; x < XT; ++x) { + fa_dpas_f16(va[x], pb, o[t][x]); + } + } + } + it.barrier(sycl::access::fence_space::local_space); + } + +#pragma unroll + for (int t = 0; t < NR16; ++t) { + const int r = t * 16 + lane; + const int c = c0 + r / G; + if ((ALLPV || w < NPV) && r < R && c < p.ne01) { + const int64_t jdu = ((int64_t) seq * p.ne01 + c) * p.ne02 + kvh * G + r % G; + const float l = sL[r]; +#pragma unroll + for (int x = 0; x < XT; ++x) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int d = w * DPS + x * 8 + m; + if (p.nsplit == 1) { + p.dst[jdu * D + d] = l > 0.0f ? o[t][x][m] / l : 0.0f; + } else { + p.parts[(jdu * p.nsplit + split) * D + d] = o[t][x][m]; + } + } + } + } + } + if (p.nsplit > 1 && tid < R && c0 + tid / G < p.ne01) { + const int64_t jdu = ((int64_t) seq * p.ne01 + c0 + tid / G) * p.ne02 + kvh * G + tid % G; + p.meta[jdu * p.nsplit + split] = sycl::float2(sM[tid], sL[tid]); + } + }; + if constexpr (LG) { + cgh.parallel_for(ndr, sycl::ext::oneapi::experimental::properties{ sycl::ext::intel::experimental::grf_size<256> }, kern); + } else { + cgh.parallel_for(ndr, kern); + } + }); +} + +template <int D, int NR16, int KG, int TC, int ST, bool Q8> +static void fattn_dpas_case(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + constexpr int TK = 16 * KG; + + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + + const int ne01 = Q->ne[1]; + const int ne11 = K->ne[1]; + const int nkvh = K->ne[2]; + const int ne03 = Q->ne[3]; + + fa_dpas_params p; + p.Q = (const char *) Q->data; + p.K = (const char *) K->data; + p.V = (const char *) V->data; + p.mask = mask ? (const char *) mask->data : nullptr; + p.sinks = dst->src[4] ? (const float *) dst->src[4]->data : nullptr; + p.dst = (float *) dst->data; + memcpy(&p.scale, (const float *) dst->op_params + 0, sizeof(float)); + p.G = Q->ne[2] / K->ne[2]; + p.nq = std::max(1, std::min(ne01, 16 * NR16 / p.G)); + p.nqc = (ne01 + p.nq - 1) / p.nq; + p.ne01 = ne01; + p.ne02 = Q->ne[2]; + p.ne11 = ne11; + p.ne33 = mask ? mask->ne[3] : 1; + p.nb01 = Q->nb[1]; + p.nb02 = Q->nb[2]; + p.nb03 = Q->nb[3]; + p.nb11 = K->nb[1]; + p.nb12 = K->nb[2]; + p.nb13 = K->nb[3]; + p.nb21 = V->nb[1]; + p.nb22 = V->nb[2]; + p.nb23 = V->nb[3]; + p.nb31 = mask ? mask->nb[1] : 0; + p.nb33 = mask ? mask->nb[3] : 0; + + // split the keys so that all work-groups run in one wave + // SLM fits about 4 work-groups per Xe2 core with 16 rows, 2 with 32. Xe2 core = 8 EUs, nsm = EUs / 16. + const int xe_cores = std::max(1, ggml_sycl_info().devices[ctx.device].nsm * 2); + const int wave = xe_cores * (NR16 == 1 ? 4 : 2); + const int ntiles = (ne11 + TK - 1) / TK; + int nsplit = std::max(1, std::min(wave / (nkvh * ne03 * p.nqc), ntiles)); + p.chunk = ((ntiles + nsplit - 1) / nsplit) * TK; + nsplit = (ne11 + p.chunk - 1) / p.chunk; + p.nsplit = nsplit; + + ggml_sycl_pool_alloc<float> parts(ctx.pool()); + ggml_sycl_pool_alloc<sycl::float2> meta(ctx.pool()); + if (nsplit > 1) { + parts.alloc((size_t) nsplit * ggml_nelements(dst)); + meta.alloc((size_t) nsplit * ggml_nrows(dst)); + } + p.parts = parts.get(); + p.meta = meta.get(); + + dpct::queue_ptr stream = ctx.stream(); + // the large register file pays off for 32-row launches from about 4K keys + if (NR16 == 2 && ne11 >= 4096) { + fattn_dpas_kernel<D, NR16, KG, TC, ST, true, Q8>(p, nkvh, ne03, stream); + } else { + fattn_dpas_kernel<D, NR16, KG, TC, ST, false, Q8>(p, nkvh, ne03, stream); + } + + if (nsplit > 1) { + float * parts_p = parts.get(); + sycl::float2 * meta_p = meta.get(); + float * dst_p = (float *) dst->data; + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor<uint8_t, 1> lm(sycl::range<1>(nsplit * sizeof(sycl::float2)), cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(Q->ne[3], Q->ne[2], (size_t) ne01 * D), sycl::range<3>(1, 1, D)), + [=](sycl::nd_item<3>) [[sycl::reqd_sub_group_size(16)]] { + flash_attn_combine_results<D>(parts_p, meta_p, dst_p, nsplit, + lm.get_multi_ptr<sycl::access::decorated::no>().get()); + }); + }); + } +} + +template <bool Q8> static void fattn_dpas_dispatch(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + const int D = K->ne[0]; + const int G = Q->ne[2] / K->ne[2]; + const int ne01 = Q->ne[1]; + + bool aligned16 = true; + for (const ggml_tensor * t : { K, V }) { + aligned16 = aligned16 && t->nb[1] % 16 == 0 && t->nb[2] % 16 == 0 && t->nb[3] % 16 == 0 && + (uintptr_t) t->data % 16 == 0; + } + + // single token decode: staged K/V tiles are faster at long context + if (D == 256 && ne01 == 1 && G <= 16 && aligned16) { + fattn_dpas_case<256, 1, 8, 1, 1, Q8>(ctx, dst); + return; + } + const bool rows16 = ne01 * G <= 16; + // 256-key tiles for one row tile; 128 at head size 512 so S and the Q copy stay in SLM for the planned waves +#define FATTN_DPAS_D(D_, KG16_) \ + case D_: \ + if (rows16) { \ + fattn_dpas_case<D_, 1, KG16_, 1, 0, Q8>(ctx, dst); \ + } else { \ + fattn_dpas_case<D_, 2, 8, 2, 0, Q8>(ctx, dst); \ + } \ + break; + // q8_0 at head size 128: its 136-byte rows gather poorly per lane, stage them in SLM by coalesced dword loads + bool aligned4 = true; + for (const ggml_tensor * t : { K, V }) { + aligned4 = aligned4 && t->nb[1] % 4 == 0 && t->nb[2] % 4 == 0 && t->nb[3] % 4 == 0 && (uintptr_t) t->data % 4 == 0; + } + if (Q8 && D == 128 && aligned4) { + if (rows16) { + fattn_dpas_case<128, 1, 16, 1, 2, Q8>(ctx, dst); + } else { + fattn_dpas_case<128, 2, 8, 2, 2, Q8>(ctx, dst); + } + return; + } + switch (D) { + FATTN_DPAS_D( 64, 16) + FATTN_DPAS_D( 96, 16) + FATTN_DPAS_D(128, 16) + FATTN_DPAS_D(256, 16) + FATTN_DPAS_D(512, 8) + default: + GGML_ABORT("fattn-dpas: unsupported head size %d", D); + } +#undef FATTN_DPAS_D +} + +void ggml_sycl_flash_attn_ext_dpas(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + if (dst->src[1]->type == GGML_TYPE_Q8_0) { + fattn_dpas_dispatch<true>(ctx, dst); + } else { + fattn_dpas_dispatch<false>(ctx, dst); + } +} + +#else + +void ggml_sycl_flash_attn_ext_dpas(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(dst); + GGML_ABORT("fattn-dpas: not built for this target"); +} + +#endif // GGML_SYCL_FA_DPAS_BUILD diff --git a/ggml/src/ggml-sycl/fattn-dpas.hpp b/ggml/src/ggml-sycl/fattn-dpas.hpp new file mode 100644 index 000000000..856cbe7fb --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-dpas.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_FATTN_DPAS_HPP +#define GGML_SYCL_FATTN_DPAS_HPP + +#include "common.hpp" + +// q4_0 / q8_0 K/V flash attention for decode and small batches on Xe2 XMX (s8 x s4 / s8 x s8 DPAS) +bool ggml_sycl_flash_attn_ext_dpas_supported(int device, const ggml_tensor * dst); +void ggml_sycl_flash_attn_ext_dpas(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_FATTN_DPAS_HPP diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp index 30947b17b..e3b3553de 100644 --- a/ggml/src/ggml-sycl/fattn-mkl.cpp +++ b/ggml/src/ggml-sycl/fattn-mkl.cpp @@ -8,13 +8,13 @@ #include "common.hpp" #include "fattn-common.hpp" -#include "fattn-buffers.hpp" #include "convert.hpp" #include "fattn.hpp" #include <oneapi/mkl.hpp> #include <cstdio> #include <chrono> +#include <vector> #define MKL_FA_CHUNK_SIZE_KV 8192 @@ -131,7 +131,8 @@ static void mkl_fa_online_softmax_chunk( int kvh_head, const sycl::half * mask_data, int64_t mask_head_stride, int64_t mask_row_stride, int mask_n_heads, - float logit_softcap, int64_t wg_size) { + float logit_softcap, int64_t wg_size, + const sycl::event & dep_event) { // One work-group per query row: exactly q_rows groups of wg_size // items. q_rows * wg_size is already a multiple of wg_size, so unlike @@ -139,6 +140,7 @@ static void mkl_fa_online_softmax_chunk( const int64_t wg = q_rows * wg_size; const int local_size = (int) wg_size; // stride in the loops below stream->submit([&](sycl::handler & cgh) { + cgh.depends_on(dep_event); cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), [=](sycl::nd_item<1> item) { const int local_id = (int)item.get_local_id(0); @@ -267,42 +269,34 @@ struct mkl_fa_kv_desc { int64_t D = 0; // ne[0] int64_t nb1 = 0; // byte stride, seq dim int64_t nb2 = 0; // byte stride, head dim + int64_t nb3 = 0; // byte stride, batch dim mkl_fa_kv_desc_mode mode = MKL_FA_KV_MODE_F16_DENSE; int64_t ts = 0; // type size (mode 3 base offset) int64_t s01 = 0; // nc row stride in blocks (mode 3) int64_t s02 = 0; // nc head stride in blocks (mode 3) }; -static mkl_fa_kv_desc mkl_fa_make_desc(const ggml_tensor * T, bool interleaved, int n_kv_heads) { +static mkl_fa_kv_desc mkl_fa_make_desc(const ggml_tensor * T, bool interleaved) { mkl_fa_kv_desc d; d.data = (const char *)T->data; d.type = T->type; d.D = T->ne[0]; d.nb1 = (int64_t)T->nb[1]; d.nb2 = (int64_t)T->nb[2]; + d.nb3 = (int64_t)T->nb[3]; d.ts = (int64_t)ggml_type_size(T->type); if (T->type == GGML_TYPE_F16) { - d.mode = interleaved ? MKL_FA_KV_MODE_F16_INTERLEAVED - : MKL_FA_KV_MODE_F16_DENSE; + // MLA's V cache is a 512-wide view of 576-wide K rows. Treat any + // padded row stride as strided even when there is only one KV head. + d.mode = interleaved || d.nb1 != d.D * (int64_t)sizeof(sycl::half) + ? MKL_FA_KV_MODE_F16_INTERLEAVED : MKL_FA_KV_MODE_F16_DENSE; } else if (ggml_is_contiguously_allocated(T) && !interleaved) { d.mode = MKL_FA_KV_MODE_QUANT_CONTIG; } else { d.mode = MKL_FA_KV_MODE_QUANT_NC; - const int64_t bs = (int64_t)ggml_blck_size(T->type); - const int64_t blk_per_row = T->ne[0] / bs; - // True Gemma interleave packs heads within a row (nb[2] < ne[1]*nb[1]) - // → reconstruct physical strides. Padded seq-views (nb[2] > ne[1]*nb[1]) - // already have correct physical strides. - const bool gemma = interleaved && - ((int64_t)T->nb[2] < (int64_t)T->ne[1] * (int64_t)T->nb[1]); - if (gemma) { - d.s01 = (int64_t)n_kv_heads * blk_per_row; - d.s02 = blk_per_row; - } else { - d.s01 = d.nb1 / d.ts; - d.s02 = d.nb2 / d.ts; - } + d.s01 = d.nb1 / d.ts; + d.s02 = d.nb2 / d.ts; } return d; } @@ -310,19 +304,20 @@ static mkl_fa_kv_desc mkl_fa_make_desc(const ggml_tensor * T, bool interleaved, // Dequant one KV-head chunk into a dense [this_chunk x D] fp16 buffer. static void mkl_fa_dequant_chunk( dpct::queue_ptr stream, const mkl_fa_kv_desc & d, ggml_tensor * dst_ctx, - sycl::half * out, int ikvh, int chunk_start, int this_chunk) { + sycl::half * out, int ikvh, int ib, int chunk_start, int this_chunk) { const int64_t D = d.D; switch (d.mode) { case MKL_FA_KV_MODE_F16_DENSE: { + // ib * nb3 selects this batch's KV (nb[3] != ne[1]*nb[1] for batched KV views) const char * base = d.data + (int64_t)ikvh * d.nb2 - + (int64_t)chunk_start * d.nb1; + + (int64_t)chunk_start * d.nb1 + (int64_t)ib * d.nb3; stream->memcpy(out, base, (size_t)this_chunk * D * sizeof(sycl::half)); break; } case MKL_FA_KV_MODE_F16_INTERLEAVED: { const char * base = d.data + (int64_t)ikvh * d.nb2 - + (int64_t)chunk_start * d.nb1; + + (int64_t)chunk_start * d.nb1 + (int64_t)ib * d.nb3; const int64_t row_halfs = d.nb1 / (int64_t)sizeof(sycl::half); const sycl::half * src = (const sycl::half *)base; stream->parallel_for( @@ -336,7 +331,7 @@ static void mkl_fa_dequant_chunk( } case MKL_FA_KV_MODE_QUANT_CONTIG: { const char * base = d.data + (int64_t)ikvh * d.nb2 - + (int64_t)chunk_start * d.nb1; + + (int64_t)chunk_start * d.nb1 + (int64_t)ib * d.nb3; to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(d.type, dst_ctx); to_fp16(base, out, (int64_t)this_chunk * D, stream); break; @@ -344,7 +339,7 @@ static void mkl_fa_dequant_chunk( default: { // MKL_FA_KV_MODE_QUANT_NC to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(d.type); const int64_t base_blocks = (int64_t)ikvh * d.s02 - + (int64_t)chunk_start * d.s01; + + (int64_t)chunk_start * d.s01 + (int64_t)ib * d.nb3 / d.ts; const char * base = d.data + base_blocks * d.ts; // ne02 = ne03 = 1 → s02/s03 inert; head+chunk offset carried by base. to_fp16(base, out, D, this_chunk, 1, 1, d.s01, d.s02, d.s02, stream); @@ -413,7 +408,9 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * const int64_t q_row_stride = Q->nb[1] / sizeof(float); const int64_t q_head_stride = Q->nb[2] / sizeof(float); - const bool V_is_K_view = V->view_src + // Alias the dequantized buffers only when K and V expose the same values. + // MLA V is a narrower view of K and needs its own strided dequantization. + const bool V_is_K_view = V->ne[0] == K->ne[0] && V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)); @@ -471,9 +468,9 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * const bool v_interleaved = ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]) && V->ne[2] > 1; - const mkl_fa_kv_desc K_desc = mkl_fa_make_desc(K, k_interleaved, n_kv_heads); + const mkl_fa_kv_desc K_desc = mkl_fa_make_desc(K, k_interleaved); const mkl_fa_kv_desc V_desc = V_is_K_view - ? K_desc : mkl_fa_make_desc(V, v_interleaved, n_kv_heads); + ? K_desc : mkl_fa_make_desc(V, v_interleaved); MKL_ACCUM(dequant_time_us, t_deq); @@ -560,9 +557,6 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr, n_query_rows, DV, wg_size); - // Sync before MKL GEMM (MKL may use an internal queue) - stream->wait(); - // 3. KV chunk loop (OUTER): dequant each chunk once, then tile queries. for (int chunk_start = 0; chunk_start < n_kv; chunk_start += chunk_size) { int this_chunk = std::min(chunk_size, n_kv - chunk_start); @@ -571,34 +565,31 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * { MKL_TAKE_TIME(t0); mkl_fa_dequant_chunk(stream, K_desc, KQV, - K_chunk_f16_ptr, ikvh, chunk_start, this_chunk); + K_chunk_f16_ptr, ikvh, ib, chunk_start, this_chunk); if (!V_is_K_view) { mkl_fa_dequant_chunk(stream, V_desc, KQV, - V_chunk_f16_ptr, ikvh, chunk_start, this_chunk); + V_chunk_f16_ptr, ikvh, ib, chunk_start, this_chunk); } - stream->wait(); // dequant must be ready before MKL GEMM MKL_ACCUM(dequant_time_us, t0); } // 3b. Query tile loop (INNER) — bounds KQ_f32/S_f16 footprint. for (int q0 = 0; q0 < n_query_rows; q0 += q_tile_rows) { int q_rows = std::min(q_tile_rows, n_query_rows - q0); + sycl::event kq_done, vkq_done; // GEMM: KQ = Q_tile × K_chunk^T { MKL_TAKE_TIME(t0); - sycl::event ev = gemm(*stream, + kq_done = gemm(*stream, transpose::trans, transpose::nontrans, this_chunk, q_rows, DKQ, alpha, K_chunk_f16_ptr, DKQ, Q_head_f16_ptr + (int64_t)q0 * DKQ, DKQ, beta, - KQ_f32_ptr, this_chunk); - try { ev.wait_and_throw(); } catch (sycl::exception & e) { - GGML_LOG_INFO("[MKL-FA] GEMM KQ: %s\n", e.what()); - GGML_ABORT("MKL GEMM KQ failed"); - } + KQ_f32_ptr, this_chunk, + std::vector<sycl::event>{*stream->ext_oneapi_get_last_event()}); MKL_ACCUM(gemm_kq_time_us, t0); } // Online softmax over this chunk for this query tile @@ -612,26 +603,22 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * kvh_base_head, mask_batch, mask_head_stride, mask_row_stride, mask_n_heads, - logit_softcap, wg_size); - stream->wait(); // S_f16 must be ready for GEMM + logit_softcap, wg_size, kq_done); MKL_ACCUM(softmax_time_us, t0); } // GEMM: VKQ_chunk = S × V_chunk { MKL_TAKE_TIME(t0); - sycl::event ev = gemm(*stream, + vkq_done = gemm(*stream, transpose::nontrans, transpose::nontrans, DV, q_rows, this_chunk, alpha, V_chunk_f16_ptr, DV, S_f16_ptr, this_chunk, beta, - VKQ_chunk_ptr, DV); - try { ev.wait_and_throw(); } catch (sycl::exception & e) { - GGML_LOG_INFO("[MKL-FA] GEMM VKQ: %s\n", e.what()); - GGML_ABORT("MKL GEMM VKQ failed"); - } + VKQ_chunk_ptr, DV, + std::vector<sycl::event>{*stream->ext_oneapi_get_last_event()}); MKL_ACCUM(gemm_vkq_time_us, t0); } // VKQ_accum[q0..] += VKQ_chunk @@ -641,6 +628,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * * wg_size; float * accum = VKQ_accum_ptr + (int64_t)q0 * DV; stream->submit([&](sycl::handler & cgh) { + cgh.depends_on(vkq_done); cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), [=](sycl::nd_item<1> item) { int64_t i = item.get_global_id(0); diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp index 6e030940a..66ff32f15 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.cpp +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -1,4 +1,5 @@ #include <array> +#include <atomic> #include <cstdint> #include <cstdio> #include <cstring> @@ -25,6 +26,20 @@ bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * return bindable(K) && bindable(V); } +#if GGML_SYCL_DNNL +// per device: set once a oneDNN SDPA execute throws, so dispatch stops picking oneDNN there +static std::atomic<bool> g_ggml_sycl_fa_onednn_failed[GGML_SYCL_MAX_DEVICES] = {}; +#endif + +bool ggml_sycl_flash_attn_ext_onednn_failed(int device) { +#if GGML_SYCL_DNNL + return g_ggml_sycl_fa_onednn_failed[device].load(std::memory_order_relaxed); +#else + GGML_UNUSED(device); + return false; +#endif +} + bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit) { #if !GGML_SYCL_DNNL GGML_UNUSED(dst); @@ -69,17 +84,6 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use } } } - // This is the improved SPDA gate. Rather than gating Alchemist GPUs from all SPDA features, we instead target only the failing shapes. - // If the GPU being assessed isn't in the grouping below, it has full access to all SPDA shapes. Otherwise, if it's an Alchemist GPU, we block only the shapes with head sizes that fail. - // It is much easier to compare the device to a small list of failing cases than to define all the passing ones. - const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch; - bool support_spda = !(arch == gpu_arch::intel_gpu_dg2_g10 || - arch == gpu_arch::intel_gpu_dg2_g11 || - arch == gpu_arch::intel_gpu_dg2_g12); - - if (!support_spda && K->ne[0] == 64) { - return false; - } // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch: // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on // some stacks; past the cap we fall back to the native FA kernel instead. @@ -157,14 +161,6 @@ static void permute_sdpa_out_sycl(const sycl::half * out, float * dst, }); } -struct sdpa_partition { - compiled_partition cp; - std::vector<logical_tensor> ins; - logical_tensor out; - size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0; - bool ok = false; -}; - // Build + compile the contiguous-input GQA SDPA graph (MatMul->Divide->Add->SoftMax->MatMul), f32 out. // Mirrors the hardware-verified scratch/onednn_sdpa_probe.cpp build_gqa (partitions=1, sdp_primitive_kernel_t). static sdpa_partition build_sdpa(const engine & eng, int H, int Hkv, int q, int seq, int d, @@ -306,29 +302,15 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso { const char * K_data = (const char *)K->data; const bool k_non_dense = ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]) && K->ne[2] > 1; - const bool k_gemma = k_non_dense && - ((int64_t)K->nb[2] < (int64_t)K->ne[1] * (int64_t)K->nb[1]); if (ggml_is_contiguously_allocated(K) && !k_non_dense) { to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(K->type, dst); to_fp16(K_data, K_ptr, ggml_nelements(K), stream); } else { - const size_t bs = ggml_blck_size(K->type); const size_t ts = ggml_type_size(K->type); to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(K->type); - int64_t s01, s02, s03; - if (k_gemma) { - const int64_t blk_per_row = (int64_t)K->ne[0] / bs; - s01 = (int64_t)Hkv * blk_per_row; - s02 = blk_per_row; - s03 = (int64_t)K->ne[1] * s01; - } else { - s01 = (int64_t)K->nb[1] / ts; - s02 = (int64_t)K->nb[2] / ts; - s03 = (int64_t)K->nb[3] / ts; - } to_fp16(K_data, K_ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], - s01, s02, s03, stream); + (int64_t)K->nb[1] / ts, (int64_t)K->nb[2] / ts, (int64_t)K->nb[3] / ts, stream); } } // Quantized V: always dequant separately. Even when K and V share @@ -340,29 +322,15 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso { const char * V_data = (const char *)V->data; const bool v_non_dense = ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]) && V->ne[2] > 1; - const bool v_gemma = v_non_dense && - ((int64_t)V->nb[2] < (int64_t)V->ne[1] * (int64_t)V->nb[1]); if (ggml_is_contiguously_allocated(V) && !v_non_dense) { to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(V->type, dst); to_fp16(V_data, V_ptr, ggml_nelements(V), stream); } else { - const size_t bs = ggml_blck_size(V->type); const size_t ts = ggml_type_size(V->type); to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(V->type); - int64_t s01, s02, s03; - if (v_gemma) { - const int64_t blk_per_row = (int64_t)V->ne[0] / bs; - s01 = (int64_t)V->ne[2] * blk_per_row; - s02 = blk_per_row; - s03 = (int64_t)V->ne[1] * s01; - } else { - s01 = (int64_t)V->nb[1] / ts; - s02 = (int64_t)V->nb[2] / ts; - s03 = (int64_t)V->nb[3] / ts; - } to_fp16(V_data, V_ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], - s01, s02, s03, stream); + (int64_t)V->nb[1] / ts, (int64_t)V->nb[2] / ts, (int64_t)V->nb[3] / ts, stream); } } } else { @@ -400,17 +368,16 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso outf_ptr = outf_pool->get(); } - // compile once per (device, shape, KV strides), reuse across layers/calls. Stride 2 always - // repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ. - static std::unordered_map<std::string, sdpa_partition> cache; + // compile once per (shape, KV strides) in this device's context, reuse across layers/calls. Stride 2 + // always repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ. char keyb[256]; - snprintf(keyb, sizeof(keyb), "%d:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", ggml_sycl_get_device(), + snprintf(keyb, sizeof(keyb), "%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", (long long) H, (long long) Hkv, (long long) q, (long long) seq, (long long) d, (long long) k_str[0], (long long) k_str[1], (long long) k_str[3], (long long) v_str[0], (long long) v_str[1], (long long) v_str[3]); - auto it = cache.find(keyb); - if (it == cache.end()) { - it = cache.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first; + auto it = ctx.sdpa_partitions.find(keyb); + if (it == ctx.sdpa_partitions.end()) { + it = ctx.sdpa_partitions.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first; } sdpa_partition & E = it->second; if (!E.ok) { @@ -446,8 +413,10 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso } } catch (const std::exception & e) { - // any oneDNN/SYCL failure is non-fatal: fall back to the existing kernel (strictly additive). - GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what()); + // any oneDNN/SYCL failure is non-fatal: warn once, stop picking oneDNN on this device, run TILE instead + if (!g_ggml_sycl_fa_onednn_failed[ctx.device].exchange(true)) { + GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what()); + } ggml_sycl_flash_attn_ext_tile(ctx, dst); } diff --git a/ggml/src/ggml-sycl/fattn-onednn.hpp b/ggml/src/ggml-sycl/fattn-onednn.hpp index 9669d1bd2..4756b240a 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.hpp +++ b/ggml/src/ggml-sycl/fattn-onednn.hpp @@ -7,6 +7,10 @@ // (f16 KV, no softcap/ALiBi, single stream, tuned head_dim, prefill-sized q.) bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit = true); +// True once a oneDNN SDPA execute has failed on this device. Not part of _supported(): that also sizes the +// FA scratch buffer, and a size that depends on runtime state can come out smaller than a later call needs. +bool ggml_sycl_flash_attn_ext_onednn_failed(int device); + // True when the oneDNN path binds an F16 KV cache in place instead of staging a dense copy of // it. Depends only on the types and strides of K and V, so the answer holds for every call. bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * V); diff --git a/ggml/src/ggml-sycl/fattn-sparse.cpp b/ggml/src/ggml-sycl/fattn-sparse.cpp index df2837094..0c56d43aa 100644 --- a/ggml/src/ggml-sycl/fattn-sparse.cpp +++ b/ggml/src/ggml-sycl/fattn-sparse.cpp @@ -202,7 +202,8 @@ bool ggml_sycl_flash_attn_ext_sparse(ggml_backend_sycl_context & ctx, ggml_tenso uint8_t * d_V = v_alloc.get(); sycl::half * d_mask = m_alloc.get(); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memset(d_cnt, 0, sizeof(int32_t)))); + // reset with a kernel: on Level Zero a queue memset is not always ordered before the next kernel + stream->single_task([=]() { *d_cnt = 0; }); sparse_fa_compact_mask(stream, (const sycl::half *) mask->data, d_idx, d_cnt, n_kv, n_kv_g); @@ -217,7 +218,7 @@ bool ggml_sycl_flash_attn_ext_sparse(ggml_backend_sycl_context & ctx, ggml_tenso d_idx, d_cnt, n_kv_g, n_rows_m, mask->nb[1] / sizeof(sycl::half)); - if (sparse_fa_debug()) { + if (sparse_fa_debug() && stream->ext_oneapi_get_state() != sycl::ext::oneapi::experimental::queue_state::recording) { int32_t h_cnt = 0; SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(&h_cnt, d_cnt, sizeof(int32_t)))); SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); diff --git a/ggml/src/ggml-sycl/fattn-tile.cpp b/ggml/src/ggml-sycl/fattn-tile.cpp index 9449d7578..db030268e 100644 --- a/ggml/src/ggml-sycl/fattn-tile.cpp +++ b/ggml/src/ggml-sycl/fattn-tile.cpp @@ -54,6 +54,6 @@ void ggml_sycl_flash_attn_ext_tile(ggml_backend_sycl_context & ctx, ggml_tensor } break; default: { GGML_ABORT("Unsupported head size"); - } break; + } } } diff --git a/ggml/src/ggml-sycl/fattn-vec.hpp b/ggml/src/ggml-sycl/fattn-vec.hpp index 9ec88c287..5e40c88fb 100644 --- a/ggml/src/ggml-sycl/fattn-vec.hpp +++ b/ggml/src/ggml-sycl/fattn-vec.hpp @@ -589,10 +589,10 @@ void ggml_sycl_flash_attn_ext_vec_case_impl(ggml_backend_sycl_context & ctx, ggm const bool need_f16_V = type_V == GGML_TYPE_F16; constexpr size_t nbytes_shared = 0; + const auto arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch; // D=512 does not fit the default register file; it spills up to 343 bytes per thread, against at most 57 for D <= 256. This kernel is decode only, so thread occupancy is not the limit. It is 1.9x faster at every KV depth on Battlemage. constexpr bool use_large_grf = D >= 512; - const auto arch = ggml_sycl_info().devices[ctx.device].hw_info.arch; const int nthreads = ggml_sycl_fattn_vec_get_nthreads_device(arch); if constexpr (D <= 256) { if (nthreads == 256) { diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index 541ae8a82..525e6fd8c 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -20,6 +20,7 @@ #include "fattn.hpp" #include "fattn-onednn.hpp" #include "fattn-sparse.hpp" +#include "fattn-dpas.hpp" #define FATTN_VEC_CASE(D, type_K, type_V) \ { \ @@ -97,6 +98,7 @@ static void ggml_sycl_flash_attn_ext_vec(ggml_backend_sycl_context & ctx, ggml_t enum best_fattn_kernel { BEST_FATTN_KERNEL_NONE = 0, BEST_FATTN_KERNEL_VEC = 100, + BEST_FATTN_KERNEL_DPAS = 120, // q4_0 / q8_0 K/V decode on Xe2 XMX BEST_FATTN_KERNEL_ONEDNN = 150, // oneDNN SDPA: native F16 (PR #25222) BEST_FATTN_KERNEL_TILE = 200, BEST_FATTN_KERNEL_MKL = 300, @@ -132,7 +134,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16). // ONEDNN requires min 32 query tokens — short-circuit decode to avoid // calling _supported() on every decode FA call. - if (Q->ne[1] >= 32 + if (Q->ne[1] >= 32 && !ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { return BEST_FATTN_KERNEL_ONEDNN; } @@ -146,15 +148,17 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // Set GGML_SYCL_ENABLE_MKL_FA=0 to force TILE/VEC path for A/B testing. // Example: GGML_SYCL_ENABLE_MKL_FA=0 llama-cli -m model.gguf -fa -ngl 99 ... // Note: MKL GEMM calls are incompatible with SYCL graph capture replay. - // MKL is validated for the mainstream GQA envelope: grouped-query - // (gqa_ratio >= 2), head_dim a multiple of 64 in [64,512] with matching - // K/V head size, mask, no sinks/ALiBi/softcap. Gemma's global layers use - // head_dim 512, so the cap must include it. Head sizes not a multiple of - // 64 (72/80/96), MHA (gqa_ratio == 1), and MLA (DKQ != DV, e.g. 576/512) - // fall through to TILE/VEC; see follow-up work. + const bool standard_shape = Q->ne[0] >= 64 && Q->ne[0] <= 512 && + Q->ne[0] % 64 == 0 && Q->ne[0] == V->ne[0]; + // GLM-4.7 Flash's MLA shape is already expressible by the MKL pipeline: + // KQ GEMM uses DKQ=576 while VKQ and output use DV=512. Keep this narrow + // until other mismatched K/V shapes have independent correctness data. + const bool glm_mla_shape = Q->ne[0] == 576 && K->ne[0] == 576 && + V->ne[0] == 512 && gqa_ratio == 20 && + K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16; + if (g_ggml_sycl_enable_mkl_fa == 1 && mask && !sinks && gqa_ratio >= 2 && - Q->ne[0] >= 64 && Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && - Q->ne[0] == V->ne[0] && + (standard_shape || glm_mla_shape) && Q->ne[1] >= 32 && K->ne[1] >= 1024 && max_bias == 0.0f && logit_softcap == 0.0f && (Q->ne[3] == K->ne[3] || K->ne[3] == 1)) { @@ -164,7 +168,10 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // nb1=75 for ne0=40 fall through to TILE. bool kv_strides_ok = true; for (const ggml_tensor * t : {K, V}) { - if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) { + const bool glm_v_stride = glm_mla_shape && t == V && + V->view_src && V->nb[1] == K->nb[1]; + if (!glm_v_stride && t->type == GGML_TYPE_F16 && + t->nb[1] % (t->ne[0] * 2) != 0) { kv_strides_ok = false; break; } @@ -239,6 +246,11 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_NONE; } + // q4_0 / q8_0 K/V decode and small batches on Xe2: read the cache directly, no F16 copy of K and V + if (ggml_sycl_flash_attn_ext_dpas_supported(device, dst)) { + return BEST_FATTN_KERNEL_DPAS; + } + // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes. // BF16 is excluded: the VEC kernel has no BF16 template (it needs GGML_SYCL_FA_ALL_QUANTS for non-F16/Q4_0/Q8_0). const bool has_bf16 = (K->type == GGML_TYPE_BF16 || V->type == GGML_TYPE_BF16); @@ -247,7 +259,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly // additive -- taken only when statically supported, otherwise falls through to VEC/TILE below. - if (ggml_sycl_flash_attn_ext_onednn_supported(dst)) { + if (!ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { return BEST_FATTN_KERNEL_ONEDNN; } @@ -276,6 +288,22 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_set_device(ctx.device); + ggml_tensor * Q = dst->src[0]; + ggml_tensor * K = dst->src[1]; + ggml_tensor * V = dst->src[2]; + if (K && Q && K->ne[0] != Q->ne[0] && Q->ne[0] > 0 && K->ne[0] % Q->ne[0] == 0) { + const int64_t d = Q->ne[0]; + const int64_t n_heads = (K->ne[0] * K->ne[2]) / d; + K->ne[0] = d; + K->ne[2] = n_heads; + } + if (V && Q && V->ne[0] != Q->ne[0] && Q->ne[0] > 0 && V->ne[0] % Q->ne[0] == 0) { + const int64_t d = Q->ne[0]; + const int64_t n_heads = (V->ne[0] * V->ne[2]) / d; + V->ne[0] = d; + V->ne[2] = n_heads; + } + // sparse nodes are gathered down to n_kv_max rows and re-dispatched here if (ggml_sycl_flash_attn_ext_sparse(ctx, dst)) { return; @@ -297,6 +325,7 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst if (k == BEST_FATTN_KERNEL_MKL) kname = "MKL"; if (k == BEST_FATTN_KERNEL_ONEDNN) kname = "ONEDNN"; if (k == BEST_FATTN_KERNEL_VEC) kname = "VEC"; + if (k == BEST_FATTN_KERNEL_DPAS) kname = "DPAS"; int64_t delta = 0; if (Dk == 256) { delta = cur_nkv - last_nkv_d256; @@ -329,6 +358,9 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst case BEST_FATTN_KERNEL_VEC: ggml_sycl_flash_attn_ext_vec(ctx, dst); break; + case BEST_FATTN_KERNEL_DPAS: + ggml_sycl_flash_attn_ext_dpas(ctx, dst); + break; case BEST_FATTN_KERNEL_MKL: ggml_sycl_flash_attn_ext_mkl(ctx, dst); break; @@ -344,10 +376,10 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst const ggml_tensor * K_diag = dst->src[1]; const ggml_tensor * V_diag = dst->src[2]; const ggml_tensor * Q_diag = dst->src[0]; - if (K_diag->ne[1] >= 1024) { + dpct::queue_ptr q = ctx.stream(); + if (K_diag->ne[1] >= 1024 && q->ext_oneapi_get_state() != sycl::ext::oneapi::experimental::queue_state::recording) { fa_diag_count++; float diag_buf[64]; - dpct::queue_ptr q = ctx.stream(); q->memcpy(diag_buf, dst->data, 64 * sizeof(float)); q->wait(); const char * kname = "???"; @@ -356,6 +388,7 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst if (kb == BEST_FATTN_KERNEL_MKL) kname = "MKL"; if (kb == BEST_FATTN_KERNEL_TILE) kname = "TILE"; if (kb == BEST_FATTN_KERNEL_VEC) kname = "VEC"; + if (kb == BEST_FATTN_KERNEL_DPAS) kname = "DPAS"; GGML_LOG_INFO("[FA-DIAG] #%d %s D=%d n_kv=%lld n_q=%lld " "n_qh=%lld n_kvh=%lld K=%s V=%s " "nb1=%zu nb2=%zu first 64 floats:\n", @@ -383,6 +416,14 @@ bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst) { return ggml_sycl_get_best_fattn_kernel(device, dst) != BEST_FATTN_KERNEL_NONE; } +bool ggml_sycl_flash_attn_ext_needs_sync(int device, const ggml_tensor * dst) { + return ggml_sycl_get_best_fattn_kernel(device, dst) == BEST_FATTN_KERNEL_MKL; +} + +bool ggml_sycl_flash_attn_ext_uses_onednn(int device, const ggml_tensor * dst) { + return ggml_sycl_get_best_fattn_kernel(device, dst) == BEST_FATTN_KERNEL_ONEDNN; +} + static uintptr_t ggml_sycl_fattn_reserve_halves(ggml_sycl_fattn_extra & extra, size_t n_halves) { if (n_halves == 0) { return 0; diff --git a/ggml/src/ggml-sycl/fattn.hpp b/ggml/src/ggml-sycl/fattn.hpp index f803aa2a8..8ba634223 100644 --- a/ggml/src/ggml-sycl/fattn.hpp +++ b/ggml/src/ggml-sycl/fattn.hpp @@ -19,6 +19,12 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst); +// Reports if flash attention runs this node with the oneMKL kernel, which waits on the host +bool ggml_sycl_flash_attn_ext_needs_sync(int device, const ggml_tensor * dst); + +// Reports if flash attention runs this node with the oneDNN kernel +bool ggml_sycl_flash_attn_ext_uses_onednn(int device, const ggml_tensor * dst); + // Scratch that flash attention needs beyond the output tensor struct ggml_sycl_fattn_extra { uintptr_t K_buffer_ptr = 0; // F16 copy of the K cache diff --git a/ggml/src/ggml-sycl/fused-gemm.cpp b/ggml/src/ggml-sycl/fused-gemm.cpp new file mode 100644 index 000000000..803a994c0 --- /dev/null +++ b/ggml/src/ggml-sycl/fused-gemm.cpp @@ -0,0 +1,1438 @@ +#include "fused-gemm.hpp" + +#include <sycl/ext/oneapi/matrix/matrix.hpp> + +#include <algorithm> +#include <string> +#include <tuple> +#include <mutex> +#include <set> +#include <unordered_map> + +namespace mx = sycl::ext::oneapi::experimental::matrix; + +// FG_ / fg_ is short for fused GEMM: the weights are dequantized inside the GEMM, into the XMX tiles. + +// A k step is one 32-value weight sub-block; iq3_s and the other superblock formats split their +// superblock into steps of this width. The sub-groups of a work-group each walk their own K range +// and are summed at the end. +static constexpr int FG_BK = QK4_NL; +static constexpr int FG_KSPLIT = 4; + +// Element traits of one joint_matrix operand type. The A stage and the B pack compute in f32 and +// convert once, in registers, when they write the element, so any type costs the same one pass. +// store: storage in SLM (A) and in the packed B buffer +// mtype: matrix_type in matrix_combinations +// mode: GGML_SYCL_DYNAMIC_PRECISION value that selects this type +// src: ggml type that needs no conversion into this type (GGML_TYPE_COUNT: none) +// slow: XMX throughput class, 0 is fastest. f16 and bf16 share the DPAS rate; tf32 does half the +// K per instruction. B60, Qwen3-30B-A3B pp512: f16 1108, bf16 1000, tf32 751 t/s +template <typename T> struct fg_elem; + +template <> struct fg_elem<sycl::half> { + using store = sycl::half; + using pair = sycl::half2; + static constexpr mx::matrix_type mtype = mx::matrix_type::fp16; + static constexpr int mode = GGML_SYCL_DYNAMIC_PRECISION_F16; + static constexpr ggml_type src = GGML_TYPE_F16; + static constexpr int mant = 10; + static constexpr int slow = 0; + static store cvt(float x) { return (store) x; } + static pair make(float x, float y) { return pair((store) x, (store) y); } +}; + +// tf32 rounds to nearest even with plain bit ops: round_to_tf32 needs a SPIR-V extension that the +// DG2 AOT target rejects +static inline uint32_t fg_round_bits(float x, int drop) { + const uint32_t u = sycl::bit_cast<uint32_t>(x); + if ((u & 0x7f800000u) == 0x7f800000u) { + return (u & 0x7fffffu) ? u | (1u << drop) : u; // nan stays nan + } + return u + ((1u << (drop - 1)) - 1) + ((u >> drop) & 1); +} + +struct alignas(4) fg_bf16x2 { + sycl::ext::oneapi::bfloat16 x, y; +}; + +template <> struct fg_elem<sycl::ext::oneapi::bfloat16> { + using store = sycl::ext::oneapi::bfloat16; + using pair = fg_bf16x2; + static constexpr mx::matrix_type mtype = mx::matrix_type::bf16; + static constexpr int mode = GGML_SYCL_DYNAMIC_PRECISION_BF16; + static constexpr ggml_type src = GGML_TYPE_BF16; + static constexpr int mant = 7; + static constexpr int slow = 0; + static store cvt(float x) { return store(x); } + static pair make(float x, float y) { return { cvt(x), cvt(y) }; } +}; + +// tf32 keeps f32 range and f16 mantissa, in f32 storage +template <> struct fg_elem<mx::precision::tf32> { + using store = float; + using pair = sycl::float2; + static constexpr mx::matrix_type mtype = mx::matrix_type::tf32; + static constexpr int mode = GGML_SYCL_DYNAMIC_PRECISION_TF32; + static constexpr ggml_type src = GGML_TYPE_COUNT; + static constexpr int mant = 10; + static constexpr int slow = 1; + static store cvt(float x) { return sycl::bit_cast<float>(fg_round_bits(x, 13) & ~0x1fffu); } + static pair make(float x, float y) { return pair(cvt(x), cvt(y)); } +}; + +// One joint_matrix combination (A type, B type, TM x TN x TK, sub-group size; C and D are f32) and +// the tiling built on it. A sub-group owns SG_ROWS rows of A (at least 16) and BN (at least 32) +// columns of B. A and B may differ: the device lists the pairs it supports. +template <typename TA, typename TB, int TM_, int TN_, int TK_, int SG_> struct fg_combo { + using ta = TA; + using tb = TB; + using EA = fg_elem<TA>; + using EB = fg_elem<TB>; + using tsa = typename EA::store; + using tsb = typename EB::store; + static constexpr int TM = TM_; + static constexpr int TN = TN_; + static constexpr int TK = TK_; + static constexpr int SG = SG_; + static constexpr int VNNI = 4 / sizeof(tsb); // K rows of B packed in one 32-bit word + static constexpr int SG_ROWS = TM > 16 ? TM : 16; + static constexpr int RPL = SG_ROWS / SG; // A rows one lane decodes per k step + static constexpr int MT = SG_ROWS / TM; + static constexpr int BN = TN > 32 ? TN : 32; + static constexpr int NT = BN / TN; + static constexpr int WG_SIZE = FG_KSPLIT * SG; + static constexpr mx::layout b_layout = VNNI == 1 ? mx::layout::row_major : mx::layout::ext_intel_packed; + // a 64-wide N is mostly padding here and a 32x64 f32 accumulator needs 128 registers per lane, + // so it spills: 13x slower on B60 + static constexpr bool efficient = TN <= 32; + static_assert(SG_ROWS % SG == 0 && SG_ROWS % TM == 0 && BN % TN == 0 && FG_BK % TK == 0, "bad tile"); + static_assert(BN <= GGML_SYCL_FG_MAX_N, "header gate must cover the tile width"); +}; + +using fg_half = sycl::half; +using fg_bf16 = sycl::ext::oneapi::bfloat16; +using fg_tf32 = mx::precision::tf32; + +// One bit of GGML_SYCL_XMX_GATHER_SHAPES per combination. Only combinations some device lists in +// matrix_combinations are built (appendix of sycl_ext_oneapi_matrix and the runtime's own list). +template <typename F> static void fg_visit_combo(int idx, F && f); +static constexpr int FG_N_COMBOS = 8; + +// A spir64_gen AOT build (GGML_SYCL_XMX_AOT_SG) drops the combinations of the other sub-group size +// entirely: ocloc rejects even an empty kernel that asks for a sub-group size it lacks. +template <int SG> static constexpr bool fg_listed() { +#if defined(GGML_SYCL_XMX_AOT_SG) + return SG == GGML_SYCL_XMX_AOT_SG; +#else + return true; +#endif +} + +template <typename S, typename F> static void fg_call_combo(F && f) { + if constexpr (fg_listed<S::SG>()) { + f(S{}); + } +} + +template <typename F> static void fg_visit_combo(int idx, F && f) { + switch (idx) { + case 0: fg_call_combo<fg_combo<fg_half, fg_half, 8, 16, 16, 16>>(f); break; // Xe2, Xe3, Xe-HPC + case 1: fg_call_combo<fg_combo<fg_half, fg_half, 16, 16, 16, 16>>(f); break; // Xe2, Xe3, Xe-HPC + case 2: fg_call_combo<fg_combo<fg_half, fg_half, 32, 64, 16, 16>>(f); break; // Xe2, Xe3, Xe-HPC + case 3: fg_call_combo<fg_combo<fg_half, fg_half, 32, 64, 32, 16>>(f); break; // Xe2, Xe3, Xe-HPC + case 4: fg_call_combo<fg_combo<fg_half, fg_half, 8, 8, 16, 8>>(f); break; // Xe-HPG (Arc A), ARL-H + case 5: fg_call_combo<fg_combo<fg_tf32, fg_tf32, 8, 16, 8, 16>>(f); break; // Xe2, Xe3, Xe-HPC + case 6: fg_call_combo<fg_combo<fg_bf16, fg_bf16, 8, 16, 16, 16>>(f); break; // Xe2, Xe3, Xe-HPC + case 7: fg_call_combo<fg_combo<fg_bf16, fg_bf16, 8, 8, 16, 8>>(f); break; // Xe-HPG (Arc A), ARL-H + default: GGML_ABORT("bad XMX combination %d", idx); + } +} + +// AOT with -fsycl-targets=intel_gpu_*: compile each tile body only for targets with its sub-group +// size, since IGC fails on the other ones. A JIT build keeps them all, but each combination lands in +// its own device image (joint_matrix is an optional kernel feature) and only a combination the +// device reports is launched, so the runtime never asks IGC for the others. +#if defined(__SYCL_DEVICE_ONLY__) +# if __SYCL_TARGET_INTEL_GPU_ACM_G10__ || __SYCL_TARGET_INTEL_GPU_ACM_G11__ || __SYCL_TARGET_INTEL_GPU_ACM_G12__ || \ + __SYCL_TARGET_INTEL_GPU_ARL_H__ +# define FG_AOT_SG 8 +# elif __SYCL_TARGET_INTEL_GPU_PVC__ || __SYCL_TARGET_INTEL_GPU_PVC_VG__ || __SYCL_TARGET_INTEL_GPU_BMG_G21__ || \ + __SYCL_TARGET_INTEL_GPU_BMG_G31__ || __SYCL_TARGET_INTEL_GPU_LNL_M__ || __SYCL_TARGET_INTEL_GPU_PTL_H__ || \ + __SYCL_TARGET_INTEL_GPU_PTL_U__ || __SYCL_TARGET_INTEL_GPU_WCL__ || __SYCL_TARGET_INTEL_GPU_NVL_S__ || \ + __SYCL_TARGET_INTEL_GPU_NVL_U__ || __SYCL_TARGET_INTEL_GPU_NVL_P__ +# define FG_AOT_SG 16 +# elif __SYCL_TARGET_INTEL_GPU_TGLLP__ || __SYCL_TARGET_INTEL_GPU_RKL__ || __SYCL_TARGET_INTEL_GPU_ADL_S__ || \ + __SYCL_TARGET_INTEL_GPU_ADL_P__ || __SYCL_TARGET_INTEL_GPU_ADL_N__ || __SYCL_TARGET_INTEL_GPU_DG1__ || \ + __SYCL_TARGET_INTEL_GPU_MTL_U__ || __SYCL_TARGET_INTEL_GPU_MTL_H__ +# define FG_AOT_SG 0 // no XMX +# endif +#endif + +template <int SG> static constexpr bool fg_built() { +#if defined(FG_AOT_SG) + return SG == FG_AOT_SG; +#else + return true; +#endif +} + +// Upper bound on the tile count when total_rows rows are routed to n_as experts: the worst case gives +// each expert one row and fills whole tiles with the rest. The bound depends only on the shape, not +// on the routing, so the pool reuses one buffer every ubatch instead of keeping one per size seen. +static constexpr int64_t grouped_gemm_max_tiles(int64_t total_rows, int64_t n_as, int64_t BN) { + return total_rows <= n_as ? total_rows : n_as + (total_rows - n_as) / BN; +} +// Tiles do not cross experts, so the bound is not ceil(total_rows / BN): 34 rows over 2 experts with +// BN = 16 split 17 + 17 need 2 + 2 tiles, where the ceil gives 3. +static_assert(grouped_gemm_max_tiles(34, 2, 16) == 4); + +// the device lists S with an f32 accumulator and output +template <typename S> static bool fg_device_has_combo(const std::vector<mx::combination> & combinations) { + for (const auto & c : combinations) { + if (c.atype == S::EA::mtype && c.btype == S::EB::mtype && c.ctype == mx::matrix_type::fp32 && + c.dtype == mx::matrix_type::fp32 && + (c.max_msize >= (size_t) S::TM || c.msize == (size_t) S::TM) && + (c.max_nsize >= (size_t) S::TN || c.nsize == (size_t) S::TN) && + (c.max_ksize >= (size_t) S::TK || c.ksize == (size_t) S::TK)) { + return true; + } + } + return false; +} + +template <typename T> static const char * fg_type_name() { + return std::is_same_v<T, fg_half> ? "f16" : std::is_same_v<T, fg_bf16> ? "bf16" : "tf32"; +} + +static std::string fg_combo_name(int idx) { + std::string name; + fg_visit_combo(idx, [&](auto s) { + using S = decltype(s); + name = std::string(fg_type_name<typename S::ta>()) + "x" + fg_type_name<typename S::tb>() + " " + + std::to_string(S::TM) + "x" + std::to_string(S::TN) + "x" + std::to_string(S::TK) + " sg" + + std::to_string(S::SG); + }); + return name; +} + +// Combinations this build has kernels for and the device lists, one bit each. Cached per device: +// on a mixed box the first caller's verdict is not the others'. +static int fg_device_combos(const sycl::device & dev) { + static std::mutex mtx; + static std::unordered_map<sycl::device, int> known; + std::lock_guard<std::mutex> lock(mtx); + const auto it = known.find(dev); + if (it != known.end()) { + return it->second; + } + int available = 0; + try { + const auto combinations = dev.get_info<sycl::ext::oneapi::experimental::info::device::matrix_combinations>(); + const auto sg_sizes = dev.get_info<sycl::info::device::sub_group_sizes>(); + for (int idx = 0; idx < FG_N_COMBOS; ++idx) { + fg_visit_combo(idx, [&](auto s) { + using S = decltype(s); + const bool sg_ok = std::find(sg_sizes.begin(), sg_sizes.end(), (size_t) S::SG) != sg_sizes.end(); + if (sg_ok && fg_device_has_combo<S>(combinations)) { + available |= 1 << idx; + } + }); + } + } catch (const sycl::exception &) { + available = 0; + } + GGML_LOG_INFO("%s: %s: XMX dequant-GEMM combinations available 0x%x, allowed 0x%x\n", __func__, + dev.get_info<sycl::info::device::name>().c_str(), available, g_ggml_sycl_xmx_gather_shapes); + known.emplace(dev, available); + return available; +} + +// Rank of combination S for a src1 of type src1_type, lower is better. Order: +// 1. throughput: a tile that does not spill, then the fastest type class of A and B +// 2. B type equal to the src1 type, so the pack is a plain copy +// 3. B at least as precise as f16 +// 4. the device's native DPAS tile (8 x SG x 32 bytes of K), then the largest M x K +// A costs nothing to convert: the A stage emits any type at the same cost. +template <typename S> static int64_t fg_rank(ggml_type src1_type) { + const int64_t spills = !S::efficient; + const int64_t slow = std::max(S::EA::slow, S::EB::slow); + const int64_t convert = S::EB::src != src1_type; + const int64_t lossy = S::EB::mant < 10; + const int64_t foreign = !(S::TM == 8 && S::TN == S::SG); + const int64_t mk = 1024 - S::TM * S::TK; + return ((((spills * 2 + slow) * 2 + convert) * 2 + lossy) * 2 + foreign) * 2048 + mk; +} + +// whether XMX operands of type mode (a GGML_SYCL_DYNAMIC_PRECISION value) meet the src1 request +// [TAG_GGML_PREC]. f16 lacks the f32 range that BF16 and F32 ask for; an F32 request goes only as far +// down as GGML_SYCL_DYNAMIC_REQUIRED_PRECISION allows. +static bool fg_mode_meets(int mode, int32_t src1_prec) { + if (src1_prec == GGML_PREC_UNDEFINED || src1_prec >= GGML_PREC_F16) { + return true; + } + if (src1_prec >= GGML_PREC_BF16) { + return mode != GGML_SYCL_DYNAMIC_PRECISION_F16; + } + switch (g_ggml_sycl_dynamic_required_precision) { + case GGML_SYCL_DYNAMIC_PRECISION_TF32: return mode == GGML_SYCL_DYNAMIC_PRECISION_TF32; + case GGML_SYCL_DYNAMIC_PRECISION_BF16: return mode != GGML_SYCL_DYNAMIC_PRECISION_F16; + default: return false; + } +} + +// Best allowed combination for this call, or -1 if none. The type is GGML_SYCL_DYNAMIC_PRECISION if it +// meets the src1 request; if not, bf16 then tf32 for a BF16 request (fastest first) and tf32 then bf16 +// for an F32 request (most mantissa first). +static int fg_pick_combo(dpct::queue_ptr stream, ggml_type src1_type, int32_t src1_prec) { + const sycl::device dev = stream->get_device(); + const int allowed = fg_device_combos(dev) & g_ggml_sycl_xmx_gather_shapes; + const bool f32_req = src1_prec != GGML_PREC_UNDEFINED && src1_prec < GGML_PREC_BF16; + const int modes[] = { + g_ggml_sycl_dynamic_precision, + f32_req ? GGML_SYCL_DYNAMIC_PRECISION_TF32 : GGML_SYCL_DYNAMIC_PRECISION_BF16, + f32_req ? GGML_SYCL_DYNAMIC_PRECISION_BF16 : GGML_SYCL_DYNAMIC_PRECISION_TF32, + }; + int best = -1; + int64_t best_rank = 0; + for (int i = 0; i < 3 && best < 0; ++i) { + const int mode = modes[i]; + if (!fg_mode_meets(mode, src1_prec)) { + continue; + } + for (int idx = 0; idx < FG_N_COMBOS; ++idx) { + if (!(allowed & (1 << idx))) { + continue; + } + fg_visit_combo(idx, [&](auto s) { + using S = decltype(s); + if (S::EA::mode != mode || S::EB::mode != mode) { + return; + } + const int64_t rank = fg_rank<S>(src1_type); + if (best < 0 || rank < best_rank) { + best = idx; + best_rank = rank; + } + }); + } + } + // log each distinct decision once + static std::mutex mtx; + static std::set<std::tuple<size_t, int, int32_t, int>> seen; + std::lock_guard<std::mutex> lock(mtx); + if (seen.emplace(std::hash<sycl::device>{}(dev), (int) src1_type, src1_prec, best).second) { + GGML_LOG_INFO("%s: src1 %s, src1 prec %d -> %s\n", __func__, ggml_type_name(src1_type), src1_prec, + best >= 0 ? fg_combo_name(best).c_str() : "none (library GEMM)"); + } + return best; +} + +// src1 [N][K] -> packed [K/V][Npad][V] so B tiles load straight from global memory +template <typename E, typename T_src> +static void fused_gemm_pack_b(const T_src * y, typename E::store * packed, int N, int Npad, int K, + dpct::queue_ptr stream) { + constexpr int V = 4 / sizeof(typename E::store); + const int kqs = K / V; + stream->parallel_for(sycl::range<1>((size_t) Npad * kqs), [=](sycl::id<1> id) { + const int idx = id[0]; + const int n = idx / kqs; + const int kq = idx - n * kqs; + typename E::store vals[V] = {}; + if (n < N) { + const T_src * src = y + (size_t) n * K + V * kq; +#pragma unroll + for (int v = 0; v < V; ++v) { + vals[v] = E::cvt((float) src[v]); + } + } + typename E::store * out = packed + ((size_t) kq * Npad + n) * V; +#pragma unroll + for (int v = 0; v < V; ++v) { + out[v] = vals[v]; + } + }); +} + +// A stage: one lane owns one row and decodes FG_BK values of it per k step, with every scale +// folded into the value so the mad below sees plain A elements. One overload per weight format. +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq4_nl * __restrict__ xrow, const int kb, typename E::pair * a) { + const block_iq4_nl blk = xrow[kb]; + const float d = (float) blk.d; +#pragma unroll + for (int j = 0; j < QK4_NL / 2; j += 2) { + const uint8_t q0 = blk.qs[j]; + const uint8_t q1 = blk.qs[j + 1]; + a[j / 2] = E::make(d * kvalues_iq4nl[q0 & 0xf], d * kvalues_iq4nl[q1 & 0xf]); + a[j / 2 + 8] = E::make(d * kvalues_iq4nl[q0 >> 4], d * kvalues_iq4nl[q1 >> 4]); + } +} + +// iq3_s: k step kb is sub-block kb % 8 of superblock kb / 8. The superblock is 110 bytes, so read +// only the fields of that sub-block instead of copying the block. Same decode as +// dequantize_block_iq3_s: grid entries are taken as dwords and the sign bit is a plain shift. +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq3_s * __restrict__ xrow, const int kb, typename E::pair * a) { + static_assert(QK_K == 256, "the iq3_s A stage assumes 8 sub-blocks per superblock"); + const block_iq3_s * blk = xrow + kb / (QK_K / 32); + const int ib8 = kb % (QK_K / 32); + const uint8_t * qs = blk->qs + 8 * ib8; + const int qh = blk->qh[ib8]; + const float d = (float) blk->d * (1 + 2 * ((blk->scales[ib8 / 2] >> (4 * (ib8 % 2))) & 0xf)); +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint32_t grid1 = iq3s_grid[qs[2 * il + 0] | ((qh << (8 - 2 * il)) & 256)]; + const uint32_t grid2 = iq3s_grid[qs[2 * il + 1] | ((qh << (7 - 2 * il)) & 256)]; + const int signs = blk->signs[4 * ib8 + il]; +#pragma unroll + for (int j = 0; j < 2; ++j) { + const float g1a = (float) ((grid1 >> (16 * j + 0)) & 0xff); + const float g1b = (float) ((grid1 >> (16 * j + 8)) & 0xff); + const float g2a = (float) ((grid2 >> (16 * j + 0)) & 0xff); + const float g2b = (float) ((grid2 >> (16 * j + 8)) & 0xff); + const int s = 2 * j; + a[4 * il + j] = E::make(d * ((signs & (1 << (s + 0))) ? -g1a : g1a), + d * ((signs & (1 << (s + 1))) ? -g1b : g1b)); + a[4 * il + j + 2] = E::make(d * ((signs & (1 << (s + 4))) ? -g2a : g2a), + d * ((signs & (1 << (s + 5))) ? -g2b : g2b)); + } + } +} + +// values per stored block, so a row of K values is K/qk blocks +template <typename block_q_t> struct fg_block_traits; +template <> struct fg_block_traits<block_iq4_nl> { static constexpr int qk = QK4_NL; }; +template <> struct fg_block_traits<block_iq3_s> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_iq3_xxs> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_iq4_xs> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_iq2_xxs> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_iq2_xs> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_iq2_s> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_iq1_s> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_iq1_m> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_q8_0> { static constexpr int qk = QK8_0; }; +template <> struct fg_block_traits<block_q2_K> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_q3_K> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_q4_K> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_q5_K> { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits<block_q6_K> { static constexpr int qk = QK_K; }; + +// The A stages below are the dequantize_block_iq* kernels rewritten for one k step. There a +// work-item handled one quarter (il) of one 32-wide sub-block (ib); here one lane produces the +// whole step, so il becomes a loop and ib is kb inside the superblock. Each quarter yields 8 +// consecutive values, i.e. 4 pairs at a[4*il], so nothing larger than 8 floats is ever live. +#define FG_SUPERBLOCK(T) \ + static_assert(QK_K == 256, "the " #T " A stage assumes 8 sub-blocks per superblock"); \ + const T * blk = xrow + kb / (QK_K / 32); \ + const int ib = kb % (QK_K / 32) + +template <typename E> +static __dpct_inline__ void fg_pack_quarter(const float * __restrict__ t, typename E::pair * a, int il) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + a[4 * il + j] = E::make(t[2 * j], t[2 * j + 1]); + } +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq4_xs * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq4_xs); + // low nibbles fill the first half of the step, high nibbles the second, so the two halves + // land at a[0..7] and a[8..15] and no quarter loop is needed + const float d = (float) blk->d * + ((((blk->scales_l[ib / 2] >> (4 * (ib % 2))) & 0xf) | (((blk->scales_h >> (2 * ib)) & 3) << 4)) - 32); + const uint8_t * q4 = blk->qs + 16 * ib; +#pragma unroll + for (int j = 0; j < 8; ++j) { + a[j] = E::make(d * kvalues_iq4nl[q4[2 * j] & 0xf], d * kvalues_iq4nl[q4[2 * j + 1] & 0xf]); + a[8 + j] = E::make(d * kvalues_iq4nl[q4[2 * j] >> 4], d * kvalues_iq4nl[q4[2 * j + 1] >> 4]); + } +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq3_xxs * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq3_xxs); + const uint8_t * q3 = blk->qs + 8 * ib; + const uint16_t * gas = (const uint16_t *) (blk->qs + QK_K / 4) + 2 * ib; + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float d = (float) blk->d * (0.5f + (aux32 >> 28)) * 0.5f; +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint8_t * grid1 = (const uint8_t *) (iq3xxs_grid + q3[2 * il + 0]); + const uint8_t * grid2 = (const uint8_t *) (iq3xxs_grid + q3[2 * il + 1]); + const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127]; + float t[8]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + t[j + 0] = d * grid1[j] * (signs & kmask_iq2xs[j + 0] ? -1.f : 1.f); + t[j + 4] = d * grid2[j] * (signs & kmask_iq2xs[j + 4] ? -1.f : 1.f); + } + fg_pack_quarter<E>(t, a, il); + } +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq2_xxs * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq2_xxs); + const uint16_t * q2 = blk->qs + 4 * ib; + const uint8_t * aux8 = (const uint8_t *) q2; + const uint32_t aux32 = q2[2] | (q2[3] << 16); + const float d = (float) blk->d * (0.5f + (aux32 >> 28)) * 0.25f; +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint8_t * grid = (const uint8_t *) (iq2xxs_grid + aux8[il]); + const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127]; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + fg_pack_quarter<E>(t, a, il); + } +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq2_xs * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq2_xs); + const uint16_t * q2 = blk->qs + 4 * ib; +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint8_t * grid = (const uint8_t *) (iq2xs_grid + (q2[il] & 511)); + const float d = (float) blk->d * (0.5f + ((blk->scales[ib] >> (4 * (il / 2))) & 0xf)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + fg_pack_quarter<E>(t, a, il); + } +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq2_s * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq2_s); +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint8_t * grid = + (const uint8_t *) (iq2s_grid + (blk->qs[4 * ib + il] | ((blk->qh[ib] << (8 - 2 * il)) & 0x300))); + const float d = (float) blk->d * (0.5f + ((blk->scales[ib] >> (4 * (il / 2))) & 0xf)) * 0.25f; + const uint8_t signs = blk->qs[QK_K / 8 + 4 * ib + il]; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + fg_pack_quarter<E>(t, a, il); + } +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq1_s * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq1_s); + const float delta = blk->qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; + const float d = (float) blk->d * (2 * ((blk->qh[ib] >> 12) & 7) + 1); +#pragma unroll + for (int il = 0; il < 4; ++il) { + uint32_t grid32[2]; + const int8_t * q = (const int8_t *) grid32; + grid32[0] = iq1s_grid_gpu[blk->qs[4 * ib + il] | (((blk->qh[ib] >> (3 * il)) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * (q[j] + delta); + } + fg_pack_quarter<E>(t, a, il); + } +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_iq1_m * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq1_m); + const uint16_t * sc = (const uint16_t *) blk->scales; + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); +#pragma unroll + for (int il = 0; il < 4; ++il) { + const int ib16 = 2 * ib + il / 2; + const float d = (float) scale.f16 * (2 * ((sc[ib16 / 4] >> (3 * (ib16 % 4))) & 0x7) + 1); + const float delta = blk->qh[2 * ib + il / 2] & (0x08 << (4 * (il % 2))) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA; + uint32_t grid32[2]; + const int8_t * q = (const int8_t *) grid32; + grid32[0] = iq1s_grid_gpu[blk->qs[4 * ib + il] | + (((blk->qh[2 * ib + il / 2] >> (4 * (il % 2))) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * (q[j] + delta); + } + fg_pack_quarter<E>(t, a, il); + } +} + + +// q8_0 and the k-quants. Each decode takes the fields of one 32-value sub-block, so the canonical +// layout and the reorder (SoA) layout of reorder_qw() share it and differ only in where the fields +// live, as in dequantize.hpp. +template <typename E> +static __dpct_inline__ void fg_decode_q8_0(const int8_t * __restrict__ qs, const float d, typename E::pair * a) { +#pragma unroll + for (int j = 0; j < QK8_0 / 2; ++j) { + a[j] = E::make(d * qs[2 * j], d * qs[2 * j + 1]); + } +} + +// same unpack as get_scale_min_k4() in dequantize.hpp +static __dpct_inline__ void fg_scale_min_k4(const int j, const uint8_t * __restrict__ q, uint8_t & d, uint8_t & m) { + if (j < 4) { + d = q[j] & 63; + m = q[j + 4] & 63; + } else { + d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + } +} + +template <typename E> +static __dpct_inline__ void fg_decode_q2_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ scales, + const float d, const float dmin, const int ib, typename E::pair * a) { + const uint8_t * q = qs + 32 * (ib / 4); + const int shift = 2 * (ib % 4); +#pragma unroll + for (int j = 0; j < 16; ++j) { + const uint8_t sc = scales[2 * ib + j / 8]; + const float dl = d * (sc & 0xF); + const float ml = dmin * (sc >> 4); + a[j] = E::make(dl * ((q[2 * j] >> shift) & 3) - ml, + dl * ((q[2 * j + 1] >> shift) & 3) - ml); + } +} + +// scales of the two 16-value halves of sub-block ib +static __dpct_inline__ void fg_q3_K_dl(const uint8_t * __restrict__ scales, const float d, const int ib, float * dl) { +#pragma unroll + for (int i = 0; i < 2; ++i) { + const int s = 2 * ib + i; + const int sc = ((scales[s % 8] >> (4 * (s / 8))) & 0xF) | (((scales[8 + s % 4] >> (2 * (s / 4))) & 3) << 4); + dl[i] = d * (sc - 32); + } +} + +template <typename E> +static __dpct_inline__ void fg_decode_q3_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ hmask, + const uint8_t * __restrict__ scales, const float d, const int ib, + typename E::pair * a) { + // block_q3_K is 110 bytes, so qs and hmask are only 2-byte aligned + const uint16_t * q = (const uint16_t *) (qs + 32 * (ib / 4)); + const uint16_t * h = (const uint16_t *) hmask; + const int shift = 2 * (ib % 4); + float dl[2]; + fg_q3_K_dl(scales, d, ib, dl); +#pragma unroll + for (int j = 0; j < 16; ++j) { + // per byte: 2 low bits, plus 4 if the hmask bit is set, minus 4 + const uint32_t v = ((q[j] >> shift) & 0x0303u) | (((h[j] >> ib) & 0x0101u) << 2); + a[j] = E::make(dl[j / 8] * ((int) (v & 0xFF) - 4), dl[j / 8] * ((int) (v >> 8) - 4)); + } +} + +// q4_K sub-block ib (0..7) uses scale/min pair ib and the low (even ib) or high (odd ib) nibbles of +// qs[32 * (ib / 2) ...], as in dequantize_row_q4_K +template <typename E> +static __dpct_inline__ void fg_decode_q4_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ scales, + const sycl::half2 dm, const int ib, typename E::pair * a) { + uint8_t sc, mb; + fg_scale_min_k4(ib, scales, sc, mb); + const float d = (float) dm[0] * sc; + const float m = (float) dm[1] * mb; + const uint8_t * q = qs + 32 * (ib / 2); + const int shift = 4 * (ib % 2); +#pragma unroll + for (int j = 0; j < 16; ++j) { + a[j] = E::make(d * ((q[2 * j] >> shift) & 0xF) - m, d * ((q[2 * j + 1] >> shift) & 0xF) - m); + } +} + +// q5_K: q4_K plus one high bit per value, bit ib of qh[l] +template <typename E> +static __dpct_inline__ void fg_decode_q5_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ qh, + const uint8_t * __restrict__ scales, const sycl::half2 dm, const int ib, + typename E::pair * a) { + uint8_t sc, mb; + fg_scale_min_k4(ib, scales, sc, mb); + const float d = (float) dm[0] * sc; + const float m = (float) dm[1] * mb; + const uint8_t * q = qs + 32 * (ib / 2); + const int shift = 4 * (ib % 2); +#pragma unroll + for (int j = 0; j < 16; ++j) { + const int l = 2 * j; + a[j] = E::make(d * (((q[l] >> shift) & 0xF) | (((qh[l] >> ib) & 1) << 4)) - m, + d * (((q[l + 1] >> shift) & 0xF) | (((qh[l + 1] >> ib) & 1) << 4)) - m); + } +} + +// q6_K: sub-block ib is quarter r = ib % 4 of half h = ib / 4. The half selects ql + 64h, qh + 32h +// and scales + 8h; the quarter selects ql + 32(r & 1), the ql nibble r / 2, the qh bit pair r and +// scales + 2r, as in dequantize_row_q6_K. The scale changes at value 16 of the sub-block. +template <typename E> +static __dpct_inline__ void fg_decode_q6_K(const uint8_t * __restrict__ ql, const uint8_t * __restrict__ qh, + const int8_t * __restrict__ scales, const float d, const int ib, + typename E::pair * a) { + const int h = ib / 4; + const int r = ib % 4; + const uint8_t * q = ql + 64 * h + 32 * (r & 1); + const uint8_t * hb = qh + 32 * h; + const int8_t * sc = scales + 8 * h + 2 * r; +#pragma unroll + for (int j = 0; j < 16; ++j) { + const int l = 2 * j; + const float dl = d * sc[j / 8]; + const int q0 = (((q[l] >> (4 * (r / 2))) & 0xF) | (((hb[l] >> (2 * r)) & 3) << 4)) - 32; + const int q1 = (((q[l + 1] >> (4 * (r / 2))) & 0xF) | (((hb[l + 1] >> (2 * r)) & 3) << 4)) - 32; + a[j] = E::make(dl * q0, dl * q1); + } +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_q8_0 * __restrict__ xrow, const int kb, typename E::pair * a) { + fg_decode_q8_0<E>(xrow[kb].qs, (float) xrow[kb].d, a); +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_q2_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q2_K); + fg_decode_q2_K<E>(blk->qs, blk->scales, (float) blk->dm[0], (float) blk->dm[1], ib, a); +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_q3_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q3_K); + fg_decode_q3_K<E>(blk->qs, blk->hmask, blk->scales, (float) blk->d, ib, a); +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_q4_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q4_K); + fg_decode_q4_K<E>(blk->qs, blk->scales, blk->dm, ib, a); +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_q5_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q5_K); + fg_decode_q5_K<E>(blk->qs, blk->qh, blk->scales, blk->dm, ib, a); +} + +template <typename E> +static __dpct_inline__ void fg_stage_a(const block_q6_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q6_K); + fg_decode_q6_K<E>(blk->ql, blk->qh, blk->scales, (float) blk->d, ib, a); +} + +// Reorder (SoA) layout: each block field is one stream over the nblocks of the matrix (of the expert +// slice for MUL_MAT_ID), in the order reorder_qw() writes them. Block ib holds k step kb. +template <typename block_q_t> struct fg_soa { + static constexpr bool supported = false; +}; + +template <> struct fg_soa<block_q8_0> { + static constexpr bool supported = true; + // [qs][d] + template <typename E> + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int, + typename E::pair * a) { + const float d = (float) ((const sycl::half *) (x + nblocks * QK8_0))[ib]; + fg_decode_q8_0<E>((const int8_t *) x + ib * QK8_0, d, a); + } +}; + +template <> struct fg_soa<block_q2_K> { + static constexpr bool supported = true; + + // [qs][scales][dm] + template <typename E> + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * scales = x + nblocks * (QK_K / 4); + const sycl::half2 dm = ((const sycl::half2 *) (scales + nblocks * (QK_K / 16)))[ib]; + fg_decode_q2_K<E>(x + ib * (QK_K / 4), scales + ib * (QK_K / 16), (float) dm[0], (float) dm[1], + kb % (QK_K / 32), a); + } +}; + +template <> struct fg_soa<block_q3_K> { + static constexpr bool supported = true; + + // [qs][hmask][scales][d] + template <typename E> + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * hmask = x + nblocks * (QK_K / 4); + const uint8_t * scales = hmask + nblocks * (QK_K / 8); + const float d = (float) ((const sycl::half *) (scales + nblocks * 12))[ib]; + const int sub = kb % (QK_K / 32); + // 16-byte aligned, see opt_for_reorder_id + using v4 = sycl::vec<uint32_t, 4>; + const v4 * q = (const v4 *) (x + ib * (QK_K / 4) + 32 * (sub / 4)); + const v4 * h = (const v4 *) (hmask + ib * (QK_K / 8)); + const v4 qv[2] = { q[0], q[1] }; + const v4 hv[2] = { h[0], h[1] }; + const int shift = 2 * (sub % 4); + float dl[2]; + fg_q3_K_dl(scales + ib * 12, d, sub, dl); +#pragma unroll + for (int w = 0; w < 8; ++w) { + const uint32_t v = ((qv[w / 4][w % 4] >> shift) & 0x03030303u) | (((hv[w / 4][w % 4] >> sub) & 0x01010101u) << 2); + const float f = dl[w / 4]; + a[2 * w] = E::make(f * ((int) (v & 0xFF) - 4), f * ((int) ((v >> 8) & 0xFF) - 4)); + a[2 * w + 1] = E::make(f * ((int) ((v >> 16) & 0xFF) - 4), f * ((int) (v >> 24) - 4)); + } + } +}; + +template <> struct fg_soa<block_q4_K> { + static constexpr bool supported = true; + // [qs][scales][dm] + template <typename E> + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * scales = x + nblocks * (QK_K / 2); + const sycl::half2 dm = ((const sycl::half2 *) (scales + nblocks * K_SCALE_SIZE))[ib]; + fg_decode_q4_K<E>(x + ib * (QK_K / 2), scales + ib * K_SCALE_SIZE, dm, kb % (QK_K / 32), a); + } +}; + +template <> struct fg_soa<block_q5_K> { + static constexpr bool supported = true; + // [qs][qh][scales][dm] + template <typename E> + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * qh = x + nblocks * (QK_K / 2); + const uint8_t * scales = qh + nblocks * (QK_K / 8); + const sycl::half2 dm = ((const sycl::half2 *) (scales + nblocks * K_SCALE_SIZE))[ib]; + fg_decode_q5_K<E>(x + ib * (QK_K / 2), qh + ib * (QK_K / 8), scales + ib * K_SCALE_SIZE, dm, + kb % (QK_K / 32), a); + } +}; + +template <> struct fg_soa<block_q6_K> { + static constexpr bool supported = true; + // [ql][qh][scales][d] + template <typename E> + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * qh = x + nblocks * (QK_K / 2); + const uint8_t * scales = qh + nblocks * (QK_K / 4); + const float d = (float) ((const sycl::half *) (scales + nblocks * (QK_K / 16)))[ib]; + fg_decode_q6_K<E>(x + ib * (QK_K / 2), qh + ib * (QK_K / 4), (const int8_t *) scales + ib * (QK_K / 16), d, + kb % (QK_K / 32), a); + } +}; + + +// one SG_ROWS x BN output tile: B columns [b0, b0 + BN) of packed_b go to dst columns [n0, n1), +// n1 - n0 <= BN +template <typename S, typename block_q_t, bool reordered> +static void fused_dequant_gemm_tile( + const block_q_t * __restrict__ x, + const typename S::tsb * __restrict__ packed_b, + float * __restrict__ dst, + const int M, const int Npad, const int K, const int ldd, + const int b0, const int n0, const int n1, + sycl::local_accessor<typename S::tsa, 1> tile_a, + sycl::local_accessor<float, 1> tile_c, + const sycl::nd_item<2> & item, ggml_sycl_grouped_gemm_rows dst_rows = {}) { + if constexpr (fg_built<S::SG>()) { + using EA = typename S::EA; + using TA = typename S::ta; + using TB = typename S::tb; + const auto sg = item.get_sub_group(); + const int sg_id = sg.get_group_id()[0]; + const int lane = sg.get_local_id()[0]; + const int m0 = item.get_group(1) * S::SG_ROWS; + const int nstep = K / FG_BK; + const int a_base = sg_id * S::SG_ROWS * FG_BK; + const int c_base = sg_id * S::SG_ROWS * S::BN; + + mx::joint_matrix<sycl::sub_group, float, mx::use::accumulator, S::TM, S::TN> acc[S::MT][S::NT]; +#pragma unroll + for (int mt = 0; mt < S::MT; ++mt) { +#pragma unroll + for (int nt = 0; nt < S::NT; ++nt) { + mx::joint_matrix_fill(sg, acc[mt][nt], 0.0f); + } + } + + // lane decodes rows lane, lane + SG, ... of the sub-group's SG_ROWS + constexpr int KPB = fg_block_traits<block_q_t>::qk / FG_BK; // k steps per block + const size_t bpr = K / fg_block_traits<block_q_t>::qk; + const size_t nblocks = (size_t) M * bpr; + size_t row_blk[S::RPL]; + bool row_ok[S::RPL]; + typename EA::pair * a[S::RPL]; +#pragma unroll + for (int r = 0; r < S::RPL; ++r) { + const int row = m0 + r * S::SG + lane; + row_ok[r] = row < M; + row_blk[r] = (size_t) (row_ok[r] ? row : 0) * bpr; + a[r] = (typename EA::pair *) &tile_a[a_base + (r * S::SG + lane) * FG_BK]; + } + + const auto b_ptr = sycl::address_space_cast<sycl::access::address_space::global_space, + sycl::access::decorated::no>(packed_b); + const int b_stride = Npad * S::VNNI; + + const int kb_begin = (sg_id * nstep) / FG_KSPLIT; + const int kb_end = ((sg_id + 1) * nstep) / FG_KSPLIT; + for (int kb = kb_begin; kb < kb_end; ++kb) { +#pragma unroll + for (int r = 0; r < S::RPL; ++r) { + if (row_ok[r]) { + if constexpr (reordered) { + fg_soa<block_q_t>::template stage<EA>((const uint8_t *) x, nblocks, row_blk[r] + kb / KPB, kb, + a[r]); + } else { + fg_stage_a<EA>(x + row_blk[r], kb, a[r]); + } + } else { +#pragma unroll + for (int j = 0; j < FG_BK / 2; ++j) { + a[r][j] = EA::make(0.0f, 0.0f); + } + } + } + sycl::group_barrier(sg); + +#pragma unroll + for (int kt = 0; kt < FG_BK / S::TK; ++kt) { + const int kq0 = (kb * FG_BK + kt * S::TK) / S::VNNI; + mx::joint_matrix<sycl::sub_group, TB, mx::use::b, S::TK, S::TN, S::b_layout> sub_b[S::NT]; +#pragma unroll + for (int nt = 0; nt < S::NT; ++nt) { + mx::joint_matrix_load(sg, sub_b[nt], b_ptr + (size_t) kq0 * b_stride + (b0 + nt * S::TN) * S::VNNI, b_stride); + } +#pragma unroll + for (int mt = 0; mt < S::MT; ++mt) { + mx::joint_matrix<sycl::sub_group, TA, mx::use::a, S::TM, S::TK, mx::layout::row_major> sub_a; + mx::joint_matrix_load(sg, sub_a, + tile_a.template get_multi_ptr<sycl::access::decorated::no>() + a_base + (mt * S::TM) * FG_BK + kt * S::TK, + FG_BK); +#pragma unroll + for (int nt = 0; nt < S::NT; ++nt) { + mx::joint_matrix_mad(sg, acc[mt][nt], sub_a, sub_b[nt], acc[mt][nt]); + } + } + } + // the next step overwrites tile_a + sycl::group_barrier(sg); + } + +#pragma unroll + for (int mt = 0; mt < S::MT; ++mt) { +#pragma unroll + for (int nt = 0; nt < S::NT; ++nt) { + mx::joint_matrix_store(sg, acc[mt][nt], + tile_c.template get_multi_ptr<sycl::access::decorated::no>() + c_base + (mt * S::TM) * S::BN + nt * S::TN, + S::BN, mx::layout::row_major); + } + } + sycl::group_barrier(item.get_group()); + + // sum the K splits; consecutive lanes write consecutive rows of one dst column + for (int idx = item.get_local_linear_id(); idx < S::SG_ROWS * S::BN; idx += S::WG_SIZE) { + const int r = idx % S::SG_ROWS; + const int c = idx / S::SG_ROWS; + const int m = m0 + r; + const int n = n0 + c; + if (m < M && n < n1) { + float sum = 0.0f; +#pragma unroll + for (int s = 0; s < FG_KSPLIT; ++s) { + sum += tile_c[s * S::SG_ROWS * S::BN + r * S::BN + c]; + } + if (dst_rows.map) { + const mmid_row_mapping row = dst_rows.map[n]; + float * out = (float *) (dst_rows.base + (row.i1 % dst_rows.ne1) * dst_rows.nb1 + row.i2 * dst_rows.nb2); + out[m] = sum; + } else { + dst[(size_t) n * ldd + m] = sum; + } + } + } + } +} + +template <typename S, typename block_q_t, bool reordered = false> +static void fused_dequant_gemm_launch(const void * src0, const typename S::tsb * packed, float * dst, const int M, + const int N, const int Npad, const int K, const int ldd, + const int64_t groups_n, const int64_t groups_m, dpct::queue_ptr stream) { + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor<typename S::tsa, 1> tile_a(FG_KSPLIT * S::SG_ROWS * FG_BK, cgh); + sycl::local_accessor<float, 1> tile_c(FG_KSPLIT * S::SG_ROWS * S::BN, cgh); + cgh.parallel_for( + sycl::nd_range<2>(sycl::range<2>(groups_n, groups_m * S::WG_SIZE), sycl::range<2>(1, S::WG_SIZE)), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(S::SG)]] { + const int n0 = item.get_group(0) * S::BN; + fused_dequant_gemm_tile<S, block_q_t, reordered>((const block_q_t *) src0, packed, dst, M, Npad, K, + ldd, n0, n0, N, tile_a, tile_c, item); + }); + }); +} + +// grouped: work-group (t, mt) is tile t of the schedule; its B columns sit at t * BN +template <typename S, typename block_q_t, bool reordered> +static void grouped_dequant_gemm_launch(const char * src0_dd, const size_t expert_stride, + const ggml_sycl_grouped_gemm_tile * tiles_ptr, const typename S::tsb * packed, + float * dst, const int M, const int Npad, const int K, const int64_t n_tiles, + const int64_t groups_m, dpct::queue_ptr stream, + ggml_sycl_grouped_gemm_rows dst_rows = {}) { + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor<typename S::tsa, 1> tile_a(FG_KSPLIT * S::SG_ROWS * FG_BK, cgh); + sycl::local_accessor<float, 1> tile_c(FG_KSPLIT * S::SG_ROWS * S::BN, cgh); + cgh.parallel_for( + sycl::nd_range<2>(sycl::range<2>(n_tiles, groups_m * S::WG_SIZE), sycl::range<2>(1, S::WG_SIZE)), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(S::SG)]] { + const int t = item.get_group(0); + const ggml_sycl_grouped_gemm_tile tile = tiles_ptr[t]; + if (tile.n0 == tile.n1) { + return; + } + const block_q_t * x = (const block_q_t *) (src0_dd + (size_t) tile.expert * expert_stride); + fused_dequant_gemm_tile<S, block_q_t, reordered>(x, packed, dst, M, Npad, K, M, t * S::BN, tile.n0, + tile.n1, tile_a, tile_c, item, dst_rows); + }); + }); +} + +// src1 f32 rows -> packed [K/V][n_tiles*BN][V], tile t holds its rows [n0, n1) at columns t*BN.., +// zero past n1. The column runs fastest so a sub-group writes one contiguous run. +template <typename S> +static void grouped_gemm_pack_b(const float * y, typename S::tsb * packed, const ggml_sycl_grouped_gemm_tile * tiles, + int Npad, int K, dpct::queue_ptr stream) { + using E = typename S::EB; + constexpr int V = S::VNNI; + const int kqs = K / V; + stream->parallel_for(sycl::range<1>((size_t) Npad * kqs), [=](sycl::id<1> id) { + const size_t idx = id[0]; + const int kq = idx / Npad; + const int n = idx - (size_t) kq * Npad; + const ggml_sycl_grouped_gemm_tile tile = tiles[n / S::BN]; + const int row = tile.n0 + n % S::BN; + // one guarded load run per work-item, as a per-element select costs ~1.5% prefill + typename S::tsb vals[V] = {}; + if (row < tile.n1) { + const float * src = y + (size_t) row * K + V * kq; +#pragma unroll + for (int v = 0; v < V; ++v) { + vals[v] = E::cvt(src[v]); + } + } + typename S::tsb * out = packed + ((size_t) kq * Npad + n) * V; +#pragma unroll + for (int v = 0; v < V; ++v) { + out[v] = vals[v]; + } + }); +} + +// q8_0 and the k-quants take only the grouped path. The plain kernel decodes A again for every BN columns +// of a dense batch, and for these formats that costs more than the one dequantization of the library GEMM. +template <typename T> static constexpr bool fg_plain_ok() { + return !std::is_same_v<T, block_q8_0> && !std::is_same_v<T, block_q2_K> && !std::is_same_v<T, block_q3_K> && + !std::is_same_v<T, block_q4_K> && !std::is_same_v<T, block_q5_K> && + !std::is_same_v<T, block_q6_K>; +} + +template <typename T, bool R> struct fg_tag { + using type = T; + static constexpr bool reordered = R; +}; + +// calls f(fg_tag<block_q_t, reordered>{}) for the weight format and layout; false if it has no A stage +template <typename T, typename F> static bool fg_visit_layout(bool reordered, F && f) { + if (!reordered) { + f(fg_tag<T, false>{}); + return true; + } + if constexpr (fg_soa<T>::supported) { + f(fg_tag<T, true>{}); + return true; + } + return false; +} + +template <typename F> static bool fg_visit_type(ggml_type type, bool reordered, F && f) { + switch (type) { + case GGML_TYPE_IQ4_NL: return fg_visit_layout<block_iq4_nl>(reordered, f); + case GGML_TYPE_IQ3_S: return fg_visit_layout<block_iq3_s>(reordered, f); + case GGML_TYPE_IQ4_XS: return fg_visit_layout<block_iq4_xs>(reordered, f); + case GGML_TYPE_IQ3_XXS: return fg_visit_layout<block_iq3_xxs>(reordered, f); + case GGML_TYPE_IQ2_XXS: return fg_visit_layout<block_iq2_xxs>(reordered, f); + case GGML_TYPE_IQ2_XS: return fg_visit_layout<block_iq2_xs>(reordered, f); + case GGML_TYPE_IQ2_S: return fg_visit_layout<block_iq2_s>(reordered, f); + case GGML_TYPE_IQ1_S: return fg_visit_layout<block_iq1_s>(reordered, f); + case GGML_TYPE_IQ1_M: return fg_visit_layout<block_iq1_m>(reordered, f); + case GGML_TYPE_Q8_0: return fg_visit_layout<block_q8_0>(reordered, f); + case GGML_TYPE_Q2_K: return fg_visit_layout<block_q2_K>(reordered, f); + case GGML_TYPE_Q3_K: return fg_visit_layout<block_q3_K>(reordered, f); + case GGML_TYPE_Q4_K: return fg_visit_layout<block_q4_K>(reordered, f); + case GGML_TYPE_Q5_K: return fg_visit_layout<block_q5_K>(reordered, f); + case GGML_TYPE_Q6_K: return fg_visit_layout<block_q6_K>(reordered, f); + default: return false; + } +} + +template <typename S> +static bool fg_fused_run(ggml_type src0_type, bool src0_reordered, const void * src0, const void * src1, ggml_type src1_type, + float * dst, int64_t M, int64_t N, int64_t K, int64_t ldd, ggml_sycl_pool & pool, + dpct::queue_ptr stream) { + const int64_t groups_n = (N + S::BN - 1) / S::BN; + const int64_t groups_m = (M + S::SG_ROWS - 1) / S::SG_ROWS; + const int Npad = (int) (groups_n * S::BN); + + // src1 is read in its own type: one pass, converted in registers only if B differs + ggml_sycl_pool_alloc<typename S::tsb> packed_b(pool, (size_t) K * Npad); + if (src1_type == GGML_TYPE_F16) { + fused_gemm_pack_b<typename S::EB>((const sycl::half *) src1, packed_b.get(), (int) N, Npad, (int) K, stream); + } else if (src1_type == GGML_TYPE_BF16) { + fused_gemm_pack_b<typename S::EB>((const fg_bf16 *) src1, packed_b.get(), (int) N, Npad, (int) K, stream); + } else { + fused_gemm_pack_b<typename S::EB>((const float *) src1, packed_b.get(), (int) N, Npad, (int) K, stream); + } + + const typename S::tsb * packed = packed_b.get(); + return fg_visit_type(src0_type, src0_reordered, [&](auto tag) { + using block_q_t = typename decltype(tag)::type; + if constexpr (fg_plain_ok<block_q_t>()) { + fused_dequant_gemm_launch<S, block_q_t, decltype(tag)::reordered>(src0, packed, dst, (int) M, (int) N, + Npad, (int) K, (int) ldd, groups_n, + groups_m, stream); + } + }); +} + +bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, bool src0_reordered, const void * src0, const void * src1, ggml_type src1_type, + int32_t src1_prec, float * dst, int64_t M, int64_t N, int64_t K, int64_t ldd, + ggml_sycl_pool & pool, dpct::queue_ptr stream) { + // every BN columns dequantize A again, so wide N is left to the library GEMM. + // src0_reordered: the persistent weight reorder may have been installed on this + // tensor by an earlier small-batch mul_mat; the A stage then has to decode the + // SoA layout. Types without an SoA stage (fg_soa<T>::supported == false) + // decline via fg_visit_layout and the caller falls through to the + // reorder-aware conversion route. + bool plain_ok = false; + fg_visit_type(src0_type, src0_reordered, [&](auto tag) { plain_ok = fg_plain_ok<typename decltype(tag)::type>(); }); + if (g_ggml_sycl_dynamic_precision == GGML_SYCL_DYNAMIC_PRECISION_F32 || + !ggml_sycl_xmx_gather_type_enabled(src0_type) || !plain_ok) { + return false; + } + if (src1_type != GGML_TYPE_F32 && src1_type != GGML_TYPE_F16 && src1_type != GGML_TYPE_BF16) { + return false; + } + if (!ggml_sycl_fused_dequant_gemm_shape_ok(src0_type, M, N, K, ldd)) { + return false; + } + const int combo = fg_pick_combo(stream, src1_type, src1_prec); + if (combo < 0) { + return false; + } + bool launched = false; + fg_visit_combo(combo, [&](auto s) { + launched = fg_fused_run<decltype(s)>(src0_type, src0_reordered, src0, src1, src1_type, dst, M, N, K, ldd, pool, stream); + }); + return launched; +} + +template <typename S> +static bool fg_grouped_run(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, + const float * src1, float * dst, const int64_t * expert_row_offsets, int64_t n_as, int64_t M, + int64_t K, std::vector<ggml_sycl_grouped_gemm_tile> & tiles, ggml_sycl_pool & pool, + dpct::queue_ptr stream) { + // the host knows every slice, so it lays out the work-groups: no search on the device + tiles.clear(); + for (int64_t e = 0; e < n_as; ++e) { + const int64_t end = expert_row_offsets[e + 1]; + for (int64_t n0 = expert_row_offsets[e]; n0 < end; n0 += S::BN) { + tiles.push_back({ (int32_t) e, (int32_t) n0, (int32_t) std::min<int64_t>(n0 + S::BN, end) }); + } + } + const int64_t n_tiles = tiles.size(); + const int64_t groups_m = (M + S::SG_ROWS - 1) / S::SG_ROWS; + const int Npad = (int) (n_tiles * S::BN); + + const int64_t max_tiles = grouped_gemm_max_tiles(expert_row_offsets[n_as], n_as, S::BN); + GGML_ASSERT(n_tiles <= max_tiles); + ggml_sycl_pool_alloc<ggml_sycl_grouped_gemm_tile> tiles_dev(pool, max_tiles); + SYCL_CHECK(CHECK_TRY_ERROR( + stream->memcpy(tiles_dev.get(), tiles.data(), n_tiles * sizeof(ggml_sycl_grouped_gemm_tile)))); + + ggml_sycl_pool_alloc<typename S::tsb> packed_b(pool, (size_t) K * max_tiles * S::BN); + grouped_gemm_pack_b<S>(src1, packed_b.get(), tiles_dev.get(), Npad, (int) K, stream); + + const typename S::tsb * packed = packed_b.get(); + const ggml_sycl_grouped_gemm_tile * tiles_ptr = tiles_dev.get(); + const char * src0_dd = (const char *) src0_base; + return fg_visit_type(src0_type, reordered, [&](auto tag) { + using T = decltype(tag); + grouped_dequant_gemm_launch<S, typename T::type, T::reordered>(src0_dd, expert_stride, tiles_ptr, packed, dst, + (int) M, Npad, (int) K, n_tiles, groups_m, + stream); + }); +} + +bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, + const float * src1, int32_t src1_prec, float * dst, + const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K, + int64_t total_rows, std::vector<ggml_sycl_grouped_gemm_tile> & tiles, + ggml_sycl_pool & pool, dpct::queue_ptr stream) { + int64_t n_active = 0; + for (int64_t e = 0; e < n_as; ++e) { + n_active += expert_row_offsets[e + 1] > expert_row_offsets[e]; + } + if (g_ggml_sycl_dynamic_precision == GGML_SYCL_DYNAMIC_PRECISION_F32 || + !ggml_sycl_xmx_gather_type_enabled(src0_type) || !fg_visit_type(src0_type, reordered, [](auto) {})) { + return false; + } + if (!ggml_sycl_grouped_dequant_gemm_shape_ok(src0_type, M, K, total_rows, n_active)) { + return false; + } + const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec); + if (combo < 0) { + return false; + } + bool launched = false; + fg_visit_combo(combo, [&](auto s) { + launched = fg_grouped_run<decltype(s)>(src0_type, reordered, src0_base, expert_stride, src1, dst, + expert_row_offsets, n_as, M, K, tiles, pool, stream); + }); + return launched; +} + +// One work-group sorts routes by expert and builds the tile table. +static constexpr int MMID_SCHED_WG = 256; + +using mmid_sched_atomic = sycl::atomic_ref<uint32_t, sycl::memory_order::relaxed, + sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + +bool ggml_sycl_build_mmid_schedule(const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids, + int64_t n_tokens, int64_t n_tiles_max, mmid_row_mapping * row_mapping, + ggml_sycl_grouped_gemm_tile * tiles, uint32_t * expert_offsets, + dpct::queue_ptr stream) { + if (n_as <= 0 || n_as > GGML_SYCL_MMID_SCHED_MAX_EXPERTS || n_ids <= 0 || n_tokens <= 0 || n_ids > INT32_MAX / n_tokens) { + return false; + } + const int64_t n_routes = n_tokens * n_ids; + if (n_tiles_max < grouped_gemm_max_tiles(n_routes, n_as, GGML_SYCL_FG_BN)) { + return false; + } + + stream->submit([&](sycl::handler & cgh) { + // l_cnt is reused: route counts, then the write cursors, then the tile count per expert + sycl::local_accessor<uint32_t, 1> l_cnt(n_as + 1, cgh); + sycl::local_accessor<uint32_t, 1> l_off(n_as + 1, cgh); + sycl::local_accessor<uint32_t, 1> l_toff(n_as + 1, cgh); + cgh.parallel_for( + sycl::nd_range<1>(MMID_SCHED_WG, MMID_SCHED_WG), + [=](sycl::nd_item<1> item) { + const auto grp = item.get_group(); + const int64_t tid = item.get_local_id(0); + uint32_t * cnt = l_cnt.get_multi_ptr<sycl::access::decorated::no>().get(); + uint32_t * off = l_off.get_multi_ptr<sycl::access::decorated::no>().get(); + uint32_t * toff = l_toff.get_multi_ptr<sycl::access::decorated::no>().get(); + + for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) { + cnt[e] = 0; + } + // a dropped route must leave a row that names (0, 0) rather than stale memory + for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) { + row_mapping[r] = { 0, 0 }; + } + sycl::group_barrier(grp); + + for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) { + const int64_t token = r / n_ids; + const int64_t slot = r - token * n_ids; + const int32_t * id_row = (const int32_t *) ((const char *) ids_dev + token * ids_token_stride); + const int32_t e = id_row[slot]; + if (e >= 0 && e < n_as) { + mmid_sched_atomic(cnt[e]).fetch_add(1u); + } + } + sycl::group_barrier(grp); + + sycl::joint_exclusive_scan(grp, cnt, cnt + n_as, off, 0u, sycl::plus<uint32_t>()); + sycl::group_barrier(grp); + if (tid == 0) { + off[n_as] = off[n_as - 1] + cnt[n_as - 1]; + } + sycl::group_barrier(grp); + + for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) { + cnt[e] = off[e]; + } + sycl::group_barrier(grp); + + // Atomic arrival order can differ from the host sort; each expert gets the same rows. + for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) { + const int64_t token = r / n_ids; + const int64_t slot = r - token * n_ids; + const int32_t * id_row = (const int32_t *) ((const char *) ids_dev + token * ids_token_stride); + const int32_t e = id_row[slot]; + if (e >= 0 && e < n_as) { + const uint32_t pos = mmid_sched_atomic(cnt[e]).fetch_add(1u); + row_mapping[pos] = { (int32_t) slot, (int32_t) token }; + } + } + sycl::group_barrier(grp); + + for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) { + cnt[e] = (off[e + 1] - off[e] + GGML_SYCL_FG_BN - 1) / GGML_SYCL_FG_BN; + } + sycl::group_barrier(grp); + + sycl::joint_exclusive_scan(grp, cnt, cnt + n_as, toff, 0u, sycl::plus<uint32_t>()); + sycl::group_barrier(grp); + if (tid == 0) { + toff[n_as] = toff[n_as - 1] + cnt[n_as - 1]; + } + sycl::group_barrier(grp); + + const uint32_t n_tiles = toff[n_as]; + for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) { + const uint32_t base = off[e]; + const uint32_t end = off[e + 1]; + for (uint32_t j = 0; j < cnt[e]; ++j) { + const uint32_t n0 = base + j * GGML_SYCL_FG_BN; + const uint32_t n1 = n0 + GGML_SYCL_FG_BN < end ? n0 + GGML_SYCL_FG_BN : end; + tiles[toff[e] + j] = { (int32_t) e, (int32_t) n0, (int32_t) n1 }; + } + } + // the launch is bounded, so the unused tail must be tiles the GEMM skips + for (int64_t t = n_tiles + tid; t < n_tiles_max; t += MMID_SCHED_WG) { + tiles[t] = { 0, 0, 0 }; + } + for (int64_t e = tid; e <= n_as; e += MMID_SCHED_WG) { + expert_offsets[e] = off[e]; + } + }); + }); + return true; +} + +template <typename S> +static bool fg_grouped_device_run(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, + ggml_sycl_grouped_gemm_rows src1, ggml_sycl_grouped_gemm_rows dst, + const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids, + int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool, + dpct::queue_ptr stream) { + if constexpr (S::BN != GGML_SYCL_FG_BN) { + return false; + } else { + const int64_t total_rows = n_tokens * n_ids; + const int64_t max_tiles = grouped_gemm_max_tiles(total_rows, n_as, S::BN); + if (max_tiles > INT32_MAX / S::BN) { + return false; + } + const int Npad = (int) (max_tiles * S::BN); + ggml_sycl_pool_alloc<mmid_row_mapping> rows(pool, total_rows); + ggml_sycl_pool_alloc<ggml_sycl_grouped_gemm_tile> tiles(pool, max_tiles); + ggml_sycl_pool_alloc<uint32_t> offsets(pool, n_as + 1); + if (!ggml_sycl_build_mmid_schedule(ids_dev, ids_token_stride, n_as, n_ids, n_tokens, max_tiles, + rows.get(), tiles.get(), offsets.get(), stream)) { + return false; + } + src1.map = rows.get(); + dst.map = rows.get(); + ggml_sycl_pool_alloc<typename S::tsb> packed_b(pool, (size_t) K * Npad); + const ggml_sycl_grouped_gemm_tile * tile_ptr = tiles.get(); + typename S::tsb * packed = packed_b.get(); + using E = typename S::EB; + constexpr int V = S::VNNI; + stream->parallel_for(sycl::range<1>((size_t) Npad * (K / V)), [=](sycl::id<1> id) { + const size_t idx = id[0]; + const int kq = idx / Npad; + const int n = idx - (size_t) kq * Npad; + const ggml_sycl_grouped_gemm_tile tile = tile_ptr[n / S::BN]; + const int row = tile.n0 + n % S::BN; + typename S::tsb vals[V] = {}; + if (row < tile.n1) { + const mmid_row_mapping mapping = src1.map[row]; + const float * in = (const float *) (src1.base + (mapping.i1 % src1.ne1) * src1.nb1 + mapping.i2 * src1.nb2) + V * kq; +#pragma unroll + for (int v = 0; v < V; ++v) { + vals[v] = E::cvt(in[v]); + } + } + typename S::tsb * out = packed + ((size_t) kq * Npad + n) * V; +#pragma unroll + for (int v = 0; v < V; ++v) { + out[v] = vals[v]; + } + }); + const int64_t groups_m = (M + S::SG_ROWS - 1) / S::SG_ROWS; + return fg_visit_type(src0_type, reordered, [&](auto tag) { + using T = decltype(tag); + grouped_dequant_gemm_launch<S, typename T::type, T::reordered>((const char *) src0_base, expert_stride, + tile_ptr, packed, (float *) dst.base, (int) M, Npad, (int) K, max_tiles, groups_m, stream, dst); + }); + } +} + +bool ggml_sycl_grouped_dequant_gemm_device_supported(ggml_type src0_type, bool reordered, int32_t src1_prec, + int64_t n_as, int64_t n_ids, int64_t n_tokens, + int64_t M, int64_t K, dpct::queue_ptr stream) { + if (n_as <= 0 || n_as > GGML_SYCL_MMID_SCHED_MAX_EXPERTS || n_ids <= 0 || n_tokens <= 0 || + n_ids > INT32_MAX / n_tokens) { + return false; + } + const int64_t total_rows = n_ids * n_tokens; + // The active expert count is unknown until the GPU builds the schedule. + if (g_ggml_sycl_dynamic_precision == GGML_SYCL_DYNAMIC_PRECISION_F32 || + !ggml_sycl_xmx_gather_type_enabled(src0_type) || !fg_visit_type(src0_type, reordered, [](auto) {}) || + !ggml_sycl_grouped_dequant_gemm_shape_ok(src0_type, M, K, total_rows, std::min(n_as, total_rows), + (g_ggml_sycl_mmid_sched & GGML_SYCL_MMID_SCHED_UNBOUNDED) != 0)) { + return false; + } + const sycl::device device = stream->get_device(); + if (device.get_info<sycl::info::device::max_work_group_size>() < MMID_SCHED_WG || + device.get_info<sycl::info::device::local_mem_size>() < 3 * (n_as + 1) * sizeof(uint32_t)) { + return false; + } + if (grouped_gemm_max_tiles(total_rows, n_as, GGML_SYCL_FG_BN) > INT32_MAX / GGML_SYCL_FG_BN) { + return false; + } + const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec); + if (combo < 0) { + return false; + } + // the device schedule is built for GGML_SYCL_FG_BN-wide tiles, see fg_grouped_device_run() + bool bn_ok = false; + fg_visit_combo(combo, [&](auto s) { bn_ok = decltype(s)::BN == GGML_SYCL_FG_BN; }); + return bn_ok; +} + +bool ggml_sycl_grouped_dequant_gemm_device(ggml_type src0_type, bool reordered, const void * src0_base, + size_t expert_stride, const ggml_sycl_grouped_gemm_rows & src1, + int32_t src1_prec, const ggml_sycl_grouped_gemm_rows & dst, + const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, + int64_t n_ids, int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool, + dpct::queue_ptr stream) { + if (src1.ne1 <= 0 || dst.ne1 <= 0 || + !ggml_sycl_grouped_dequant_gemm_device_supported(src0_type, reordered, src1_prec, n_as, n_ids, + n_tokens, M, K, stream)) { + return false; + } + const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec); + bool launched = false; + fg_visit_combo(combo, [&](auto s) { + launched = fg_grouped_device_run<decltype(s)>(src0_type, reordered, src0_base, expert_stride, src1, dst, + ids_dev, ids_token_stride, n_as, n_ids, n_tokens, M, K, pool, stream); + }); + return launched; +} diff --git a/ggml/src/ggml-sycl/fused-gemm.hpp b/ggml/src/ggml-sycl/fused-gemm.hpp new file mode 100644 index 000000000..0e807a2c0 --- /dev/null +++ b/ggml/src/ggml-sycl/fused-gemm.hpp @@ -0,0 +1,122 @@ +#ifndef GGML_SYCL_FUSED_GEMM_HPP +#define GGML_SYCL_FUSED_GEMM_HPP + +#include "common.hpp" + + +// Routed row n is at base + (map[n].i1 % ne1)*nb1 + map[n].i2*nb2. +struct ggml_sycl_grouped_gemm_rows { + char * base; + const mmid_row_mapping * map; + int64_t ne1; + size_t nb1; + size_t nb2; +}; + +static constexpr int GGML_SYCL_FG_BN = 32; +static constexpr int64_t GGML_SYCL_MMID_SCHED_MAX_EXPERTS = 4096; + +// Shape and type gates for the kernels below. Device capability is separate: it needs a queue to ask. +static constexpr int GGML_SYCL_FG_MAX_N = 64; // widest N taken; each shape covers it in BN-wide tiles + +// weight formats the fused A stage decodes; K must cover whole stored blocks +constexpr bool ggml_sycl_fused_dequant_gemm_type_ok(ggml_type src0_type, int64_t K) { + // iq4_nl and q8_0 store 32 values per block; every other format here is a 256-value superblock + // that the A stage walks in steps of 32, so K must cover whole superblocks. + if (src0_type == GGML_TYPE_IQ4_NL || src0_type == GGML_TYPE_Q8_0) { + return K % 32 == 0; + } + const bool superblock = + src0_type == GGML_TYPE_Q2_K || + src0_type == GGML_TYPE_Q3_K || + src0_type == GGML_TYPE_Q4_K || + src0_type == GGML_TYPE_Q5_K || + src0_type == GGML_TYPE_Q6_K || + src0_type == GGML_TYPE_IQ3_S || + src0_type == GGML_TYPE_IQ4_XS || + src0_type == GGML_TYPE_IQ3_XXS || + src0_type == GGML_TYPE_IQ2_XXS || + src0_type == GGML_TYPE_IQ2_XS || + src0_type == GGML_TYPE_IQ2_S || + src0_type == GGML_TYPE_IQ1_S || + src0_type == GGML_TYPE_IQ1_M; + return superblock && QK_K == 256 && K % QK_K == 0; +} + +constexpr bool ggml_sycl_fused_dequant_gemm_shape_ok(ggml_type src0_type, int64_t M, int64_t N, int64_t K, + int64_t ldd) { + return ggml_sycl_fused_dequant_gemm_type_ok(src0_type, K) && M > 0 && N > 0 && K > 0 && + N <= GGML_SYCL_FG_MAX_N && + M <= INT32_MAX && N <= INT32_MAX && K <= INT32_MAX && ldd <= INT32_MAX; +} + +// grouped variant: the per-expert fused kernel is only worth it while each expert is narrow, +// so wider average slices are left to the per-expert library GEMM loop +constexpr bool ggml_sycl_grouped_dequant_gemm_shape_ok(ggml_type src0_type, int64_t M, int64_t K, + int64_t total_rows, int64_t n_active, bool unbounded = false) { + return ggml_sycl_fused_dequant_gemm_shape_ok(src0_type, M, 1, K, M) && total_rows > 0 && + total_rows <= INT32_MAX && (unbounded || total_rows <= n_active * GGML_SYCL_FG_MAX_N); +} + +// Runtime type gate, kept out of the constexpr predicates above so those stay pure. +inline bool ggml_sycl_xmx_gather_type_enabled(ggml_type src0_type) { + switch (src0_type) { + case GGML_TYPE_IQ4_NL: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ4_NL ) != 0; + case GGML_TYPE_IQ3_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ3_S ) != 0; + case GGML_TYPE_IQ4_XS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ4_XS ) != 0; + case GGML_TYPE_IQ3_XXS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ3_XXS ) != 0; + case GGML_TYPE_IQ2_XXS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_XXS ) != 0; + case GGML_TYPE_IQ2_XS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_XS ) != 0; + case GGML_TYPE_IQ2_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_S ) != 0; + case GGML_TYPE_IQ1_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_S ) != 0; + case GGML_TYPE_IQ1_M: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_M ) != 0; + case GGML_TYPE_Q8_0: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q8_0 ) != 0; + case GGML_TYPE_Q2_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q2_K ) != 0; + case GGML_TYPE_Q3_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q3_K ) != 0; + case GGML_TYPE_Q4_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q4_K ) != 0; + case GGML_TYPE_Q5_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q5_K ) != 0; + case GGML_TYPE_Q6_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q6_K ) != 0; + default: return false; + } +} + +// dst[n*ldd + m] = sum_k dequant(src0)[m*K + k] * src1[n*K + k], src1 is F32, F16 or BF16. +// The XMX combination is picked per call from the src1 type and its precision request src1_prec +// (op_params[3], [TAG_GGML_PREC]); the accumulator is f32, which meets any request. +// q8_0 and the k-quants are not handled here, only in the grouped path below. +// Returns false when the case is not handled (type, device, precision, or shape). +bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, bool src0_reordered, const void * src0, const void * src1, ggml_type src1_type, + int32_t src1_prec, float * dst, int64_t M, int64_t N, int64_t K, int64_t ldd, + ggml_sycl_pool & pool, dpct::queue_ptr stream); + +// One launch for every expert of a MUL_MAT_ID: rows of src1/dst are grouped by expert, expert e +// owns rows [expert_row_offsets[e], expert_row_offsets[e+1]) and reads its weights at +// src0_base + e*expert_stride. tiles is host scratch that must stay alive until the queue drains. +// reordered: each expert slice is in the reorder (SoA) layout of reorder_qw(). +// dst[n*M + m] = sum_k dequant(src0_e)[m*K + k] * src1[n*K + k] +// Returns false when the case is not handled (type, layout, device, precision, or shape). +bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, + const float * src1, int32_t src1_prec, float * dst, + const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K, + int64_t total_rows, std::vector<ggml_sycl_grouped_gemm_tile> & tiles, + ggml_sycl_pool & pool, dpct::queue_ptr stream); + +// Build routing and tile tables on the device; unused tile slots are empty. +bool ggml_sycl_build_mmid_schedule(const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids, + int64_t n_tokens, int64_t n_tiles_max, mmid_row_mapping * row_mapping, + ggml_sycl_grouped_gemm_tile * tiles, uint32_t * expert_offsets, + dpct::queue_ptr stream); + +bool ggml_sycl_grouped_dequant_gemm_device_supported(ggml_type src0_type, bool reordered, int32_t src1_prec, + int64_t n_as, int64_t n_ids, int64_t n_tokens, + int64_t M, int64_t K, dpct::queue_ptr stream); + +// Returns false before queuing work when the type, shape, layout or device is unsupported. +bool ggml_sycl_grouped_dequant_gemm_device(ggml_type src0_type, bool reordered, const void * src0_base, + size_t expert_stride, const ggml_sycl_grouped_gemm_rows & src1, + int32_t src1_prec, const ggml_sycl_grouped_gemm_rows & dst, + const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, + int64_t n_ids, int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool, + dpct::queue_ptr stream); + +#endif // GGML_SYCL_FUSED_GEMM_HPP diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp index d3e995233..9b553829b 100644 --- a/ggml/src/ggml-sycl/fusion.cpp +++ b/ggml/src/ggml-sycl/fusion.cpp @@ -4,7 +4,7 @@ #include <algorithm> // mul_mat(gate) + mul_mat(up) + GLU: graph shape and tensor properties only. Backend state -// (weight layout, split buffers, DMMV) is checked by ggml_sycl_mul_mat_glu_mmvq_fused(). +// (weight layout, split buffers, DMMV) is checked by ggml_sycl_op_mul_mat_glu_fused(). static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const ggml_tensor * up, const ggml_tensor * glu) { // the fused epilogue implements these two; the rest fall back to the standalone GLU kernels @@ -32,12 +32,29 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg return false; } - // fused GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type + // fused K-quant GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type // q4_K, the plain-layout kernel covers q5_K / iq4_xs pairs incl. mixed gate/up types const bool reorder_pair = wu->type == GGML_TYPE_Q4_K && wg->type == GGML_TYPE_Q4_K; const bool plain_pair = (wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_IQ4_XS) && (wg->type == GGML_TYPE_Q5_K || wg->type == GGML_TYPE_IQ4_XS); - if ((!reorder_pair && !plain_pair) || wu->ne[0] % QK_K != 0) { +#ifdef GGML_SYCL_MMVQ_HAS_XMX + // same-type pairs of the types of the XMX kernels, which also take a few more columns + // (the XMX kernels walk whole QK_K super-blocks, so they need rows of a multiple of QK_K) + const bool xmx_pair = wu->type == wg->type && ggml_sycl_xmx_supports_type(wu->type) && wu->ne[0] % QK_K == 0; +#else + const bool xmx_pair = false; +#endif // GGML_SYCL_MMVQ_HAS_XMX + // the ESIMD kernel covers same-type q2_K..q6_K and q8_0, single column only (as unfused DMMV) +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + const bool esimd_pair = g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 && + (wu->type == GGML_TYPE_Q2_K || wu->type == GGML_TYPE_Q3_K || wu->type == GGML_TYPE_Q4_K || + wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_Q6_K || wu->type == GGML_TYPE_Q8_0); +#else + const bool esimd_pair = false; +#endif + // q8_0 rows are walked in 32-wide blocks, so they only need to be a multiple of QK8_0 + const int64_t row_align = esimd_pair && wu->type == GGML_TYPE_Q8_0 ? QK8_0 : QK_K; + if ((!reorder_pair && !plain_pair && !xmx_pair && !esimd_pair) || wu->ne[0] % row_align != 0) { return false; } @@ -57,13 +74,20 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg return false; } // mat-vec only: one column per decoded token, up to the batch the reorder kernels cover - if (act->ne[1] > MMVQ_MAX_BATCH_SIZE) { + const int max_cols = xmx_pair ? std::max(MMVQ_MAX_BATCH_SIZE, g_ggml_sycl_xmx_glu_max_cols) : MMVQ_MAX_BATCH_SIZE; + if (act->ne[1] > max_cols) { return false; } return true; } +// the unary ops the fused unary chains in element_wise.cpp have a functor for +static bool ggml_sycl_fused_unary_has_kernel(ggml_unary_op unary_op) { + return unary_op == GGML_UNARY_OP_SILU || unary_op == GGML_UNARY_OP_SIGMOID || + unary_op == GGML_UNARY_OP_SOFTPLUS; +} + bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops, std::initializer_list<enum ggml_unary_op> unary_ops) { #ifndef NDEBUG @@ -184,9 +208,7 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ return false; } - // the ops ggml_sycl_op_unary_mul_fused() has a kernel for - if (unary_op != GGML_UNARY_OP_SILU && unary_op != GGML_UNARY_OP_SIGMOID && - unary_op != GGML_UNARY_OP_SOFTPLUS) { + if (!ggml_sycl_fused_unary_has_kernel(unary_op)) { return false; } @@ -233,6 +255,55 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ return true; } + // ADD(bias) + UNARY + MUL(scale): the delta-net alpha gate, softplus(alpha + dt) * a. + // The broadcast is what stops the same-shape UNARY + MUL branch above firing past one token. + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_UNARY && + ops.begin()[2] == GGML_OP_MUL && unary_ops.size() == 1) { + const ggml_tensor * add = cgraph->nodes[node_idx]; + const ggml_tensor * unary = cgraph->nodes[node_idx + 1]; + const ggml_tensor * mul = cgraph->nodes[node_idx + 2]; + + const ggml_unary_op unary_op = ggml_get_unary_op(unary); + if (unary_op != unary_ops.begin()[0]) { + return false; + } + + if (!ggml_sycl_fused_unary_has_kernel(unary_op)) { + return false; + } + + // ggml_can_fuse() has already pinned the chain: unary consumes add, mul consumes + // unary, add and unary have one use each, and all three have the same shape + const ggml_tensor * a = add->src[0]; + const ggml_tensor * bias = add->src[1]; + const ggml_tensor * scale = (mul->src[0] == unary) ? mul->src[1] : mul->src[0]; + + if (a->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { + return false; + } + + // the activation and the destination are indexed flat + if (!ggml_is_contiguous(a) || !ggml_is_contiguous(mul) || !ggml_are_same_shape(a, mul)) { + return false; + } + + // the kernel reads the bias and the scale as v[col], so each must be a single + // contiguous row spanning ne0 + if (bias->ne[0] != a->ne[0] || scale->ne[0] != a->ne[0] || + ggml_nrows(bias) != 1 || ggml_nrows(scale) != 1 || + !ggml_is_contiguous(bias) || !ggml_is_contiguous(scale)) { + return false; + } + + // the 32-bit fastdiv is inexact past 2^31; decline, the unfused path handles it + if (ggml_nelements(mul) >= ((int64_t) 1 << 31)) { + return false; + } + + return true; + } + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_ADD && ops.begin()[2] == GGML_OP_UNARY && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) { const ggml_tensor * ssm_conv = cgraph->nodes[node_idx]; diff --git a/ggml/src/ggml-sycl/fwht.cpp b/ggml/src/ggml-sycl/fwht.cpp index fb48d7fec..5af549c23 100644 --- a/ggml/src/ggml-sycl/fwht.cpp +++ b/ggml/src/ggml-sycl/fwht.cpp @@ -46,8 +46,8 @@ static constexpr float H20[20][20] = { #undef P #undef N -template <int N> -static void fwht_kernel(const float * __restrict__ src, float * __restrict__ dst, const int64_t n_rows, +template <int N, typename T> +static void fwht_kernel(const T * __restrict__ src, float * __restrict__ dst, const int64_t n_rows, const float scale, const sycl::nd_item<2> & item) { const sycl::sub_group sg = item.get_sub_group(); @@ -67,7 +67,7 @@ static void fwht_kernel(const float * __restrict__ src, float * __restrict__ dst #pragma unroll for (int i = 0; i < el_w; ++i) { - reg[i] = src[i * WARP_SIZE + lane] * scale; + reg[i] = static_cast<float>(src[i * WARP_SIZE + lane]) * scale; } // Butterflies inside the sub-group. The partner of a lane with bit h clear is the @@ -107,8 +107,8 @@ static void fwht_kernel(const float * __restrict__ src, float * __restrict__ dst } } -template <int N> -static void launch_fwht(const float * src, float * dst, const int64_t n_rows, const float scale, +template <int N, typename T> +static void launch_fwht(const T * src, float * dst, const int64_t n_rows, const float scale, dpct::queue_ptr stream) { constexpr int rows_per_block = 4; @@ -120,7 +120,7 @@ static void launch_fwht(const float * src, float * dst, const int64_t n_rows, co stream->parallel_for(sycl::nd_range<2>(global, local), [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - fwht_kernel<N>(src, dst, n_rows, scale, item); + fwht_kernel<N, T>(src, dst, n_rows, scale, item); }); } @@ -128,8 +128,8 @@ static void launch_fwht(const float * src, float * dst, const int64_t n_rows, co // keeps N/NT values rather than N/WARP_SIZE. Butterflies below the sub-group width // still shuffle; those up to NT go through work-group local memory; the rest stay // in registers. -template <int N, int NT> -static void fwht_kernel_wide(const float * __restrict__ src, +template <int N, int NT, typename T> +static void fwht_kernel_wide(const T * __restrict__ src, float * __restrict__ dst, const int64_t n_rows, const float scale, @@ -151,7 +151,7 @@ static void fwht_kernel_wide(const float * __restrict__ src, float reg[el_w]; #pragma unroll for (int i = 0; i < el_w; ++i) { - reg[i] = src[i * NT + tid] * scale; + reg[i] = static_cast<float>(src[i * NT + tid]) * scale; } const sycl::sub_group sg = item.get_sub_group(); @@ -207,8 +207,8 @@ static void fwht_kernel_wide(const float * __restrict__ src, } } -template <int N, int NT> -static void launch_fwht_wide(const float * src, +template <int N, int NT, typename T> +static void launch_fwht_wide(const T * src, float * dst, const int64_t n_rows, const float scale, @@ -220,13 +220,13 @@ static void launch_fwht_wide(const float * src, sycl::local_accessor<float, 1> smem(sycl::range<1>(N), cgh); cgh.parallel_for(sycl::nd_range<2>(global, local), [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - fwht_kernel_wide<N, NT>(src, dst, n_rows, scale, item, get_pointer(smem)); + fwht_kernel_wide<N, NT, T>(src, dst, n_rows, scale, item, get_pointer(smem)); }); }); } -template <int N, int m> -static void kronecker_kernel(const float * __restrict__ src, +template <int N, int m, typename T> +static void kronecker_kernel(const T * __restrict__ src, float * __restrict__ dst, const int64_t n_rows, const float scale, @@ -255,7 +255,7 @@ static void kronecker_kernel(const float * __restrict__ src, #pragma unroll for (int j = 0; j < m; ++j) { - reg[i * m + j] = src[b_idx * m + j] * scale; + reg[i * m + j] = static_cast<float>(src[b_idx * m + j]) * scale; } } @@ -321,8 +321,8 @@ static void kronecker_kernel(const float * __restrict__ src, } } -template <int N, int m> -static void launch_kronecker(const float * src, +template <int N, int m, typename T> +static void launch_kronecker(const T * src, float * dst, const int64_t n_rows, const float scale, @@ -337,25 +337,16 @@ static void launch_kronecker(const float * src, stream->parallel_for(sycl::nd_range<2>(global, local), [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - kronecker_kernel<N, m>(src, dst, n_rows, scale, item); + kronecker_kernel<N, m, T>(src, dst, n_rows, scale, item); }); } -bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { - if (src->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; - } - if (!ggml_are_same_shape(src, dst)) { - return false; - } - if (!ggml_is_contiguous(src) || !ggml_is_contiguous(dst)) { - return false; - } - +template <typename T> +static bool ggml_sycl_op_fwht_impl(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { const int n = (int) src->ne[0]; const int64_t rows = ggml_nrows(src); - const float * src_d = (const float *) src->data; + const T * src_d = (const T *) src->data; float * dst_d = (float *) dst->data; dpct::queue_ptr stream = ctx.stream(); @@ -402,3 +393,24 @@ bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, return false; } } + +bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { + if (dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_are_same_shape(src, dst)) { + return false; + } + if (!ggml_is_contiguous(src) || !ggml_is_contiguous(dst)) { + return false; + } + + switch (src->type) { + case GGML_TYPE_F32: + return ggml_sycl_op_fwht_impl<float>(ctx, src, dst); + case GGML_TYPE_F16: + return ggml_sycl_op_fwht_impl<sycl::half>(ctx, src, dst); + default: + return false; + } +} diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp index 4e84bd222..7b53e3dcc 100644 --- a/ggml/src/ggml-sycl/getrows.cpp +++ b/ggml/src/ggml-sycl/getrows.cpp @@ -137,6 +137,47 @@ static void k_get_rows_float( dst_row[i00] = src0_row[i00]; } +// src0 is in reorder layout: all qs of the tensor, then all d +template<int qk, int qr, dequantize_kernel_t_reorder dequantize_kernel, typename dst_t> +static void k_get_rows_reorder( + const void * src0_qs, const void * src0_d, const int32_t * src1, dst_t * dst, + int64_t ne00, int64_t ne12, + size_t s1, size_t s2, size_t s3, + size_t s10, size_t s11, size_t s12, + const sycl::nd_item<3> &item_ct1) { + + const int i00 = (item_ct1.get_group(2) * item_ct1.get_local_range(2) + + item_ct1.get_local_id(2)) * + 2; + const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) + + item_ct1.get_local_id(1); + const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) / + ne12; + const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) % + ne12; + + if (i00 >= ne00) { + return; + } + + const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; + + dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; + + const int64_t ib = (int64_t) i01*(ne00/qk) + i00/qk; // block index in the whole tensor + const int iqs = (i00%qk)/qr; + const int iybs = i00 - i00%qk; + const int y_offset = qr == 1 ? 1 : qk/2; + + dfloat2 v; + dequantize_kernel(src0_d, ib, src0_qs, ib*(qk/qr) + iqs, v); + + dst_row[iybs + iqs + 0] = v.x(); + dst_row[iybs + iqs + y_offset] = v.y(); +} + template <int qk, int qr, dequantize_kernel_t dq> static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const void *src0_dd, @@ -173,6 +214,139 @@ static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *sr GGML_UNUSED(ctx); } +template <int qk, int qr, dequantize_kernel_t_reorder dq> +static void get_rows_sycl_reorder(const ggml_tensor *src0, const ggml_tensor *src1, + ggml_tensor *dst, const void *src0_dd, + const int32_t *src1_dd, float *dst_dd, + queue_ptr stream) { + + GGML_TENSOR_BINARY_OP_LOCALS + + // reorder is done only on 2D weights + GGML_ASSERT(ne02 == 1 && ne03 == 1); + GGML_ASSERT(ne00 % 2 == 0); + + const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); + const int block_num_x = (ne00 + 2*SYCL_GET_ROWS_BLOCK_SIZE - 1) / (2*SYCL_GET_ROWS_BLOCK_SIZE); + const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x); + + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + + const void * src0_d = (const char *) src0_dd + ne00*ne01/qr; + + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_get_rows_reorder<qk, qr, dq>( + src0_dd, src0_d, src1_dd, dst_dd, ne00, ne12, s1, s2, + s3, s10, s11, s12, item_ct1); + }); + + GGML_UNUSED(src0); +} + +// K-quants in reorder layout: one work-group dequantizes one QK_K block of one row +template <int dims, int nthreads, int nlocal, typename F> +static void get_rows_sycl_reorder_k(const ggml_tensor *src0, const ggml_tensor *src1, + ggml_tensor *dst, const int32_t *src1_dd, float *dst_dd, + queue_ptr stream, F dequantize_block) { + + GGML_TENSOR_BINARY_OP_LOCALS + + // reorder is done only on 2D weights + GGML_ASSERT(ne02 == 1 && ne03 == 1); + GGML_ASSERT(ne00 % QK_K == 0); + + const int64_t nblk = ne00 / QK_K; + const int64_t nb = nblk * ne01; // blocks in the whole tensor + const int64_t ngroups = ne10 * ne11 * ne12 * nblk; + + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor<uint8_t, 1> scales_local(sycl::range<1>(nlocal), cgh); + + auto kernel = [=](sycl::nd_item<dims> item_ct1) { + const int64_t g = item_ct1.get_group(dims - 1); + const int64_t b = g % nblk; + const int64_t r = g / nblk; + const int64_t i10 = r % ne10; + const int64_t i11 = (r / ne10) % ne11; + const int64_t i12 = r / (ne10 * ne11); + + const int64_t i01 = src1_dd[i10*s10 + i11*s11 + i12*s12]; + + float * y = dst_dd + i10*s1 + i11*s2 + i12*s3 + b*QK_K; + dequantize_block(y, i01*nblk + b, get_pointer(scales_local), item_ct1, nb); + }; + + if constexpr (dims == 1) { + cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(ngroups * nthreads), sycl::range<1>(nthreads)), kernel); + } else { + cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, ngroups * nthreads), sycl::range<3>(1, 1, nthreads)), kernel); + } + }); + + GGML_UNUSED(src0); +} + +// IQ3_S / IQ3_XXS in the persistent reorder layout: one work-group of 32 threads +// dequantizes one QK_K block of one requested row, reusing the reorder-aware +// block dequantizers that the dequantize-based mul_mat path already uses. +template <typename F> +static void get_rows_sycl_reorder_iq3(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst, + const int32_t * src1_dd, float * dst_dd, + F dequantize_block) { + GGML_TENSOR_BINARY_OP_LOCALS + + // the persistent reorder is only installed for 2D weights + GGML_ASSERT(ne02 == 1 && ne03 == 1); + GGML_ASSERT(ne00 % QK_K == 0); + + const int64_t nblk = ne00 / QK_K; + const int64_t nb = nblk * ne01; // blocks in the whole tensor + const int64_t ngroups = ne10 * ne11 * ne12 * nblk; + + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + + queue_ptr stream = ctx.stream(); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, ngroups * 32), sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + const int64_t g = item_ct1.get_group(2); + const int64_t b = g % nblk; + const int64_t r = g / nblk; + const int64_t i10 = r % ne10; + const int64_t i11 = (r / ne10) % ne11; + const int64_t i12 = r / (ne10 * ne11); + + const int64_t i01 = src1_dd[i10*s10 + i11*s11 + i12*s12]; + + float * y = dst_dd + i10*s1 + i11*s2 + i12*s3 + b*QK_K; + dequantize_block(y, i01*nblk + b, item_ct1, nb); + }); + + GGML_UNUSED(dst); +} + template <int qk, int qr, dequantize_kernel_f32_t dq> static void get_rows_sycl_f32(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const void *src0_dd, @@ -332,6 +506,11 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type)); const int32_t * src1_i32 = (const int32_t *) dst->src[1]->data; + // a weight shared with MUL_MAT (e.g. tied embeddings) can be in the + // persistent reorder layout (incl. IQ3_S / IQ3_XXS since the SYCL IQ3 + // reorder work) + const bool src0_reordered = dst->src[0]->extra && + ((const ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder; /* TODO: Refactor and remove duplicates */ switch (dst->src[0]->type) { case GGML_TYPE_F16: @@ -375,6 +554,15 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_IQ3_XXS: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_iq3(ctx, dst->src[0], dst->src[1], dst, src1_i32, (float *) dst->data, + [=](float * y, int64_t iblk, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_iq3_xxs_reorder(vx, y, iblk, item_ct1, nb, + iq3xxs_grid, ksigns_iq2xs, kmask_iq2xs); + }); + break; + } get_rows_sycl<QK_K, 1, dequantize_iq3_xxs>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; @@ -387,6 +575,15 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_IQ3_S: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_iq3(ctx, dst->src[0], dst->src[1], dst, src1_i32, (float *) dst->data, + [=](float * y, int64_t iblk, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_iq3_s_reorder(vx, y, iblk, item_ct1, nb, + kmask_iq2xs, iq3s_grid); + }); + break; + } get_rows_sycl<QK_K, 1, dequantize_iq3_s>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; @@ -399,14 +596,37 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_Q2_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_q2_K_reorder(vx, y, ib, item_ct1, nb); + GGML_UNUSED(scales_local); + }); + break; + } get_rows_sycl_f32<QK_K, 1, dequantize_q2_K_f32>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_Q3_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_q3_K_reorder(vx, y, ib, item_ct1, nb); + GGML_UNUSED(scales_local); + }); + break; + } get_rows_sycl<QK_K, 1, dequantize_q3_K>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_Q4_0: + if (src0_reordered) { + get_rows_sycl_reorder<QK4_0, QR4_0, dequantize_q4_0_reorder>(dst->src[0], dst->src[1], dst, dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + } get_rows_sycl<QK4_0, QR4_0, dequantize_q4_0>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; @@ -415,6 +635,14 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_Q4_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<1, 32, K_SCALE_SIZE>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<1> & item_ct1, int64_t nb) { + dequantize_block_q4_K_reorder(vx, y, ib, scales_local, item_ct1, nb); + }); + break; + } get_rows_sycl_f32<QK_K, 1, dequantize_q4_K_f32>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; @@ -427,14 +655,36 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_Q5_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<3, 32, K_SCALE_SIZE>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_q5_K_reorder(vx, y, ib, scales_local, item_ct1, nb); + }); + break; + } get_rows_sycl_f32<QK_K, 1, dequantize_q5_K_f32>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_Q6_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_q6_K_reorder(vx, y, ib, item_ct1, nb); + GGML_UNUSED(scales_local); + }); + break; + } get_rows_sycl<QK_K, 1, dequantize_q6_K>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_Q8_0: + if (src0_reordered) { + get_rows_sycl_reorder<QK8_0, QR8_0, dequantize_q8_0_reorder>(dst->src[0], dst->src[1], dst, dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + } get_rows_sycl<QK8_0, QR8_0, dequantize_q8_0>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; @@ -468,6 +718,5 @@ void ggml_sycl_op_get_rows_back(ggml_backend_sycl_context & ctx, ggml_tensor * d break; default: GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0->type)); - break; } } diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 49148d7c5..fdd526425 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -14,6 +14,7 @@ #include <array> #include <assert.h> #include <atomic> +#include <cctype> #include <cinttypes> #include <cstddef> #include <cstdint> @@ -25,6 +26,7 @@ #include <stdio.h> #include <vector> #include <cmath> +#include <cstring> #include <iostream> #include <fstream> #include <stdio.h> @@ -45,8 +47,12 @@ # include <sycl/ext/oneapi/virtual_mem/virtual_mem.hpp> # define GGML_SYCL_SUPPORT_VMM #endif -#if defined(__INTEL_LLVM_COMPILER) - #define GGML_SYCL_DMMV_HAS_ESIMD +// beginning in oneDNN 3.12.3, "Enabled matmul and inner product primitives compatibility with SYCL Graph native recording mode on Intel GPUs." +#if GGML_SYCL_DNNL +# include "dnnl_version.h" +# if (DNNL_VERSION_MAJOR > 3 || (DNNL_VERSION_MAJOR == 3 && (DNNL_VERSION_MINOR > 12 || (DNNL_VERSION_MINOR == 12 && DNNL_VERSION_PATCH >= 3)))) +# define GGML_SYCL_GRAPH_DNNL_SUPPORTED +# endif #endif #include <sycl/half_type.hpp> @@ -60,6 +66,7 @@ #include "ggml-sycl/common.hpp" #include "ggml-sycl/element_wise.hpp" #include "ggml-sycl/fwht.hpp" +#include "ggml-sycl/fused-gemm.hpp" #include "ggml-sycl/gemm.hpp" #include "ggml-sycl/getrows.hpp" #include "ggml-sycl/mem.hpp" @@ -98,25 +105,57 @@ int g_ggml_sycl_enable_dnn = 1; int g_ggml_sycl_fa_onednn = 1; int g_ggml_sycl_fa_onednn_max_kv = 0; int g_ggml_sycl_enable_mkl_fa = 1; +int g_ggml_sycl_fa_dpas = 1; int g_ggml_sycl_memtrace = 0; int g_ggml_sycl_memtrace_step = 64; int g_ggml_sycl_enable_vmm = 1; int g_ggml_sycl_enable_fusion = 1; int g_ggml_sycl_enable_esimd = 1; int g_ggml_sycl_mmvq_wide = 1; +int g_ggml_sycl_enable_xmx_esimd = 1; +int g_ggml_sycl_xmx_max_cols = GGML_SYCL_XMX_MAX_COLS_DEFAULT; +int g_ggml_sycl_xmx_glu_max_cols = GGML_SYCL_XMX_GLU_MAX_COLS; int g_ggml_sycl_prioritize_dmmv = 0; +int g_ggml_sycl_xmx_gather_types = GGML_SYCL_XMX_GATHER_TYPES_DEFAULT; +int g_ggml_sycl_xmx_gather_shapes = GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT; +int g_ggml_sycl_mmid_sched = GGML_SYCL_MMID_SCHED_DEVICE; +int g_ggml_sycl_dynamic_precision = GGML_SYCL_DYNAMIC_PRECISION_DEFAULT; +int g_ggml_sycl_dynamic_required_precision = GGML_SYCL_DYNAMIC_PRECISION_F32; +static const char * ggml_sycl_dynamic_precision_names[] = { "F16", "BF16", "TF32", "F32" }; + +// value of a GGML_SYCL_DYNAMIC_PRECISION-style variable; def if unset or invalid +static int ggml_sycl_get_env_precision(const char * name, int def) { + const char * env = getenv(name); + if (!env) { + return def; + } + std::string mode(env); + for (char & c : mode) { + c = (char) std::toupper((unsigned char) c); + } + for (int i = GGML_SYCL_DYNAMIC_PRECISION_F16; i <= GGML_SYCL_DYNAMIC_PRECISION_F32; i++) { + if (mode == ggml_sycl_dynamic_precision_names[i]) { + return i; + } + } + GGML_LOG_WARN("%s: unknown %s=%s, using %s\n", __func__, name, env, ggml_sycl_dynamic_precision_names[def]); + return def; +} int g_ggml_sycl_use_async_mem_op = 0; int g_ggml_sycl_use_async_mem_op_requested = 1; int g_ggml_sycl_use_level_zero_api = 0; int g_ggml_sycl_enable_flash_attention = 1; int g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; int g_ggml_sycl_usm_system = 0; +int g_ggml_sycl_bmg_vram_check = 1; int g_ggml_sycl_enable_host_pinned_mem = 1; int g_ggml_sycl_host_pinned_mem_2g = 0; +int g_ggml_sycl_upload_staging_slots = 4; int g_ggml_sycl_get_mem_api = MEMORY_API_TYPE_LEVEL_ZERO; int g_ggml_sycl_enable_sparse_fa = 0; int g_ggml_sycl_debug_sparse_fa = 0; int g_ggml_sycl_sparse_fa_margin = 256; +int g_ggml_sycl_graph_eviction_timeout = 10; #if GGML_SYCL_DNNL // ask oneDNN which matmul it picks for a small f16 problem @@ -174,10 +213,10 @@ static ggml_sycl_device_info ggml_sycl_init() { int64_t total_vram = 0; /* This is a bit misleading; reserved for later */ -// #if defined(SYCL_USE_XMX) -// GGML_LOG_INFO("%s: SYCL_USE_XMX: yes\n", __func__); +// #if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) +// GGML_LOG_INFO("%s: GGML_SYCL_SUPPORT_XMX: yes\n", __func__); // #else -// GGML_LOG_INFO("%s: SYCL_USE_XMX: no\n", __func__); +// GGML_LOG_INFO("%s: GGML_SYCL_SUPPORT_XMX: no\n", __func__); // #endif for (int i = 0; i < info.device_count; ++i) { dpct::device_info prop; @@ -218,6 +257,11 @@ static ggml_sycl_device_info ggml_sycl_init() { info.devices[i].warp_size = WARP_SIZE; info.devices[i].usm_system_support = device.has(sycl::aspect::usm_system_allocations); +#ifdef GGML_SYCL_GRAPH + info.devices[i].graph_support = device.has(sycl::aspect::ext_oneapi_limited_graph); + info.devices[i].graph_update_support = device.has(sycl::aspect::ext_oneapi_graph); +#endif + info.max_work_group_sizes[i] = prop.get_max_work_group_size(); info.devices[i].max_wg_per_cu = info.max_work_group_sizes[i] / prop.get_max_compute_units(); info.devices[i].hw_info = get_device_hw_info(&device); @@ -256,7 +300,7 @@ static ggml_sycl_device_info ggml_sycl_init() { return info; } -const ggml_sycl_device_info & ggml_sycl_info() { +ggml_sycl_device_info & ggml_sycl_info() { static ggml_sycl_device_info info = ggml_sycl_init(); return info; } @@ -364,14 +408,14 @@ static ze_result_t init_zes() { return res; } -ze_result_t get_zes_init_res() { +static ze_result_t get_zes_init_res() { static ze_result_t zes_init_res = init_zes(); GGML_SYCL_DEBUG("[SYCL] call %s: zesInit result: %d\n", __func__, (int) zes_init_res); return zes_init_res; } #endif -void initialize_sycl_begining() { +static void initialize_sycl_begining() { #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API //must be called in initialization stage, before any other Level Zero API calls GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); @@ -394,13 +438,26 @@ static void ggml_check_sycl() try { g_ggml_sycl_fa_onednn = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN", 1); g_ggml_sycl_fa_onednn_max_kv = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN_MAX_KV", 0); g_ggml_sycl_enable_mkl_fa = ggml_sycl_get_env("GGML_SYCL_ENABLE_MKL_FA", 1); + g_ggml_sycl_fa_dpas = ggml_sycl_get_env("GGML_SYCL_FA_DPAS", 1); g_ggml_sycl_memtrace = ggml_sycl_get_env("GGML_SYCL_MEMTRACE", 0); g_ggml_sycl_memtrace_step = ggml_sycl_get_env("GGML_SYCL_MEMTRACE_STEP", 64); g_ggml_sycl_enable_vmm = ggml_sycl_get_env("GGML_SYCL_ENABLE_VMM", 1); g_ggml_sycl_enable_fusion = ggml_sycl_get_env("GGML_SYCL_ENABLE_FUSION", 1); g_ggml_sycl_enable_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_ESIMD", 1); g_ggml_sycl_mmvq_wide = ggml_sycl_get_env("GGML_SYCL_MMVQ_WIDE", 1); + g_ggml_sycl_enable_xmx_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_XMX_ESIMD", 1); + g_ggml_sycl_xmx_max_cols = std::max(0, ggml_sycl_get_env("GGML_SYCL_XMX_MAX_COLS", GGML_SYCL_XMX_MAX_COLS_DEFAULT)); + g_ggml_sycl_xmx_glu_max_cols = + std::clamp(ggml_sycl_get_env("GGML_SYCL_XMX_GLU_MAX_COLS", GGML_SYCL_XMX_GLU_MAX_COLS), 0, GGML_SYCL_XMX_GLU_MAX_COLS); g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); + g_ggml_sycl_graph_eviction_timeout = ggml_sycl_get_env("GGML_SYCL_GRAPH_EVICTION_TIMEOUT", 10); + g_ggml_sycl_xmx_gather_types = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_TYPES", GGML_SYCL_XMX_GATHER_TYPES_DEFAULT); + g_ggml_sycl_mmid_sched = ggml_sycl_get_env("GGML_SYCL_MMID_SCHED", GGML_SYCL_MMID_SCHED_DEVICE) & 3; + g_ggml_sycl_xmx_gather_shapes = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_SHAPES", GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT); + g_ggml_sycl_dynamic_precision = + ggml_sycl_get_env_precision("GGML_SYCL_DYNAMIC_PRECISION", GGML_SYCL_DYNAMIC_PRECISION_DEFAULT); + g_ggml_sycl_dynamic_required_precision = + ggml_sycl_get_env_precision("GGML_SYCL_DYNAMIC_REQUIRED_PRECISION", GGML_SYCL_DYNAMIC_PRECISION_F32); #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API g_ggml_sycl_use_level_zero_api = ggml_sycl_get_env("GGML_SYCL_USE_LEVEL_ZERO_API", 1); @@ -421,11 +478,13 @@ static void ggml_check_sycl() try { #endif g_ggml_sycl_usm_system = ggml_sycl_get_env("GGML_SYCL_USM_SYSTEM", 0); + g_ggml_sycl_bmg_vram_check = ggml_sycl_get_env("GGML_SYCL_BMG_VRAM_CHECK", 1); g_ggml_sycl_enable_host_pinned_mem = ggml_sycl_get_env("GGML_SYCL_ENABLE_HOST_PINNED_MEM", 1); g_ggml_sycl_host_pinned_mem_2g = ggml_sycl_get_env("GGML_SYCL_HOST_PINNED_MEM_2G", 0) & g_ggml_sycl_enable_host_pinned_mem; + g_ggml_sycl_upload_staging_slots = std::max(0, ggml_sycl_get_env("GGML_SYCL_UPLOAD_STAGING_SLOTS", 4)); g_ggml_sycl_enable_sparse_fa = ggml_sycl_get_env("GGML_SYCL_SPARSE_FA", 0); g_ggml_sycl_debug_sparse_fa = ggml_sycl_get_env("GGML_SYCL_SPARSE_FA_DEBUG", 0); @@ -452,6 +511,12 @@ static void ggml_check_sycl() try { GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: no\n"); #endif +#if defined(GGML_SYCL_SUPPORT_XMX) + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_XMX: yes\n"); +#else + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_XMX: no\n"); +#endif + #if defined(GGML_SYCL_GRAPH) GGML_LOG_INFO(" GGML_SYCL_GRAPH: yes\n"); #else @@ -493,6 +558,7 @@ static void ggml_check_sycl() try { #endif GGML_LOG_INFO(" GGML_SYCL_FA_ONEDNN_MAX_KV: %d\n", g_ggml_sycl_fa_onednn_max_kv); GGML_LOG_INFO(" GGML_SYCL_ENABLE_MKL_FA: %d\n", g_ggml_sycl_enable_mkl_fa); + GGML_LOG_INFO(" GGML_SYCL_FA_DPAS: %d\n", g_ggml_sycl_fa_dpas); GGML_LOG_INFO(" GGML_SYCL_MEMTRACE: %d\n", g_ggml_sycl_memtrace); GGML_LOG_INFO(" GGML_SYCL_MEMTRACE_STEP: %d\n", g_ggml_sycl_memtrace_step); #ifdef SYCL_FLASH_ATTN @@ -504,11 +570,19 @@ static void ggml_check_sycl() try { #ifdef GGML_SYCL_GRAPH GGML_LOG_INFO(" GGML_SYCL_ENABLE_GRAPH: %d\n", g_ggml_sycl_enable_graph); + GGML_LOG_INFO(" GGML_SYCL_GRAPH_EVICTION_TIMEOUT: %d\n", g_ggml_sycl_graph_eviction_timeout); #else GGML_LOG_INFO(" GGML_SYCL_ENABLE_GRAPH: graph disabled by compile flag\n"); #endif GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize); + GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_TYPES: %d\n", g_ggml_sycl_xmx_gather_types); + GGML_LOG_INFO(" GGML_SYCL_MMID_SCHED: %d\n", g_ggml_sycl_mmid_sched); + GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_SHAPES: %d\n", g_ggml_sycl_xmx_gather_shapes); + GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_PRECISION: %s\n", + ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_precision]); + GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_REQUIRED_PRECISION: %s\n", + ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_required_precision]); #if defined(GGML_SYCL_SUPPORT_VMM) GGML_LOG_INFO(" GGML_SYCL_ENABLE_VMM: %d\n", g_ggml_sycl_enable_vmm); @@ -517,13 +591,20 @@ static void ggml_check_sycl() try { #endif GGML_LOG_INFO(" GGML_SYCL_ENABLE_FUSION: %d\n", g_ggml_sycl_enable_fusion); + GGML_LOG_INFO(" GGML_SYCL_UPLOAD_STAGING_SLOTS: %d\n", g_ggml_sycl_upload_staging_slots); #if defined(__INTEL_LLVM_COMPILER) GGML_LOG_INFO(" GGML_SYCL_ENABLE_ESIMD: %d\n", g_ggml_sycl_enable_esimd); #else GGML_LOG_INFO(" GGML_SYCL_ENABLE_ESIMD: %d disabled by compile flag\n", g_ggml_sycl_enable_esimd); #endif + GGML_LOG_INFO(" GGML_SYCL_MMVQ_WIDE: %d\n", g_ggml_sycl_mmvq_wide); + + GGML_LOG_INFO(" GGML_SYCL_ENABLE_XMX_ESIMD: %d\n", g_ggml_sycl_enable_xmx_esimd); + GGML_LOG_INFO(" GGML_SYCL_XMX_MAX_COLS: %d\n", g_ggml_sycl_xmx_max_cols); + GGML_LOG_INFO(" GGML_SYCL_XMX_GLU_MAX_COLS: %d\n", g_ggml_sycl_xmx_glu_max_cols); + GGML_LOG_INFO(" GGML_SYCL_PRIORITIZE_DMMV: %d\n", g_ggml_sycl_prioritize_dmmv); g_ggml_sycl_use_async_mem_op_requested = ggml_sycl_get_env("GGML_SYCL_USE_ASYNC_MEM_OP", 1); @@ -536,6 +617,7 @@ static void ggml_check_sycl() try { #endif GGML_LOG_INFO(" GGML_SYCL_USM_SYSTEM: %d\n", g_ggml_sycl_usm_system); + GGML_LOG_INFO(" GGML_SYCL_BMG_VRAM_CHECK: %d\n", g_ggml_sycl_bmg_vram_check); GGML_LOG_INFO(" GGML_SYCL_ENABLE_HOST_PINNED_MEM: %d\n", g_ggml_sycl_enable_host_pinned_mem); GGML_LOG_INFO(" GGML_SYCL_HOST_PINNED_MEM_2G: %d\n", g_ggml_sycl_host_pinned_mem_2g); @@ -544,16 +626,25 @@ static void ggml_check_sycl() try { GGML_LOG_INFO(" GGML_SYCL_SPARSE_FA_MARGIN: %d\n", g_ggml_sycl_sparse_fa_margin); /* NOT REMOVE, keep it for next optimize for XMX. -#if defined(SYCL_USE_XMX) - fprintf(stderr, "%s: SYCL_USE_XMX: yes\n", __func__); +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) + fprintf(stderr, "%s: GGML_SYCL_SUPPORT_XMX: yes\n", __func__); #else - fprintf(stderr, "%s: SYCL_USE_XMX: no\n", __func__); + fprintf(stderr, "%s: GGML_SYCL_SUPPORT_XMX: no\n", __func__); #endif */ +#ifdef GGML_SYCL_GRAPH + if (g_ggml_sycl_enable_graph) { + for (unsigned int i = 0; i < dpct::dev_mgr::instance().device_count(); ++i) { + if (!dpct::dev_mgr::instance().get_device(i).has(sycl::aspect::ext_oneapi_limited_graph)) { + GGML_LOG_WARN("SYCL device %u does not support SYCL graphs\n", i); + } + } + } + // Async USM allocation/free is also useful outside the graph path: it avoids the host waits in the reorder // staging path while preserving queue ordering semantics. Graph support still depends on the extension being // available, but it no longer needs to control the non-graph fast path. -#if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC +#if SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC g_ggml_sycl_use_async_mem_op = g_ggml_sycl_use_async_mem_op_requested || g_ggml_sycl_enable_graph; if (g_ggml_sycl_use_async_mem_op) { for (unsigned int i = 0; i < dpct::dev_mgr::instance().device_count(); ++i) { @@ -563,6 +654,7 @@ static void ggml_check_sycl() try { } } } +#endif #endif if (CHECK_TRY_ERROR(g_all_sycl_device_count = dpct::dev_mgr::instance().device_count()) != 0) { @@ -629,12 +721,23 @@ inline void free_aligned_mem_host(void * memblock) { // sycl buffer struct ggml_backend_sycl_buffer_context { + // pinned staging for uploads; the host fills one slot while the previous one transfers + static constexpr size_t staging_slot_size = 8*1024*1024; + + struct host_staging { + void * data = nullptr; + std::vector<sycl::event> events; + std::vector<bool> submitted; + int next = 0; + }; + int device; void * dev_ptr = nullptr; queue_ptr stream; std::string name; optimize_feature opt_feature; std::vector<ggml_tensor_extra_gpu *> tensor_extras; + host_staging staging; bool is_usm_system; ggml_backend_sycl_buffer_context(int device, void * dev_ptr, queue_ptr stream, bool is_usm_system) : @@ -644,7 +747,22 @@ struct ggml_backend_sycl_buffer_context { opt_feature = ggml_sycl_info().devices[device].opt_feature; } + // waits for every queued upload, then releases the pinned block + void drop_host_staging() { + for (size_t i = 0; i < staging.submitted.size(); ++i) { + if (staging.submitted[i]) { + staging.events[i].wait_and_throw(); + staging.submitted[i] = false; + } + } + if (staging.data != nullptr) { + sycl::free(staging.data, *stream); + staging.data = nullptr; + } + } + ~ggml_backend_sycl_buffer_context() { + drop_host_staging(); if (dev_ptr != nullptr) { ggml_sycl_set_device(device); if (is_usm_system) @@ -697,7 +815,9 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, return GGML_STATUS_SUCCESS; } - if (g_ggml_sycl_enable_optimize) { + // a compute buffer copy of a host weight (e.g. -ncmoe experts offloaded for prompt processing) is + // refilled in the normal layout before every run, so reordering it is wasted work + if (g_ggml_sycl_enable_optimize && ggml_backend_buffer_get_usage(buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) { // set reorder extra buffer based on supported type switch (tensor->type) { case GGML_TYPE_Q4_0: @@ -706,7 +826,10 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K:{ + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_MXFP4:{ ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; tensor->extra = extra; ctx->tensor_extras.push_back(extra); @@ -736,6 +859,19 @@ catch (sycl::exception const &exc) { std::exit(1); } +// allocates the pinned upload staging on first use; returns false if staging is disabled or the allocation failed +static bool ggml_sycl_buffer_init_staging(ggml_backend_sycl_buffer_context * ctx) { + const int n_slots = g_ggml_sycl_upload_staging_slots; + if (n_slots > 0 && ctx->staging.data == nullptr) { + ctx->staging.data = sycl::malloc_host(n_slots * ctx->staging_slot_size, *ctx->stream); + if (ctx->staging.data != nullptr) { + ctx->staging.events.resize(n_slots); + ctx->staging.submitted.assign(n_slots, false); + } + } + return ctx->staging.data != nullptr; +} + static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor *tensor, const void *data, size_t offset, @@ -745,6 +881,32 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_SYCL_DEBUG(" size=%zu offset=%zu\n", size, offset); ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; ggml_sycl_set_device(ctx->device); + + // copy through pinned memory so the device never reads mmap()ed pages directly + // chunks pipeline on the in-order compute queue, so no drain per tensor is needed + if (ggml_sycl_buffer_init_staging(ctx)) { + queue_ptr stream = ctx->stream; + char * dst = (char *) tensor->data + offset; + const char * src = (const char *) data; + size_t remaining = size; + while (remaining > 0) { + const size_t chunk = std::min(remaining, ctx->staging_slot_size); + const int slot = ctx->staging.next; + ctx->staging.next = (ctx->staging.next + 1) % (int) ctx->staging.submitted.size(); + if (ctx->staging.submitted[slot]) { + ctx->staging.events[slot].wait_and_throw(); + } + void * stage = (char *) ctx->staging.data + slot * ctx->staging_slot_size; + memcpy(stage, src, chunk); + ctx->staging.events[slot] = stream->memcpy(dst, stage, chunk); + ctx->staging.submitted[slot] = true; + src += chunk; + dst += chunk; + remaining -= chunk; + } + return; + } + auto stream = &(dpct::dev_mgr::instance().get_device(ctx->device).default_queue()); SYCL_CHECK(CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw())); #ifndef _WIN32 @@ -764,6 +926,49 @@ catch (sycl::exception const &exc) { std::exit(1); } +// Copy n_copies rows in one call. -sm tensor uploads split weights this way; the generic fallback does one set_tensor per row. +static void ggml_backend_sycl_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, + const void * data, size_t offset, size_t size, + size_t n_copies, size_t stride_tensor, + size_t stride_data) try { + ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *) buffer->context; + const char * src = (const char *) data; + ggml_sycl_set_device(ctx->device); + + if (stride_tensor != size || size > ctx->staging_slot_size || !ggml_sycl_buffer_init_staging(ctx)) { + // dst rows not contiguous, row larger than a staging slot, or no staging: same as the generic fallback + for (size_t i = 0; i < n_copies; ++i) { + ggml_backend_sycl_buffer_set_tensor(buffer, tensor, src + i * stride_data, offset + i * stride_tensor, size); + } + return; + } + + // dst rows are contiguous: pack as many rows as fit into a staging slot and copy each slot with one memcpy + // a slot is only waited on when it is reused, so packing the next slot overlaps with the previous copies + queue_ptr stream = ctx->stream; + char * dst = (char *) tensor->data + offset; + const size_t rows_per_slot = ctx->staging_slot_size / size; + for (size_t i0 = 0; i0 < n_copies; i0 += rows_per_slot) { + const size_t n = std::min(rows_per_slot, n_copies - i0); + const int slot = ctx->staging.next; + ctx->staging.next = (ctx->staging.next + 1) % (int) ctx->staging.submitted.size(); + if (ctx->staging.submitted[slot]) { + ctx->staging.events[slot].wait_and_throw(); + } + char * stage = (char *) ctx->staging.data + slot * ctx->staging_slot_size; + for (size_t i = 0; i < n; ++i) { + memcpy(stage + i * size, src + (i0 + i) * stride_data, size); + } + ctx->staging.events[slot] = stream->memcpy(dst + i0 * size, stage, n * size); + ctx->staging.submitted[slot] = true; + } +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + static void ggml_backend_sycl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor *tensor, void *data, size_t offset, @@ -792,6 +997,18 @@ static bool ggml_sycl_is_l0_discrete_gpu(int device) { } #endif +static void memcpy_host_forward(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, + const void *ptr_src, size_t size) { + char *host_buf = (char *)malloc(size); + if (host_buf == nullptr) { + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory for host staging\n", __func__, size); + GGML_ABORT("fatal error"); + } + q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); + q_dst.memcpy((char *)ptr_dst, host_buf, size).wait(); + free(host_buf); +} + static void dev2dev_memcpy(int device_dst, sycl::queue &q_dst, int device_src, sycl::queue &q_src, void *ptr_dst, const void *ptr_src, size_t size) { @@ -835,10 +1052,7 @@ static void dev2dev_memcpy(int device_dst, sycl::queue &q_dst, int device_src, s } else { GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by host forward for SYCL/L0 fallback\n"); } - char *host_buf = (char *)malloc(size); - q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); - q_dst.memcpy((char *)ptr_dst, host_buf, size).wait(); - free(host_buf); + memcpy_host_forward(q_dst, q_src, ptr_dst, ptr_src, size); } static bool @@ -973,11 +1187,14 @@ static const ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = { /* .memset_tensor = */ ggml_backend_sycl_buffer_memset_tensor, /* .set_tensor = */ ggml_backend_sycl_buffer_set_tensor, /* .get_tensor = */ ggml_backend_sycl_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, + /* .set_tensor_2d = */ ggml_backend_sycl_buffer_set_tensor_2d, /* .get_tensor_2d = */ NULL, /* .cpy_tensor = */ ggml_backend_sycl_buffer_cpy_tensor, /* .clear = */ ggml_backend_sycl_buffer_clear, /* .reset = */ ggml_backend_sycl_buffer_reset, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; // sycl buffer type @@ -1063,17 +1280,18 @@ static size_t ggml_backend_sycl_buffer_type_get_alignment(ggml_backend_buffer_ty GGML_UNUSED(buft); } -bool is_bmg_g31_arch(int device) { +static bool is_bmg_g31_arch(int device) { return ggml_sycl_info().devices[device].hw_info.arch == gpu_arch::intel_gpu_bmg_g31; } static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - size_t max_alloc_size = dpct::get_current_device().get_max_mem_alloc_size(); + ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; + int device = ctx->device; + // use the buft device, the current device may differ on multi-GPU systems + size_t max_alloc_size = dpct::get_device(device).get_max_mem_alloc_size(); if (g_ggml_sycl_host_pinned_mem_2g) { return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024); } else { - ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; - int device = ctx->device; if(is_bmg_g31_arch(device)) { //Todo, it's workaround for BMG-G31, which has a known issue with large allocations. //The max alloc size is reduced to 60% of the reported max alloc size. @@ -1289,7 +1507,12 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; ctx->tensor_extras.push_back(extra); - ctx->streams.push_back(&(dpct::get_current_device().default_queue())); + // consumers index the queues by device id, so size the vector once with one queue per device + if (ctx->streams.empty()) { + for (int i = 0; i < ggml_sycl_info().device_count; ++i) { + ctx->streams.push_back(&(dpct::get_device(i).default_queue())); + } + } for (int i = 0; i < ggml_sycl_info().device_count; ++i) { int64_t row_low, row_high; @@ -1480,6 +1703,9 @@ static struct ggml_backend_buffer_i ggml_backend_sycl_split_buffer_interface = { /* .cpy_tensor = */ NULL, /* .clear = */ ggml_backend_sycl_split_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; // sycl split buffer type @@ -1980,8 +2206,6 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { queue_ptr qptr; int device; - inline static int counter{ 0 }; - struct ggml_sycl_buffer { void * ptr = nullptr; size_t size = 0; @@ -1991,6 +2215,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { static constexpr int MAX_POOL_SIZE{ 64 }; std::vector<ggml_sycl_buffer> buffer_pool = std::vector<ggml_sycl_buffer>(MAX_POOL_SIZE); size_t pool_size = 0; + int counter = 0; explicit ggml_sycl_pool_host(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {} @@ -2004,41 +2229,55 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { b.size = 0; } } - counter = 0; } void * alloc(size_t size, size_t * actual_size) override { if (counter == MAX_POOL_SIZE) { - ggml_sycl_buffer b = buffer_pool[0]; - void * ptr = b.ptr; - *actual_size = b.size; - counter = 1; - return ptr; - } - ggml_sycl_buffer & b = buffer_pool[counter]; - - if (b.ptr == nullptr) { - void * ptr; - + // the pool wrapped around, no slot is reserved for this allocation + void * ptr = nullptr; SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr))); if (!ptr) { GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); return nullptr; } - pool_size += size; + pool_size += size; *actual_size = size; - counter = counter + 1; + counter = 0; return ptr; - } else { - ++counter; - b.size = size; - return b.ptr; } + ggml_sycl_buffer & b = buffer_pool[counter]; + ++counter; + + if (b.ptr != nullptr) { + if (b.size >= size) { + // hand out the cached buffer, free() puts it back into the pool + void * ptr = b.ptr; + *actual_size = b.size; + b.ptr = nullptr; + b.size = 0; + return ptr; + } + // the cached buffer is too small, replace it + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr))); + pool_size -= b.size; + b.ptr = nullptr; + b.size = 0; + } + + void * ptr; + SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr))); + if (!ptr) { + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); + return nullptr; + } + pool_size += size; + *actual_size = size; + return ptr; } void free(void * ptr, size_t size) override { // if the pool is not completed add the pointer to it in place of the first nullptr found. - // Otherwise do nothing, pointers will be freed once the pool is deallocated. + // Otherwise free the buffer, all slots are cached. for (int i = 0; i < MAX_POOL_SIZE; ++i) { ggml_sycl_buffer & b = buffer_pool[i]; if (b.ptr == nullptr) { @@ -2047,6 +2286,8 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { return; } } + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr))); + pool_size -= size; } }; @@ -2064,11 +2305,6 @@ std::unique_ptr<ggml_sycl_pool> ggml_backend_sycl_context::new_pool_for_device(q return std::unique_ptr<ggml_sycl_pool>(new ggml_sycl_pool_leg(qptr, device)); } - -std::unique_ptr<ggml_sycl_fattn_kv_buffers> ggml_backend_sycl_context::new_fattn_kv_buffers(queue_ptr qptr, int device) { - return std::unique_ptr<ggml_sycl_fattn_kv_buffers>(new ggml_sycl_fattn_kv_buffers(qptr, device)); -} - /// kernels typedef void (*ggml_sycl_op_mul_mat_t)( ggml_backend_sycl_context & ctx, @@ -2281,23 +2517,6 @@ k_argsort_f32_i32(const float *x, int *dst, const int ncols, int ncols_pad, } } -static void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past, - const sycl::nd_item<3> &item_ct1) { - const int col = item_ct1.get_local_range(1) * item_ct1.get_group(1) + - item_ct1.get_local_id(1); - const int row = item_ct1.get_local_range(2) * item_ct1.get_group(2) + - item_ct1.get_local_id(2); - - if (col >= ncols) { - return; - } - - const int i = row*ncols + col; - //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i]; - //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU - dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX; -} - static void scale_f32(const float * x, float * dst, const float scale, const float bias, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -2797,6 +3016,35 @@ static void top_k_f32_sycl( return; } + // Short rows: one work-item per column and k work-group argmax rounds. + // Ties take the lower column. + const int wg = std::max(WARP_SIZE, next_power_of_2((int) ncols)); + if (wg <= ggml_sycl_info().max_work_group_sizes[ctx.device]) { + main_stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(nrows * wg), sycl::range<1>(wg)), + [=](sycl::nd_item<1> item_ct1) { + const int64_t row = item_ct1.get_group(0); + const int tid = item_ct1.get_local_id(0); + const auto grp = item_ct1.get_group(); + + // NaN is never preferred, and every round still has a winner since k <= ncols + const float x = tid < ncols ? src[row * ncols + tid] : -INFINITY; + const float val = sycl::isnan(x) ? -INFINITY : x; + bool taken = tid >= ncols; + for (int i = 0; i < k; i++) { + const float cand = taken ? -INFINITY : val; + const float best = sycl::reduce_over_group(grp, cand, sycl::maximum<float>()); + const int winner = sycl::reduce_over_group(grp, !taken && cand == best ? tid : INT_MAX, + sycl::minimum<int>()); + if (tid == winner) { + dst_indices[row * k + i] = tid; + taken = true; + } + } + }); + return; + } + const int block_size = 128; const sycl::range<1> block_dims(block_size); @@ -2874,20 +3122,6 @@ static void argmax_f32_i32_sycl(const float *x, int *dst, const int ncols, }); }); } -static void diag_mask_inf_f32_sycl(const float *x, float *dst, - const int ncols_x, const int nrows_x, - const int rows_per_channel, const int n_past, - queue_ptr stream) { - const sycl::range<3> block_dims(1, SYCL_DIAG_MASK_INF_BLOCK_SIZE, 1); - const int block_num_x = (ncols_x + SYCL_DIAG_MASK_INF_BLOCK_SIZE - 1) / SYCL_DIAG_MASK_INF_BLOCK_SIZE; - const sycl::range<3> block_nums(1, block_num_x, nrows_x); - stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - diag_mask_inf_f32(x, dst, ncols_x, - rows_per_channel, n_past, - item_ct1); - }); -} static dpct::err0 ggml_sycl_cpy_tensor_2d(void *dst, const struct ggml_tensor *src, @@ -2969,6 +3203,11 @@ catch (sycl::exception const &exc) { std::exit(1); } +#if GGML_SYCL_DNNL +// F32 GEMMs below this size (row_diff * src1_ncols * ne10) call oneMKL directly instead of oneDNN +static constexpr int64_t SYCL_F32_DNNL_MIN_GEMM_FLOPS = 256 * 256 * 256; +#endif + inline void ggml_sycl_op_mul_mat_sycl( ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, @@ -3027,8 +3266,39 @@ inline void ggml_sycl_op_mul_mat_sycl( } #endif + // dequantize inside the GEMM instead of writing the f16 weights out and reading them back; src1 + // goes in its own type, so there is no separate conversion pass. The A stage + // decodes the weight bytes in place, so an installed persistent reorder must be + // visible to it: pass the flag through; types without an SoA A-stage decline + // inside and this falls through to the reorder-aware conversion route below. + bool src0_reordered = false; + if (const auto * src0_extra = static_cast<const ggml_tensor_extra_gpu *>(src0->extra)) { + src0_reordered = src0_extra->optimized_feature.reorder; + } + if (ggml_is_quantized(src0->type) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && + ggml_sycl_fused_dequant_gemm(src0->type, src0_reordered, src0_dd_i, src1_ddf_i, src1->type, ggml_sycl_src1_prec(dst), dst_dd_i, + row_diff, src1_ncols, ne10, ldc, ctx.pool(), stream)) { + return; + } + + // the f16 route converts src1 to f16 [TAG_GGML_PREC] + use_fp16 = use_fp16 && ggml_sycl_src1_f16_ok(dst); if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && use_fp16 && ggml_is_contiguous(src0) && - row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) { + row_diff == src0->ne[1] && ggml_prec_get_acc(dst) == GGML_PREC_UNDEFINED) { + ggml_sycl_pool_alloc<sycl::half> src1_as_f16(ctx.pool()); + if (src1->type != GGML_TYPE_F16) { + scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2, + " : converting src1 to fp16"); + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst); + GGML_ASSERT(to_fp16_sycl != nullptr); + size_t ne = src1_ncols*ne10; + src1_as_f16.alloc(ne); + to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream); + } + const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16 + ? (const sycl::half *)src1_ddf_i + : src1_as_f16.get(); + ggml_sycl_pool_alloc<sycl::half> src0_as_f16(ctx.pool()); if (src0->type != GGML_TYPE_F16) { scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2, @@ -3043,20 +3313,6 @@ inline void ggml_sycl_op_mul_mat_sycl( ? (const sycl::half *)src0_dd_i : src0_as_f16.get(); - ggml_sycl_pool_alloc<sycl::half> src1_as_f16(ctx.pool()); - if (src1->type != GGML_TYPE_F16) { - scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2, - " : converting src1 to fp16"); - const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst); - GGML_ASSERT(to_fp16_sycl != nullptr); - size_t ne = src1_ncols*ne10; - src1_as_f16.alloc(ne); - to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream); - } - const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16 - ? (const sycl::half *)src1->data + src1_padded_row_size - : src1_as_f16.get(); - #if GGML_SYCL_DNNL if (g_ggml_sycl_enable_dnn && ggml_sycl_dnnl_has_optimized_gemm(ggml_sycl_get_device())) { DnnlGemmWrapper::row_gemm(ctx,row_diff, src1_ncols , ne10, src0_ptr, @@ -3101,7 +3357,7 @@ inline void ggml_sycl_op_mul_mat_sycl( { #if GGML_SYCL_DNNL const int64_t gemm_flops = (int64_t)row_diff * src1_ncols * ne10; - const bool use_mkl_direct = gemm_flops < 256 * 256 * 256; + const bool use_mkl_direct = gemm_flops < SYCL_F32_DNNL_MIN_GEMM_FLOPS; if (g_ggml_sycl_enable_dnn && !use_mkl_direct && ggml_sycl_dnnl_has_optimized_gemm(ggml_sycl_get_device())) { DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, src0_ddf_i, DnnlGemmWrapper::to_dt<float>(), src1_ddf1_i, DnnlGemmWrapper::to_dt<float>(), @@ -3138,6 +3394,8 @@ inline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst) float * dst_dd = static_cast<float *>(dst->data); const int64_t ne = ggml_nelements(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ne <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ne, 1, main_stream); } @@ -3152,6 +3410,8 @@ inline void ggml_sycl_op_sum_rows(ggml_backend_sycl_context & ctx, ggml_tensor * const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); } @@ -3168,6 +3428,8 @@ inline void ggml_sycl_op_mean(ggml_backend_sycl_context & ctx, ggml_tensor * dst const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); @@ -3237,27 +3499,12 @@ inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * d const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); } -inline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); - dpct::queue_ptr main_stream = ctx.stream(); - SYCL_CHECK(ggml_sycl_set_device(ctx.device)); - const float * src0_dd = static_cast<const float *>(dst->src[0]->data); - float * dst_dd = static_cast<float *>(dst->data); - - const int64_t ne00 = dst->src[0]->ne[0]; - const int64_t ne01 = dst->src[0]->ne[1]; - const int nrows0 = ggml_nrows(dst->src[0]); - - const int n_past = ((int32_t *) dst->op_params)[0]; - - diag_mask_inf_f32_sycl(src0_dd, dst_dd, ne00, nrows0, ne01, n_past, main_stream); -} - static void tri_f32_sycl( const float * src, float * dst, @@ -3330,6 +3577,9 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * ds memcpy(&scale, (float *) dst->op_params + 0, sizeof(float)); memcpy(&bias, (float *) dst->op_params + 1, sizeof(float)); + // the kernel indexes in 32-bit + GGML_ASSERT(ggml_nelements(dst->src[0]) <= INT32_MAX); + scale_f32_sycl(src0_dd, dst_dd, scale, bias, ggml_nelements(dst->src[0]), main_stream); /* DPCT1010:87: SYCL uses exceptions to report errors and does not use the @@ -3410,7 +3660,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten const size_t q8_1_bs = QK8_1; ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; const bool src0_is_contiguous = ggml_is_contiguous(src0); const bool src1_is_contiguous = ggml_is_contiguous(src1); @@ -3492,7 +3741,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten queue_ptr stream = ctx.stream(i, 0); if (src0_is_contiguous) { - dev[i].src0_dd = (char *) src0->data; + // a split tensor keeps its rows in the per-device slices of the tensor extra + dev[i].src0_dd = split ? (char *) src0_extra->data_device[i] : (char *) src0->data; } else { dev[i].src0_dd = dev[i].src0_dd_alloc.alloc(ctx.pool(i), ggml_nbytes(src0)); } @@ -3588,7 +3838,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten src1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs) .wait())); } else { - const char * src1_ddf_i_source = (const char *) src1_extra->data_device[ctx.device] + + // src1 is not split, on the main device it lives at dev[ctx.device].src1_ddf + const char * src1_ddf_i_source = (const char *) dev[ctx.device].src1_ddf + (i0 * ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type); SYCL_CHECK( @@ -4130,6 +4381,8 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) { case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: return true; default: return false; @@ -4181,7 +4434,15 @@ static inline void * sycl_ext_malloc_device(dpct::queue_ptr stream, size_t size) bool use_async = g_ggml_sycl_use_async_mem_op; #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC if (use_async) { - void * ptr = syclex::async_malloc(*stream, sycl::usm::alloc::device, size); + // async_malloc throws on out-of-memory; return nullptr like the sync path so callers can fall back. + void * ptr = nullptr; + try { + ptr = syclex::async_malloc(*stream, sycl::usm::alloc::device, size); + } catch (const sycl::exception & e) { + GGML_LOG_WARN("%s: async_malloc of %zu bytes failed: %s\n", __func__, size, e.what()); + ggml_sycl_memtrace_fail(GGML_SYCL_MEM_ASYNC, size); + return nullptr; + } ggml_sycl_memtrace_add(GGML_SYCL_MEM_ASYNC, ptr, size); return ptr; } @@ -4248,460 +4509,467 @@ struct sycl_reorder_temp_buffer { bool host_fallback = false; }; -static bool reorder_qw_q4_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, - dpct::queue_ptr stream) { - sycl_reorder_temp_buffer tmp(stream, size); +// Reorder a 2D tensor of quantized blocks in place into [head of every block][tail of every block] without a +// full-size temporary copy. The head is the contiguous `head_bytes` at `head_offset` inside each block. Heads only +// move towards the front, so they are packed in chunks that never overwrite unread blocks. The tail overlaps +// blocks that are not packed yet, so `write_tail(x, ib, tail, nblocks)` first writes every block's remaining fields +// (in their final layout, relative to the start of the tail) into a side buffer that is copied to the end afterwards. +template <typename block_t, typename tail_fn_t> +static bool reorder_qw_planar(const char * name, uint8_t * data_device, size_t size, size_t head_offset, + size_t head_bytes, tail_fn_t write_tail, dpct::queue_ptr stream) { + GGML_ASSERT(size % sizeof(block_t) == 0); + GGML_ASSERT(head_offset + head_bytes <= sizeof(block_t)); + + const int64_t nblocks = size / sizeof(block_t); + const size_t tail_bytes = (sizeof(block_t) - head_bytes) * nblocks; + + sycl_reorder_temp_buffer tail_tmp(stream, tail_bytes); + if (!tail_tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", name, tail_bytes); + return false; + } + uint8_t * tail_buf = static_cast<uint8_t *>(tail_tmp.ptr); + + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_blocks = std::max<int64_t>(1, std::min<int64_t>(nblocks, max_chunk_bytes / sizeof(block_t))); + sycl_reorder_temp_buffer tmp(stream, chunk_blocks * sizeof(block_t)); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", name, + (size_t) chunk_blocks * sizeof(block_t)); return false; } uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); + sycl::event event = stream->parallel_for(nblocks, [=](auto i) { + const int64_t ib = i; + write_tail(((const block_t *) data_device)[ib], ib, tail_buf, nblocks); + }); if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); + event.wait_and_throw(); } - GGML_ASSERT((size % sizeof(block_q4_0) == 0)); - GGML_ASSERT((offset % sizeof(block_q4_0) == 0)); - int offset_blks = offset / sizeof(block_q4_0); - auto qs_ptr = data_device + offset_blks * QK4_0 / 2; - auto d_ptr = (sycl::half*)(qs_ptr + ncols * nrows / 2) + offset_blks; + for (int64_t b0 = 0; b0 < nblocks; b0 += chunk_blocks) { + const int64_t n_chunk = std::min(chunk_blocks, nblocks - b0); - auto reorder_event = stream->parallel_for( - size / sizeof(block_q4_0), - [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - const block_q4_0* x = (const block_q4_0*)tmp_buf; - const int ib = i; + SYCL_CHECK(CHECK_TRY_ERROR(event = stream->memcpy(tmp_buf, data_device + b0 * sizeof(block_t), + n_chunk * sizeof(block_t)))); + if (!g_ggml_sycl_use_async_mem_op) { + event.wait(); + } - for (int j = 0; j < QK4_0/2; j ++) - { - *(qs_ptr + ib * QK4_0 / 2 + j) = x[ib].qs[j]; + uint8_t * head_ptr = data_device + b0 * head_bytes; + event = stream->parallel_for(n_chunk, [=](auto i) { + const int64_t ib = i; + const uint8_t * src = tmp_buf + ib * sizeof(block_t) + head_offset; + for (size_t j = 0; j < head_bytes; ++j) { + head_ptr[ib * head_bytes + j] = src[j]; } - *(d_ptr + ib) = x[ib].d; }); + if (!g_ggml_sycl_use_async_mem_op) { + event.wait_and_throw(); + } + } + + SYCL_CHECK(CHECK_TRY_ERROR(event = stream->memcpy(data_device + head_bytes * nblocks, tail_buf, tail_bytes))); if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); + event.wait(); } return true; } -static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, +static bool reorder_qw_q4_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, dpct::queue_ptr stream) { - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - GGML_ASSERT((size % sizeof(block_q8_0) == 0)); - GGML_ASSERT((offset % sizeof(block_q8_0) == 0)); - int offset_blks = offset / sizeof(block_q8_0); - auto qs_ptr = data_device + offset_blks * QK8_0; - auto d_ptr = (sycl::half*)(qs_ptr + ncols * nrows) + offset_blks; - - auto reorder_event = stream->parallel_for( - size / sizeof(block_q8_0), - [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - const block_q8_0* x = (const block_q8_0*)tmp_buf; - const int ib = i; + GGML_ASSERT(offset == 0); + GGML_ASSERT((size_t) ncols * nrows / 2 == size / sizeof(block_q4_0) * (QK4_0 / 2)); + return reorder_qw_planar<block_q4_0>(__func__, data_device, size, offsetof(block_q4_0, qs), QK4_0 / 2, + [](const block_q4_0 & x, int64_t ib, uint8_t * tail, int64_t) { + ((sycl::half *) tail)[ib] = x.d; + }, stream); +} - for (int j = 0; j < QK8_0; j++) - { - *((int8_t*)qs_ptr + ib * QK8_0 + j) = x[ib].qs[j]; - } - *(d_ptr + ib) = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; +static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, + dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + GGML_ASSERT((size_t) ncols * nrows == size / sizeof(block_q8_0) * QK8_0); + return reorder_qw_planar<block_q8_0>(__func__, data_device, size, offsetof(block_q8_0, qs), QK8_0, + [](const block_q8_0 & x, int64_t ib, uint8_t * tail, int64_t) { + ((sycl::half *) tail)[ib] = x.d; + }, stream); } static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q4_K) == 0); - GGML_ASSERT(offset % sizeof(block_q4_K) == 0); + GGML_ASSERT(offset == 0); + return reorder_qw_planar<block_q4_K>(__func__, data_device, size, offsetof(block_q4_K, qs), QK_K / 2, + [](const block_q4_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * scales = tail; + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales[ib * K_SCALE_SIZE + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + K_SCALE_SIZE * nblocks))[ib] = x.dm; + }, stream); +} - const int nblocks = size / sizeof(block_q4_K); +// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout, a few experts at a time. +static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max<int64_t>(1, std::min<int64_t>(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, size); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - auto * qs_ptr = data_device; - auto * scales_ptr = qs_ptr + QK_K / 2 * nblocks; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * nblocks); + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q4_K * x = (const block_q4_K *) tmp_buf; - const int ib = i; + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; - } + auto * qs_ptr = base; + auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0); - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + for (int j = 0; j < QK8_0; ++j) { + *((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j]; + } + d_ptr[ib] = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } // Reorder each expert slice into a self-contained SoA layout. +// Like the MXFP4 version below, this goes a few experts at a time through a small temporary: +// a whole-tensor temporary (GBs for large MoE models) can exceed the VRAM left on a nearly full card. static bool reorder_qw_q4_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { GGML_ASSERT(expert_bytes % sizeof(block_q4_K) == 0); - const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q4_K)); - const size_t total_bytes = expert_bytes * (size_t) n_expert; + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q4_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max<int64_t>(1, std::min<int64_t>(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, total_bytes); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + // the queue is in-order: each chunk's copy into tmp_buf waits for the previous chunk's kernel + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; + + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - const int total_blocks = blocks_per_expert * (int) n_expert; - auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { - const int gb = gb_; - const int e = gb / blocks_per_expert; - const int ib = gb % blocks_per_expert; - const block_q4_K * x = (const block_q4_K *) (tmp_buf + (size_t) e * expert_bytes); - uint8_t * base = data_device + (size_t) e * expert_bytes; + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q4_K * x = (const block_q4_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - auto * qs_ptr = base; - auto * scales_ptr = qs_ptr + QK_K / 2 * blocks_per_expert; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); + auto * qs_ptr = base; + auto * scales_ptr = qs_ptr + QK_K / 2 * blocks_per_expert; + auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; - } - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + for (int j = 0; j < QK_K / 2; ++j) { + qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + } + dm_ptr[ib] = x[ib].dm; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } -// Reorder each Q5_K expert slice into [qs][qh][scales][dm]. +// Reorder each Q5_K expert slice into [qs][qh][scales][dm], a few experts at a time (see Q4_K above). static bool reorder_qw_q5_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { GGML_ASSERT(expert_bytes % sizeof(block_q5_K) == 0); - const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q5_K)); - const size_t total_bytes = expert_bytes * (size_t) n_expert; + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q5_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max<int64_t>(1, std::min<int64_t>(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, total_bytes); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - const int total_blocks = blocks_per_expert * (int) n_expert; - auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { - const int gb = gb_; - const int e = gb / blocks_per_expert; - const int ib = gb % blocks_per_expert; - const block_q5_K * x = (const block_q5_K *) (tmp_buf + (size_t) e * expert_bytes); - uint8_t * base = data_device + (size_t) e * expert_bytes; + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - auto * qs_ptr = base; - auto * qh_ptr = qs_ptr + (QK_K / 2) * blocks_per_expert; - auto * scales_ptr = qh_ptr + (QK_K / 8) * blocks_per_expert; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q5_K * x = (const block_q5_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; - } - for (int j = 0; j < QK_K / 8; ++j) { - qh_ptr[ib * (QK_K / 8) + j] = x[ib].qh[j]; - } - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + auto * qs_ptr = base; + auto * qh_ptr = qs_ptr + (QK_K / 2) * blocks_per_expert; + auto * scales_ptr = qh_ptr + (QK_K / 8) * blocks_per_expert; + auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); + + for (int j = 0; j < QK_K / 2; ++j) { + qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + } + for (int j = 0; j < QK_K / 8; ++j) { + qh_ptr[ib * (QK_K / 8) + j] = x[ib].qh[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + } + dm_ptr[ib] = x[ib].dm; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } -// Reorder each Q6_K expert slice into [ql][qh][scales][d]. +// Reorder each Q6_K expert slice into [ql][qh][scales][d], a few experts at a time (see Q4_K above). static bool reorder_qw_q6_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { GGML_ASSERT(expert_bytes % sizeof(block_q6_K) == 0); - const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q6_K)); - const size_t total_bytes = expert_bytes * (size_t) n_expert; + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q6_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max<int64_t>(1, std::min<int64_t>(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, total_bytes); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - const int total_blocks = blocks_per_expert * (int) n_expert; - auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { - const int gb = gb_; - const int e = gb / blocks_per_expert; - const int ib = gb % blocks_per_expert; - const block_q6_K * x = (const block_q6_K *) (tmp_buf + (size_t) e * expert_bytes); - uint8_t * base = data_device + (size_t) e * expert_bytes; + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - auto * ql_ptr = base; - auto * qh_ptr = ql_ptr + (QK_K / 2) * blocks_per_expert; - auto * scales_ptr = qh_ptr + (QK_K / 4) * blocks_per_expert; - auto * d_ptr = (sycl::half *) (scales_ptr + (QK_K / 16) * blocks_per_expert); + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q6_K * x = (const block_q6_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - for (int j = 0; j < QK_K / 2; ++j) { - ql_ptr[ib * (QK_K / 2) + j] = x[ib].ql[j]; - } - for (int j = 0; j < QK_K / 4; ++j) { - qh_ptr[ib * (QK_K / 4) + j] = x[ib].qh[j]; - } - for (int j = 0; j < QK_K / 16; ++j) { - scales_ptr[ib * (QK_K / 16) + j] = x[ib].scales[j]; + auto * ql_ptr = base; + auto * qh_ptr = ql_ptr + (QK_K / 2) * blocks_per_expert; + auto * scales_ptr = qh_ptr + (QK_K / 4) * blocks_per_expert; + auto * d_ptr = (sycl::half *) (scales_ptr + (QK_K / 16) * blocks_per_expert); + + for (int j = 0; j < QK_K / 2; ++j) { + ql_ptr[ib * (QK_K / 2) + j] = x[ib].ql[j]; + } + for (int j = 0; j < QK_K / 4; ++j) { + qh_ptr[ib * (QK_K / 4) + j] = x[ib].qh[j]; + } + for (int j = 0; j < QK_K / 16; ++j) { + scales_ptr[ib * (QK_K / 16) + j] = x[ib].scales[j]; + } + d_ptr[ib] = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - d_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } -static bool reorder_qw_q2_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q2_K) == 0); - GGML_ASSERT(offset % sizeof(block_q2_K) == 0); - - const int nblocks = size / sizeof(block_q2_K); +// Reorder each MXFP4 expert slice into [qs][e]: 16-byte nibble blocks, then one E8M0 byte per block. +// Experts are self-contained, so the tensor is reordered a few experts at a time through a small +// temporary: a whole-tensor temporary (hundreds of MB) can exceed the VRAM left on a nearly full card, +// and on Windows the driver then pages device memory out to host RAM instead of failing. +static bool reorder_qw_mxfp4_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_mxfp4) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_mxfp4)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max<int64_t>(1, std::min<int64_t>(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, size); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + // the queue is in-order: each chunk's copy into tmp_buf waits for the previous chunk's kernel + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - auto * qs_ptr = data_device; - auto * scales_ptr = qs_ptr + (QK_K / 4) * nblocks; - sycl::half2 * dm_ptr = (sycl::half2 *) (scales_ptr + (QK_K / 16) * nblocks); + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q2_K * x = (const block_q2_K *) tmp_buf; - const int ib = i; + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_mxfp4 * x = (const block_mxfp4 *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - for (int j = 0; j < QK_K / 4; ++j) { - qs_ptr[ib * (QK_K / 4) + j] = x[ib].qs[j]; - } + uint8_t * qs_ptr = base; + uint8_t * e_ptr = qs_ptr + (QK_MXFP4 / 2) * (size_t) blocks_per_expert; - for (int j = 0; j < QK_K / 16; ++j) { - scales_ptr[ib * (QK_K / 16) + j] = x[ib].scales[j]; + for (int j = 0; j < QK_MXFP4 / 2; ++j) { + qs_ptr[(size_t) ib * (QK_MXFP4 / 2) + j] = x[ib].qs[j]; + } + e_ptr[ib] = x[ib].e; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } -static bool reorder_qw_q3_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q3_K) == 0); - GGML_ASSERT(offset % sizeof(block_q3_K) == 0); - - const int nblocks = size / sizeof(block_q3_K); - - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * qs_ptr = data_device; - auto * hmask_ptr = qs_ptr + (QK_K / 4) * nblocks; - auto * scales_ptr = hmask_ptr + (QK_K / 8) * nblocks; - sycl::half * d_ptr = (sycl::half *) (scales_ptr + 12 * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q3_K * x = (const block_q3_K *) tmp_buf; - const int ib = i; - - for (int j = 0; j < QK_K / 4; ++j) { - qs_ptr[ib * (QK_K / 4) + j] = x[ib].qs[j]; - } +static bool reorder_qw_q2_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + return reorder_qw_planar<block_q2_K>(__func__, data_device, size, offsetof(block_q2_K, qs), QK_K / 4, + [](const block_q2_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * scales = tail; + for (int j = 0; j < QK_K / 16; ++j) { + scales[ib * (QK_K / 16) + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + (QK_K / 16) * nblocks))[ib] = x.dm; + }, stream); +} - for (int j = 0; j < QK_K / 8; ++j) { - hmask_ptr[ib * (QK_K / 8) + j] = x[ib].hmask[j]; - } +static bool reorder_qw_q3_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + return reorder_qw_planar<block_q3_K>(__func__, data_device, size, offsetof(block_q3_K, qs), QK_K / 4, + [](const block_q3_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * hmask = tail; + uint8_t * scales = hmask + (QK_K / 8) * nblocks; + for (int j = 0; j < QK_K / 8; ++j) { + hmask[ib * (QK_K / 8) + j] = x.hmask[j]; + } + for (int j = 0; j < 12; ++j) { + scales[ib * 12 + j] = x.scales[j]; + } + ((sycl::half *) (scales + 12 * nblocks))[ib] = x.d; + }, stream); +} - for (int j = 0; j < 12; ++j) { - scales_ptr[ib * 12 + j] = x[ib].scales[j]; +// Reorder each Q3_K expert slice on its own. +static bool reorder_qw_q3_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + for (int64_t e = 0; e < n_expert; ++e) { + if (!reorder_qw_q3_k(data_device + e * expert_bytes, expert_bytes, 0, stream)) { + return false; } - - d_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } static bool reorder_qw_q5_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q5_K) == 0); - GGML_ASSERT(offset % sizeof(block_q5_K) == 0); - - const int nblocks = size / sizeof(block_q5_K); - - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * qs_ptr = data_device; - auto * qh_ptr = qs_ptr + (QK_K / 2) * nblocks; - auto * scales_ptr = qh_ptr + (QK_K / 8) * nblocks; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q5_K * x = (const block_q5_K *) tmp_buf; - const int ib = i; - - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; - } - - for (int j = 0; j < QK_K / 8; ++j) { - qh_ptr[ib * (QK_K / 8) + j] = x[ib].qh[j]; - } - - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; - } - - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; + GGML_ASSERT(offset == 0); + return reorder_qw_planar<block_q5_K>(__func__, data_device, size, offsetof(block_q5_K, qs), QK_K / 2, + [](const block_q5_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * scales = qh + (QK_K / 8) * nblocks; + for (int j = 0; j < QK_K / 8; ++j) { + qh[ib * (QK_K / 8) + j] = x.qh[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales[ib * K_SCALE_SIZE + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + K_SCALE_SIZE * nblocks))[ib] = x.dm; + }, stream); } static bool reorder_qw_q6_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q6_K) == 0); - GGML_ASSERT(offset % sizeof(block_q6_K) == 0); - - const int nblocks = size / sizeof(block_q6_K); - - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * ql_ptr = data_device; - auto * qh_ptr = ql_ptr + (QK_K / 2) * nblocks; - auto * scales_ptr = qh_ptr + (QK_K / 4) * nblocks; - sycl::half * dm_ptr = (sycl::half *) (scales_ptr + (QK_K / 16) * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q6_K * x = (const block_q6_K *) tmp_buf; - const int ib = i; - - const uint8_t * ql = x[ib].ql; - const uint8_t * qh = x[ib].qh; - uint8_t * base_ql_ptr = ql_ptr + (QK_K / 2) * ib; - uint8_t * base_qh_ptr = qh_ptr + (QK_K / 4) * ib; - uint8_t * base_scales_ptr = scales_ptr + (QK_K / 16) * ib; - - for (int j = 0; j < QK_K / 2; ++j) { - base_ql_ptr[j] = ql[j]; - } - for (int j = 0; j < QK_K / 4; ++j) { - base_qh_ptr[j] = qh[j]; - } - - for (int j = 0; j < QK_K / 16; ++j) { - base_scales_ptr[j] = x[ib].scales[j]; - } + GGML_ASSERT(offset == 0); + return reorder_qw_planar<block_q6_K>(__func__, data_device, size, offsetof(block_q6_K, ql), QK_K / 2, + [](const block_q6_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * scales = qh + (QK_K / 4) * nblocks; + for (int j = 0; j < QK_K / 4; ++j) { + qh[ib * (QK_K / 4) + j] = x.qh[j]; + } + for (int j = 0; j < QK_K / 16; ++j) { + scales[ib * (QK_K / 16) + j] = x.scales[j]; + } + ((sycl::half *) (scales + (QK_K / 16) * nblocks))[ib] = x.d; + }, stream); +} + +static bool reorder_qw_iq3_s(uint8_t * data_device, size_t size, dpct::queue_ptr stream) { + return reorder_qw_planar<block_iq3_s>(__func__, data_device, size, offsetof(block_iq3_s, qs), QK_K / 4, + [](const block_iq3_s & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * signs = qh + (QK_K / 32) * nblocks; + uint8_t * metadata = signs + (QK_K / 8) * nblocks + ib * (sizeof(ggml_half) + IQ3S_N_SCALE); + for (int j = 0; j < QK_K / 32; ++j) { + qh[ib * (QK_K / 32) + j] = x.qh[j]; + } + for (int j = 0; j < QK_K / 8; ++j) { + signs[ib * (QK_K / 8) + j] = x.signs[j]; + } + *reinterpret_cast<ggml_half *>(metadata) = x.d; + for (int j = 0; j < IQ3S_N_SCALE; ++j) { + metadata[sizeof(ggml_half) + j] = x.scales[j]; + } + }, stream); +} - dm_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; +static bool reorder_qw_iq3_xxs(uint8_t * data_device, size_t size, dpct::queue_ptr stream) { + return reorder_qw_planar<block_iq3_xxs>(__func__, data_device, size, offsetof(block_iq3_xxs, qs), 3 * QK_K / 8, + [](const block_iq3_xxs & x, int64_t ib, uint8_t * tail, int64_t) { + reinterpret_cast<ggml_half *>(tail)[ib] = x.d; + }, stream); } static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { @@ -4715,12 +4983,18 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { if (src0->ne[2] > 1) { GGML_ASSERT((size_t) size == (size_t) src0->ne[2] * src0->nb[2]); switch (src0->type) { + case GGML_TYPE_Q8_0: + return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream); + case GGML_TYPE_Q3_K: + return reorder_qw_q3_k_moe(data_device, src0->nb[2], src0->ne[2], stream); case GGML_TYPE_Q4_K: return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream); case GGML_TYPE_Q5_K: return reorder_qw_q5_k_moe(data_device, src0->nb[2], src0->ne[2], stream); case GGML_TYPE_Q6_K: return reorder_qw_q6_k_moe(data_device, src0->nb[2], src0->ne[2], stream); + case GGML_TYPE_MXFP4: + return reorder_qw_mxfp4_moe(data_device, src0->nb[2], src0->ne[2], stream); default: return false; } @@ -4741,6 +5015,10 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { return reorder_qw_q5_k(data_device, size, 0, stream); case GGML_TYPE_Q6_K: return reorder_qw_q6_k(data_device, size, 0, stream); + case GGML_TYPE_IQ3_XXS: + return reorder_qw_iq3_xxs(data_device, size, stream); + case GGML_TYPE_IQ3_S: + return reorder_qw_iq3_s(data_device, size, stream); default: return false; } @@ -4794,7 +5072,16 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) { return; } - if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K) { + if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q3_K && src0->type != GGML_TYPE_Q4_K && + src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) { + return; + } + // The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned. + if (src0->type == GGML_TYPE_MXFP4 && (src0->nb[2] % 16 != 0 || (uintptr_t) src0->data % 16 != 0)) { + return; + } + // The grouped GEMM loads reordered Q3_K qs and hmask as 16-byte vectors. + if (src0->type == GGML_TYPE_Q3_K && ((src0->nb[2] / sizeof(block_q3_K)) % 16 != 0 || (uintptr_t) src0->data % 16 != 0)) { return; } ggml_tensor_extra_gpu * extra = static_cast<ggml_tensor_extra_gpu *>(src0->extra); @@ -4817,9 +5104,38 @@ static bool can_use_dequantize_mul_mat_vec(const ggml_tensor * src0, const ggml_ src0->ne[0] % dmmv_x_required == 0 && src1->ne[1] == 1; } -static bool can_use_mul_mat_vec_q(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +// reordered weights of the types in ggml_sycl_xmx_supports_type() take the XMX kernel for wider batches +static bool can_use_xmx_batch(int device, const ggml_tensor * src0, const ggml_tensor * src1) { +#ifdef GGML_SYCL_MMVQ_HAS_XMX + const auto * extra = static_cast<const ggml_tensor_extra_gpu *>(src0->extra); + return ggml_sycl_xmx_enabled(device) && ggml_sycl_xmx_supports_type(src0->type) && extra && + extra->optimized_feature.reorder && src1->ne[1] <= g_ggml_sycl_xmx_max_cols && src1->ne[2] == 1 && + src1->ne[3] == 1; +#else + GGML_UNUSED(device); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + return false; +#endif // GGML_SYCL_MMVQ_HAS_XMX +} + +static bool can_use_mul_mat_vec_q(int device, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { return ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && - src1->ne[1] <= MMVQ_MAX_BATCH_SIZE; + (src1->ne[1] <= MMVQ_MAX_BATCH_SIZE || can_use_xmx_batch(device, src0, src1)); +} + +static bool can_use_mul_mat_q(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + // mmvq and mmq need the __dp4a instruction which is available for gen12+ + // Workaround in https://github.com/ggml-org/llama.cpp/commit/95f84d5ce8b449a9b16009434aca800df504a02e + if (!ggml_sycl_supports_mmq(src0->type) || src0->type == GGML_TYPE_IQ2_XXS) { + return false; + } +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) + if (src1->ne[1] > MMQ_MAX_BATCH_SIZE) { + return false; + } +#endif // GGML_SYCL_SUPPORT_XMX && !GGML_SYCL_FORCE_MMQ + return src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; } static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -4859,19 +5175,17 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor // check data types and tensor shapes for custom matrix multiplication kernels: bool use_dequantize_mul_mat_vec = can_use_dequantize_mul_mat_vec(src0, src1, dst); + if (src1->ne[1] == 1 && ggml_sycl_xmx_min_cols(src0->type) == 1 && can_use_xmx_batch(ctx.device, src0, src1)) { + use_dequantize_mul_mat_vec = false; + } +#ifdef GGML_SYCL_F16 + // dmmv may convert src1 to f16 in this build [TAG_GGML_PREC] + use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && ggml_sycl_src1_f16_ok(dst); +#endif - bool use_mul_mat_vec_q = can_use_mul_mat_vec_q(src0, src1, dst); - - bool use_mul_mat_q = ggml_sycl_supports_mmq(src0->type) - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; - + bool use_mul_mat_vec_q = can_use_mul_mat_vec_q(ctx.device, src0, src1, dst); - // mmvq and mmq need the __dp4a instruction which is available for gen12+ - // Workaround in https://github.com/ggml-org/llama.cpp/commit/95f84d5ce8b449a9b16009434aca800df504a02e - use_mul_mat_q = use_mul_mat_q && (src0->type != GGML_TYPE_IQ2_XXS); -#ifdef SYCL_USE_XMX - use_mul_mat_q = use_mul_mat_q && (src1->ne[1] <= MMQ_MAX_BATCH_SIZE); -#endif // SYCL_USE_XMX + bool use_mul_mat_q = can_use_mul_mat_q(src0, src1, dst); // When reorder is enabled, both ESIMD, MMVQ and DMMV kernels may be used. For // best performance use ESIMD when supported, followed by MMVQ, and finally DMMV. @@ -4939,6 +5253,12 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg return false; } + // the plain kernel only covers q5_K / iq4_xs pairs: bail before quantizing the activation + const auto plain_type = [](ggml_type t) { return t == GGML_TYPE_Q5_K || t == GGML_TYPE_IQ4_XS; }; + if (!plain_type(wu->type) || !plain_type(wg->type)) { + return false; + } + // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (plain layout)"); @@ -4960,11 +5280,74 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg (int) ne11, /*stride_col_y=*/src1_padded_cols / QK8_1, /*stride_col_dst=*/(int) glu->ne[0], stream); + GGML_UNUSED(gate); +} + +// Gate and up weights of one type on the XMX engines: one launch reads the activations once and applies the GLU. +// Both weights must already be in the reorder layout. Returns false if the kernel declined. +static bool ggml_sycl_mul_mat_glu_xmx(ggml_backend_sycl_context & ctx, + ggml_tensor * glu, + ggml_tensor * up, + const ggml_tensor * wu, + const ggml_tensor * wg, + const ggml_tensor * act) { + const int64_t ne00 = wu->ne[0]; + const int64_t ne11 = act->ne[1]; + + const queue_ptr stream = ctx.stream(); + const int src1_padded_cols = GGML_PAD((int) ne00, MATRIX_ROW_PADDING); + const int stride_y_bytes = src1_padded_cols * (int) sizeof(block_q8_1) / QK8_1; + + // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises + scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (XMX)"); + + ggml_sycl_pool_alloc<char> src1_q8_alloc(ctx.pool(), (size_t) ne11 * stride_y_bytes); + char * src1_ddq = src1_q8_alloc.get(); + + quantize_row_q8_1_sycl<quantize_and_reorder_q8_1_soa>((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, + src1_padded_cols, stream); + + return ggml_sycl_mul_mat_vec_q_glu_xmx(ctx.device, wu->type, ggml_get_glu_op(glu), wu->data, wg->data, src1_ddq, + (float *) glu->data, (int) ne00, (int) wu->ne[1], (int) ne11, stride_y_bytes, + /*stride_col_dst=*/(int) glu->ne[0], stream); } +#ifdef GGML_SYCL_DMMV_HAS_ESIMD +// {mul_mat(gate), mul_mat(up), GLU} on the reorder layout via the ESIMD DMMV kernel, for +// same-type K-quant or Q8_0 pairs and a single activation column. It reads the F32 activation +// directly, so one launch replaces three and no q8_1 quantization is needed. +static bool ggml_sycl_mul_mat_glu_dmmv_esimd(ggml_backend_sycl_context & ctx, ggml_tensor * glu, + ggml_tensor * gate, ggml_tensor * up, const ggml_tensor * wu, + const ggml_tensor * wg, const ggml_tensor * act) { + // the kernel reads the activation while writing the output; glu may reuse the dead activation's memory + const char * act_begin = (const char *) act->data; + const char * glu_begin = (const char *) glu->data; + if (act_begin < glu_begin + ggml_nbytes(glu) && glu_begin < act_begin + ggml_nbytes(act)) { + return false; + } + + // install the reorder (SoA) layout the kernel needs, as the unfused DMMV path would + opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::DMMV); + opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::DMMV); + + const auto * extra_u = static_cast<const ggml_tensor_extra_gpu *>(wu->extra); + const auto * extra_g = static_cast<const ggml_tensor_extra_gpu *>(wg->extra); + if (!extra_u || !extra_g || !extra_u->optimized_feature.reorder || !extra_g->optimized_feature.reorder) { + return false; + } + + // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises + scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (ESIMD)"); + + return ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(wu->type, ggml_get_glu_op(glu), wu->data, wg->data, + (const float *) act->data, (float *) glu->data, + (int) wu->ne[0], (int) wu->ne[1], ctx.stream()); +} +#endif + // Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx. // Returns false if it declined, in which case the caller runs the three nodes normally. -static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) { +static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) { if (!ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) { return false; } @@ -4982,17 +5365,56 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg return false; } +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + // single-column same-type K-quant or Q8_0 pairs take the ESIMD kernel, as the unfused mul-mats + // would via DMMV; if the reorder layout could not be installed, fall through to mmvq + if (g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 && + ggml_sycl_supports_reorder_esimd(wu->type) && + ggml_sycl_mul_mat_glu_dmmv_esimd(ctx, glu, gate, up, wu, wg, act)) { + return true; + } +#endif + // with DMMV prioritised the unfused path would not have gone through mmvq at all if (g_ggml_sycl_prioritize_dmmv) { return false; } - // quant pairs the reorder kernel cannot serve (mixed gate/up types) take the - // standard-layout fused path instead; q4_K keeps the reorder path below - if (wg->type != GGML_TYPE_Q4_K || wu->type != GGML_TYPE_Q4_K) { + // gate and up of one type the XMX kernel handles, it needs the reorder layout like the unfused mmvq path + if (wg->type == wu->type && gate->src[1] == act && act->ne[1] >= ggml_sycl_xmx_min_cols(wu->type) && + act->ne[1] <= g_ggml_sycl_xmx_glu_max_cols) { + opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ); + opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::MMVQ); + if (can_use_xmx_batch(ctx.device, wu, act) && can_use_xmx_batch(ctx.device, wg, act) && + ggml_sycl_mul_mat_glu_xmx(ctx, glu, up, wu, wg, act)) { + return true; + } + } + + // the kernels below take up to MMVQ_MAX_BATCH_SIZE columns, a wider batch got here for the XMX kernel + if (act->ne[1] > MMVQ_MAX_BATCH_SIZE) { + return false; + } + + // quant pairs the reorder kernel does not serve (mixed gate/up types, q5_K off BMG) take the + // standard-layout fused path instead; same-type q4_K / q5_K keep the reorder path below + const bool reorder_pair = wg->type == wu->type && + (wu->type == GGML_TYPE_Q4_K || (wu->type == GGML_TYPE_Q5_K && ggml_sycl_q5_k_mmvq_reuse(ctx.device))); + if (!reorder_pair) { + const auto is_plain_type = [](ggml_type type) { + return type == GGML_TYPE_Q5_K || type == GGML_TYPE_IQ4_XS; + }; + if (!is_plain_type(wu->type) || !is_plain_type(wg->type)) { + return false; + } return ggml_sycl_mul_mat_glu_mmvq_plain(ctx, glu, gate, up, wu, wg, act); } + // past 5 columns the two unfused q5_K GEMVs are faster than the fused kernel + if (wu->type == GGML_TYPE_Q5_K && act->ne[1] > 5) { + return false; + } + // install the reorder (SoA) layout the fused kernel needs, as the unfused mmvq path would; // a no-op once done. after the bail checks so a declined op does not pay for it. opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ); @@ -5143,29 +5565,66 @@ __dpct_inline__ static void k_copy_dst_from_contiguous( } } -// Fused MoE TG fast path. Returns false to fall back to the per-expert loop below. -static bool ggml_sycl_mul_mat_id_mmvq_fused( - ggml_backend_sycl_context & ctx, const ggml_tensor * src0, - const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) -{ +__dpct_inline__ static void k_zero_dst_rows( + char *__restrict__ dst_original, + const mmid_row_mapping *__restrict__ row_mapping, int64_t ne0, size_t nb1, + size_t nb2, const sycl::nd_item<3> &item_ct1) { + int32_t i = item_ct1.get_group(2); + + const int32_t i1 = row_mapping[i].i1; + const int32_t i2 = row_mapping[i].i2; + + float * dst_row_original = (float *)(dst_original + i1*nb1 + i2*nb2); + + for (int j = item_ct1.get_local_id(2); j < ne0; + j += item_ct1.get_local_range(2)) { + dst_row_original[j] = 0.0f; + } +} + +// layout checks for the fused MMVQ path; the type check is ggml_sycl_mul_mat_vec_q_id_supports_type +static bool ggml_sycl_mul_mat_id_mmvq_eligible( + const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const ggml_tensor * dst) { const int64_t ne10 = src1->ne[0]; const int64_t ne11 = src1->ne[1]; const int64_t ne12 = src1->ne[2]; + if (ne12 != 1) return false; if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) return false; if (ne10 != src0->ne[0] || ne10 % QK8_1 != 0) return false; if (!ggml_is_contiguous(src1)) return false; const int64_t n_ids_per_group = ids->ne[0]; + if (ids->type != GGML_TYPE_I32 || ids->nb[0] != sizeof(int32_t)) return false; if (ids->ne[1] != 1) return false; if (ne11 != 1 && ne11 != n_ids_per_group) return false; + return true; +} + +// returns true when ggml_sycl_mul_mat_id takes the fused MMVQ path +static bool ggml_sycl_mul_mat_id_should_use_mmvq(const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + return ggml_sycl_mul_mat_id_mmvq_eligible(src0, dst->src[1], dst->src[2], dst) && + ggml_sycl_mul_mat_vec_q_id_supports_type(src0->type); +} + +// Fused MoE TG fast path. Returns false to fall back to the per-expert loop below. +static bool ggml_sycl_mul_mat_id_mmvq_fused( + ggml_backend_sycl_context & ctx, const ggml_tensor * src0, + const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) +{ + if (!ggml_sycl_mul_mat_id_should_use_mmvq(dst)) return false; + + const int64_t ne10 = src1->ne[0]; + const int64_t ne11 = src1->ne[1]; + const int64_t n_ids_per_group = ids->ne[0]; const queue_ptr stream = ctx.stream(); const int src1_padded_cols = GGML_PAD((int) ne10, MATRIX_ROW_PADDING); const int n_experts_used = (int) n_ids_per_group; const int nrows = (int) src0->ne[1]; - // Lazily reorder the (Q4_K) expert weights into a per-expert SoA layout, then run the reorder + // Lazily reorder supported expert weights into a per-expert SoA layout, then run the reorder // GEMV. Placed after the bail checks so a non-dispatchable op does not pay the reorder cost. opt_for_reorder_id(&ctx, src0); const ggml_tensor_extra_gpu * src0_extra = @@ -5211,14 +5670,18 @@ static void mmid_counting_sort_rows( int64_t n_ids, int64_t n_as, int64_t n_routed_rows, std::vector<int64_t> & expert_counts, std::vector<int64_t> & expert_row_offsets, - std::vector<mmid_row_mapping> & routed_row_src) { + std::vector<mmid_row_mapping> & routed_row_src, + std::vector<mmid_row_mapping> & skipped_row_dst) { // frequencies: how many routed rows each expert "owns" expert_counts.assign(n_as, 0); for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) { for (int64_t id = 0; id < n_ids; id++) { const int32_t row_id_i = *(const int32_t *) (ids_host + iid1*ids->nb[1] + id*ids->nb[0]); - GGML_ASSERT(row_id_i >= 0 && row_id_i < n_as); + GGML_ASSERT(row_id_i == -1 || (row_id_i >= 0 && row_id_i < n_as)); + if (row_id_i == -1) { + continue; + } expert_counts[row_id_i]++; } } @@ -5231,10 +5694,17 @@ static void mmid_counting_sort_rows( std::vector<int64_t> expert_row_next = expert_row_offsets; routed_row_src.resize(n_routed_rows); + skipped_row_dst.clear(); for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) { for (int64_t id = 0; id < n_ids; id++) { const int32_t row_id_i = *(const int32_t *) (ids_host + iid1*ids->nb[1] + id*ids->nb[0]); - GGML_ASSERT(row_id_i >= 0 && row_id_i < n_as); + GGML_ASSERT(row_id_i == -1 || (row_id_i >= 0 && row_id_i < n_as)); + + // a skipped slot has no expert, its dst row is zeroed instead + if (row_id_i == -1) { + skipped_row_dst.push_back({(int32_t) id, (int32_t) iid1}); + continue; + } // find and validate the next free row for a given expert (row_id_i) const int64_t routed_row = expert_row_next[row_id_i]++; @@ -5243,6 +5713,19 @@ static void mmid_counting_sort_rows( routed_row_src[routed_row] = {(int32_t) id, (int32_t) iid1}; } } + routed_row_src.resize(expert_row_offsets[n_as]); +} + +// true when the device MoE schedule may handle this MUL_MAT_ID: enabled, prefill, and an +// f32/i32 layout it accepts; ggml_sycl_grouped_dequant_gemm_device decides type and width +static bool ggml_sycl_mul_mat_id_device_sched_eligible(const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * ids = dst->src[2]; + return (g_ggml_sycl_mmid_sched & GGML_SYCL_MMID_SCHED_DEVICE) && src1->ne[2] != 1 && ggml_is_contiguous(src0) && + ids->type == GGML_TYPE_I32 && ids->nb[0] == sizeof(int32_t) && + src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && + src1->nb[1] == sizeof(float)*src1->ne[0] && dst->nb[1] == sizeof(float)*dst->ne[0]; } static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, @@ -5265,6 +5748,45 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, return; } } + GGML_ASSERT(!ggml_sycl_mul_mat_id_should_use_mmvq(dst)); + + // The per-expert loop below reads the experts in whatever layout they have: reorder MXFP4 here as well, so prompt processing does not depend on a single-token decode having run first. + if (src0->type == GGML_TYPE_MXFP4) { + opt_for_reorder_id(&ctx, src0); + } + + if (ggml_sycl_mul_mat_id_device_sched_eligible(dst)) { + if (g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 && + ggml_sycl_xmx_gather_type_enabled(src0->type)) { + opt_for_reorder_id(&ctx, src0); + } + const bool reordered = src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + const ggml_sycl_grouped_gemm_rows src1_rows = { (char *) src1->data, nullptr, ne11, nb11, nb12 }; + const ggml_sycl_grouped_gemm_rows dst_rows = { (char *) dst->data, nullptr, ne1, nb1, nb2 }; + if (ggml_sycl_grouped_dequant_gemm_device(src0->type, reordered, src0->data, nb02, src1_rows, + ggml_sycl_src1_prec(dst), dst_rows, (const int32_t *) ids->data, ids->nb[1], n_as, n_ids, + ids->ne[1], ne01, ne10, ctx.pool(), stream)) { + // the device schedule drops skipped slots (id == -1), so their dst rows are zeroed here + const char * ids_data = (const char *) ids->data; + const size_t ids_nb1 = ids->nb[1]; + char * dst_data = (char *) dst->data; + const int nth = (int) std::min<int64_t>(ne0, 256); + stream->parallel_for( + sycl::nd_range<2>(sycl::range<2>(ids->ne[1], n_ids * nth), sycl::range<2>(1, nth)), + [=](sycl::nd_item<2> item) { + const int64_t iid1 = item.get_group(0); + const int64_t id = item.get_group(1); + if (*(const int32_t *) (ids_data + iid1*ids_nb1 + id*sizeof(int32_t)) != -1) { + return; + } + float * row = (float *) (dst_data + id*nb1 + iid1*nb2); + for (int64_t j = item.get_local_id(1); j < ne0; j += nth) { + row[j] = 0.0f; + } + }); + return; + } + } std::vector<char> ids_host(ggml_nbytes(ids)); const char * ids_dev = (const char *) ids->data; @@ -5272,7 +5794,9 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, SYCL_CHECK(CHECK_TRY_ERROR( stream->memcpy(ids_host.data(), ids_dev, ggml_nbytes(ids)))); - // also ensures ctx.mmid_row_mapping_host is drained before we use it again + // also ensures ctx.mmid_row_mapping_host and ctx.mmid_tile_schedule_host are drained before we + // refill them: the grouped GEMM enqueues an async copy out of the tile schedule, so removing + // this wait would let the next node overwrite a buffer the device is still reading SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); ggml_tensor src0_row = *src0; @@ -5302,7 +5826,7 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) { for (int64_t id = 0; id < n_ids; id++) { const int32_t i02 = *(const int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]); - GGML_ASSERT(i02 >= 0 && i02 < n_as); + GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as)); const int64_t i11 = id % ne11; const int64_t i12 = iid1; @@ -5310,6 +5834,12 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, const int64_t i1 = id; const int64_t i2 = i12; + if (i02 == -1) { // skipped slot + SYCL_CHECK(CHECK_TRY_ERROR( + stream->memset(dst_original + i1*nb1 + i2*nb2, 0, ne0*sizeof(float)))); + continue; + } + src0_row.data = src0_original + i02*nb02; src1_row.data = src1_original + i11*nb11 + i12*nb12; dst_row.data = dst_original + i1*nb1 + i2*nb2; @@ -5331,20 +5861,43 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, std::vector<int64_t> expert_row_offsets; // the sources (slot/token pairs) of contiguous rows to guide k_copy_src1_to_contiguous std::vector<mmid_row_mapping> & routed_row_src = ctx.mmid_row_mapping_host; + // the slot/token pairs of the skipped slots, whose dst rows are zeroed + std::vector<mmid_row_mapping> skipped_row_dst; mmid_counting_sort_rows(ids, ids_host.data(), n_ids, n_as, n_routed_rows, - expert_row_counts, expert_row_offsets, routed_row_src); + expert_row_counts, expert_row_offsets, routed_row_src, skipped_row_dst); + + const int64_t n_valid_rows = (int64_t) routed_row_src.size(); ggml_sycl_pool_alloc<mmid_row_mapping> dev_row_mapping(ctx.pool(), n_routed_rows); SYCL_CHECK(CHECK_TRY_ERROR( - stream->memcpy(dev_row_mapping.get(), routed_row_src.data(), n_routed_rows*sizeof(mmid_row_mapping)))); + stream->memcpy(dev_row_mapping.get(), routed_row_src.data(), n_valid_rows*sizeof(mmid_row_mapping)))); const unsigned int max_work_group_size = ggml_sycl_info().max_work_group_sizes[ctx.device]; assert(max_work_group_size % (WARP_SIZE * WARP_SIZE) == 0); - { + if (!skipped_row_dst.empty()) { + ggml_sycl_pool_alloc<mmid_row_mapping> dev_skipped(ctx.pool(), skipped_row_dst.size()); + SYCL_CHECK(CHECK_TRY_ERROR( + stream->memcpy(dev_skipped.get(), skipped_row_dst.data(), skipped_row_dst.size()*sizeof(mmid_row_mapping)))); + + sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne0, max_work_group_size)); + sycl::range<3> grid_dims(1, 1, skipped_row_dst.size()); + stream->submit([&](sycl::handler &cgh) { + mmid_row_mapping *__restrict dev_skipped_get = dev_skipped.get(); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_zero_dst_rows(dst_original, dev_skipped_get, ne0, nb1, nb2, item_ct1); + }); + }); + SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); + } + + if (n_valid_rows > 0) { sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne10, max_work_group_size)); - sycl::range<3> grid_dims(1, 1, n_routed_rows); + sycl::range<3> grid_dims(1, 1, n_valid_rows); stream->submit([&](sycl::handler &cgh) { char *__restrict src1_contiguous_get = src1_contiguous.get(); @@ -5363,7 +5916,25 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, }); } - for (int64_t i02 = 0; i02 < n_as; i02++) { + bool grouped = false; + if (ggml_is_contiguous(src0) && src1->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F32 && nb11 == sizeof(float)*ne10 && nb1 == sizeof(float)*ne0) { + // the grouped GEMM reads the reorder (SoA) layout faster, and the first decode step installs it + // anyway: install it here already, so prefill does not depend on whether a decode ran before + if (g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 && + ggml_sycl_xmx_gather_type_enabled(src0->type)) { + opt_for_reorder_id(&ctx, src0); + } + const bool src0_reordered = + src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + grouped = ggml_sycl_grouped_dequant_gemm(src0->type, src0_reordered, src0_original, nb02, + (const float *) src1_contiguous.get(), ggml_sycl_src1_prec(dst), + (float *) dst_contiguous.get(), + expert_row_offsets.data(), n_as, ne01, ne10, n_routed_rows, + ctx.mmid_tile_schedule_host, ctx.pool(), stream); + } + + for (int64_t i02 = 0; i02 < n_as && !grouped; i02++) { const int64_t num_src1_rows = expert_row_counts[i02]; if (num_src1_rows == 0) { @@ -5392,9 +5963,9 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, ggml_sycl_mul_mat(ctx, &src0_row, &src1_row, &dst_row); } - { + if (n_valid_rows > 0) { sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne0, max_work_group_size)); - sycl::range<3> grid_dims(1, 1, n_routed_rows); + sycl::range<3> grid_dims(1, 1, n_valid_rows); stream->submit([&](sycl::handler &cgh) { const char *__restrict dst_contiguous_get = dst_contiguous.get(); @@ -5424,11 +5995,6 @@ static void ggml_sycl_scale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) ggml_sycl_op_scale(ctx, dst); } -static void ggml_sycl_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); - ggml_sycl_op_diag_mask_inf(ctx, dst); -} - static void ggml_sycl_pool2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); ggml_sycl_op_pool2d(ctx, dst); @@ -5491,11 +6057,11 @@ static void ggml_sycl_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dst) static void ggml_sycl_set_main_device(const int main_device) try { - if (dpct::get_current_device_id() == static_cast<unsigned int> (main_device)) { + if (get_current_device_id() == main_device) { return; } check_allow_gpu_index(main_device); - dpct::select_device(main_device); + SYCL_CHECK(ggml_sycl_set_device(main_device)); if (g_ggml_sycl_debug) { dpct::device_info prop; @@ -5572,7 +6138,6 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg ggml_sycl_dup(ctx, dst); break; case GGML_OP_ADD: - case GGML_OP_ADD1: // TODO: more efficient implementation ggml_sycl_add(ctx, dst); break; case GGML_OP_ADD_ID: @@ -5775,9 +6340,6 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_TRI: ggml_sycl_op_tri(ctx, dst); break; - case GGML_OP_DIAG_MASK_INF: - ggml_sycl_diag_mask_inf(ctx, dst); - break; case GGML_OP_SOFT_MAX: ggml_sycl_op_soft_max(ctx, dst); break; @@ -6111,8 +6673,6 @@ static int ggml_sycl_try_gdn_cache_fusion(const ggml_cgraph * cgraph, int node_i } static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { - ggml_sycl_set_main_device(sycl_ctx->device); - for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; if (ggml_sycl_is_view_or_noop(node)) { @@ -6177,6 +6737,16 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc i++; continue; } + // ADD(bias) + UNARY + MUL(scale) with both broadcast over dim 0, the form the branch + // above cannot take; ggml_get_unary_op() asserts, so check the op first. + if (node->op == GGML_OP_ADD && i + 2 < cgraph->n_nodes && + cgraph->nodes[i + 1]->op == GGML_OP_UNARY && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_MUL }, + { ggml_get_unary_op(cgraph->nodes[i + 1]) })) { + ggml_sycl_op_add_unary_mul_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); + i += 2; + continue; + } // Batch consecutive independent same-shape F32 L2_NORM siblings (the GDN q/k // norms) into one launch; sources are strided views of the fused qkv buffer, so @@ -6203,7 +6773,7 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } - if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_mmvq_fused(*sycl_ctx, cgraph, i)) { + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_fused(*sycl_ctx, cgraph, i)) { i += 2; continue; } @@ -6217,42 +6787,236 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc } #ifdef GGML_SYCL_GRAPH -static bool check_graph_compatibility(ggml_cgraph * cgraph) { - if (ggml_sycl_info().device_count > 1) { +static const void * ggml_sycl_graph_get_key(ggml_cgraph * cgraph) { + return cgraph->nodes[0]; +} + +static bool ggml_sycl_graph_update_required(ggml_sycl_graph * graph, ggml_cgraph * cgraph) { + bool res = false; + + if (cgraph->uid != 0 && cgraph->uid == graph->uid) { + GGML_SYCL_DEBUG("[SYCL-GRAPH] graph id %" PRIu64 " reused\n", cgraph->uid); + GGML_ASSERT((int) graph->node_props.size() == cgraph->n_nodes); + return false; + } + + graph->uid = cgraph->uid; + + if ((int) graph->node_props.size() != cgraph->n_nodes) { + res = true; + graph->node_props.resize(cgraph->n_nodes); + } + + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_sycl_graph::node_properties prop = {}; + memcpy(&prop.node, cgraph->nodes[i], sizeof(ggml_tensor)); + + for (int j = 0; j < GGML_MAX_SRC; ++j) { + if (cgraph->nodes[i]->src[j]) { + prop.node_src_data_ptrs[j] = cgraph->nodes[i]->src[j]->data; + memcpy(prop.node_src_ne[j], cgraph->nodes[i]->src[j]->ne, sizeof(prop.node_src_ne[j])); + memcpy(prop.node_src_nb[j], cgraph->nodes[i]->src[j]->nb, sizeof(prop.node_src_nb[j])); + } + } + + if (res || memcmp(&graph->node_props[i], &prop, sizeof(prop)) != 0) { + graph->node_props[i] = prop; + res = true; + } + } + + return res; +} + +// Reports if ggml_sycl_mul_mat() gives this node to oneDNN. oneDNN chains the submission on events +// made before recording started, which SYCL graphs do not allow. +static bool mul_mat_uses_onednn(int device, ggml_tensor * dst) { +#if GGML_SYCL_DNNL + if (!g_ggml_sycl_enable_dnn) { + return false; + } + + ggml_tensor * src0 = dst->src[0]; + ggml_tensor * src1 = dst->src[1]; + + // the branch order below follows the dispatch in ggml_sycl_mul_mat(); the F16 branches end in + // a vector kernel or the batched oneMKL path, neither of which uses oneDNN + if (!ggml_backend_buffer_is_sycl_split(src0->buffer) && src0->type == GGML_TYPE_F16) { + if (ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { + return false; + } + if (!ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1 && src1->ne[3] == 1) { + return false; + } + if (!ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2] * src1->ne[3] > 1) { + return false; + } + } + + // ggml_sycl_op_mul_mat_sycl() is the last resort of the dispatch and the only user of oneDNN. + // The reorder decision that ggml_sycl_mul_mat() makes between DMMV and MMVQ is left out here: + // neither uses a library. + if (can_use_dequantize_mul_mat_vec(src0, src1, dst) || can_use_mul_mat_vec_q(device, src0, src1, dst) || + can_use_mul_mat_q(src0, src1, dst)) { + return false; + } + + // the branch order below follows ggml_sycl_op_mul_mat_sycl(); graphs are single device only, so + // row_diff == src0->ne[1] and src1_ncols == src1->ne[1] there +#if defined(GGML_SYCL_HAS_BF16) + if (src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0)) { + return true; + } +#endif +#ifdef GGML_SYCL_F16 + if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && + ggml_prec_get_acc(dst) == GGML_PREC_UNDEFINED) { + return true; + } +#endif + // the F32 branch only uses oneDNN for large GEMMs and calls oneMKL directly otherwise + const int64_t gemm_flops = src0->ne[1] * src1->ne[1] * src0->ne[0]; + return gemm_flops >= SYCL_F32_DNNL_MIN_GEMM_FLOPS; +#else + GGML_UNUSED(device); + GGML_UNUSED(dst); + return false; +#endif +} + +// returns true when ggml_sycl_mul_mat_id takes the fallback path that requires stream synchronization: +// neither the fused MMVQ path nor the device-scheduled grouped GEMM (GGML_SYCL_MMID_SCHED) handles dst +static bool ggml_sycl_mul_mat_id_needs_sync(ggml_backend_sycl_context * ctx, const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * ids = dst->src[2]; + if (!src0 || !src1 || !ids || ggml_backend_buffer_is_sycl_split(src0->buffer)) { + return true; + } + const bool reordered = src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + if (ggml_sycl_mul_mat_id_mmvq_eligible(src0, src1, ids, dst)) { + return reordered ? !ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(src0->type) : + !ggml_sycl_mul_mat_vec_q_id_supports_type(src0->type); + } + return !ggml_sycl_mul_mat_id_device_sched_eligible(dst) || + !ggml_sycl_grouped_dequant_gemm_device_supported(src0->type, reordered, ggml_sycl_src1_prec(dst), + src0->ne[2], ids->ne[0], ids->ne[1], src0->ne[1], src0->ne[0], ctx->stream()); +} + +// mirrors the paths in ggml_sycl_mul_mat_id() that call opt_for_reorder_id() +static bool mul_mat_id_reorders_src0(const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + if (ggml_sycl_mul_mat_id_should_use_mmvq(dst) || src0->type == GGML_TYPE_MXFP4) { + return true; + } + return ggml_sycl_mul_mat_id_device_sched_eligible(dst) && + g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 && + ggml_sycl_xmx_gather_type_enabled(src0->type); +} + +// true when the node and its SYCL sources live in this device's buffers (no split or other-device buffers) +static bool graph_node_is_in_local_device(ggml_backend_buffer_type_t device_buft, const ggml_tensor * node) { + if (!node->buffer || node->buffer->buft != device_buft) { + return false; + } + for (int j = 0; j < GGML_MAX_SRC; ++j) { + const ggml_tensor * src = node->src[j]; + if (!src || !src->buffer) { + continue; + } + if (ggml_backend_buffer_is_sycl_split(src->buffer) || + (ggml_backend_buffer_is_sycl(src->buffer) && src->buffer->buft != device_buft)) { + return false; + } + } + return true; +} + +static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgraph * cgraph) { + if (ggml_sycl_info().active_device_count > 1) { // A sycl_ex::command_graph object can only be created for a single device - GGML_LOG_INFO("%s: disabling SYCL graphs due to multiple devices\n", __func__); + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to multiple devices\n", __func__); return false; } + const ggml_backend_buffer_type_t device_buft = ggml_backend_sycl_buffer_type(ctx->device); + for (int i = 0; i < cgraph->n_nodes; i++) { - const ggml_op node_op = cgraph->nodes[i]->op; + ggml_tensor * node = cgraph->nodes[i]; + + // skip the nodes that ggml_backend_sycl_graph_compute_impl() does not run + if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0 || ggml_sycl_is_view_or_noop(node)) { + continue; + } + + const ggml_op node_op = node->op; + + if (!graph_node_is_in_local_device(device_buft, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to node %s not being device local\n", __func__, + ggml_op_name(node_op)); + return false; + } + + // Reorder expert weights before recording, so the reorder state that + // ggml_sycl_mul_mat_id_needs_sync() sees is the one the recorded graph runs with. + if (node_op == GGML_OP_MUL_MAT_ID && mul_mat_id_reorders_src0(node)) { + opt_for_reorder_id(ctx, node->src[0]); + } + switch (node_op) { default: break; - case GGML_OP_CONCAT: - // ggml_sycl_op_concat() does a blocking host wait after memcpy operations, - // but wait() can't be called on the events returned by a queue recording - // to a graph. - [[fallthrough]]; case GGML_OP_MUL_MAT_ID: - // ggml_sycl_mul_mat_id() does a blocking host wait on the sycl queue after - // submitting a memcpy operation, but wait() can't be called on a queue that - // is recording to a graph. - GGML_LOG_INFO("%s: disabling SYCL graphs due to unsupported node type %s\n", __func__, - ggml_op_name(node_op)); - return false; + // The ggml_sycl_mul_mat_id() fallback path does a blocking host wait on the sycl + // queue after submitting a memcpy operation, but wait() can't be called on a queue + // that is recording to a graph. + if (ggml_sycl_mul_mat_id_needs_sync(ctx, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to host wait in %s\n", __func__, + ggml_op_name(node_op)); + return false; + } + break; + case GGML_OP_FLASH_ATTN_EXT: + if (ggml_sycl_flash_attn_ext_needs_sync(ctx->device, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to host wait in %s\n", __func__, + ggml_op_name(node_op)); + return false; + } +#ifndef GGML_SYCL_GRAPH_DNNL_SUPPORTED + if (ggml_sycl_flash_attn_ext_uses_onednn(ctx->device, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to %s using oneDNN\n", __func__, + ggml_op_name(node_op)); + return false; + } +#endif + break; case GGML_OP_MUL_MAT: // We cannot use graphs with ggml_sycl_mul_mat() when SYCL async memory allocation extensions are not available, // as SYCL malloc / free and host wait calls are not supported when recording to a graph which are all present // in reordering. if (!g_ggml_sycl_use_async_mem_op) { - GGML_LOG_INFO( + GGML_LOG_DEBUG( "%s: disabling SYCL graphs due to unsupported node type when using a compiler without the " "oneAPI async memory allocation extension " "%s\n", __func__, ggml_op_name(node_op)); return false; } +#ifndef GGML_SYCL_GRAPH_DNNL_SUPPORTED + if (mul_mat_uses_onednn(ctx->device, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to %s using oneDNN\n", __func__, + ggml_op_name(node_op)); + return false; + } +#endif + break; + case GGML_OP_OUT_PROD: + case GGML_OP_CONV_3D: + case GGML_OP_SOLVE_TRI: + // these call oneMKL and wait on the host + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to unsupported node type %s\n", __func__, + ggml_op_name(node_op)); + return false; } } return true; @@ -6262,44 +7026,102 @@ static bool check_graph_compatibility(ggml_cgraph * cgraph) { static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { auto * sycl_ctx = static_cast<ggml_backend_sycl_context *>(backend->context); + // set the device on every call: a graph replay does not go through + // ggml_backend_sycl_graph_compute_impl() + ggml_sycl_set_main_device(sycl_ctx->device); + #ifdef GGML_SYCL_GRAPH - bool use_sycl_graph = false; + bool use_sycl_graph = false; + ggml_sycl_graph * graph = nullptr; + if (g_ggml_sycl_enable_graph) { - use_sycl_graph = check_graph_compatibility(cgraph); + if (!ggml_sycl_info().devices[sycl_ctx->device].graph_support) { + // the device does not change, so report this once + if (sycl_ctx->graph_compatible != false) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to unsupported graph feature on device\n", __func__); + GGML_LOG_INFO("%s: disabling SYCL graphs\n", __func__); + sycl_ctx->graph_compatible = false; + } + } else { + graph = sycl_ctx->sycl_graph(ggml_sycl_graph_get_key(cgraph)); + + // The nodes of a cgraph do not change while its uid does not, so the compatibility scan + // runs again only when the uid changes. A uid of 0 means it is not set. + if (cgraph->uid == 0 || cgraph->uid != graph->compatible_uid) { + graph->compatible_uid = cgraph->uid; + graph->compatible = check_graph_compatibility(sycl_ctx, cgraph); + + if (sycl_ctx->graph_compatible != graph->compatible) { + if (graph->compatible) { + GGML_LOG_INFO("%s: enabling SYCL graphs\n", __func__); + } else { + GGML_LOG_INFO("%s: disabling SYCL graphs\n", __func__); + } + sycl_ctx->graph_compatible = graph->compatible; + } + } + + use_sycl_graph = graph->compatible; + } } if (use_sycl_graph) { - const bool graph_support = dpct::get_device(sycl_ctx->device).has(sycl::aspect::ext_oneapi_limited_graph); - if (!graph_support) { - GGML_SYCL_DEBUG("[SYCL-GRAPH] can not use graphs on device:%d\n", sycl_ctx->device); - ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); - return GGML_STATUS_SUCCESS; + const bool properties_changed = ggml_sycl_graph_update_required(graph, cgraph); + bool replay_graph = false; + bool sycl_graph_update_required = false; + + if (!graph->warmup_complete) { + // Warmup: need at least 2 calls with no property change on the 2nd call. + if (!properties_changed) { + graph->warmup_complete = true; + replay_graph = true; + sycl_graph_update_required = true; + GGML_SYCL_DEBUG("[SYCL-GRAPH] warmup complete\n"); + } + } else { + // Post-warmup: replay graph while properties remain stable. + if (properties_changed) { + graph->warmup_complete = false; + GGML_SYCL_DEBUG("[SYCL-GRAPH] warmup reset\n"); + } else { + replay_graph = true; + sycl_graph_update_required = graph->exec_graph == nullptr; + } } - sycl_ex::command_graph model_sycl_graph(*(sycl_ctx->stream()), {sycl_ex::property::graph::assume_buffer_outlives_graph{}}); + if (replay_graph) { + if (sycl_graph_update_required) { + const bool graph_update_support = ggml_sycl_info().devices[sycl_ctx->device].graph_update_support; - model_sycl_graph.begin_recording(*(sycl_ctx->stream())); - ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); - model_sycl_graph.end_recording(); - - const bool graph_update_support = dpct::get_device(sycl_ctx->device).has(sycl::aspect::ext_oneapi_graph); - if (!sycl_ctx->exec_graph || !graph_update_support) { - auto exec_graph = graph_update_support ? model_sycl_graph.finalize(sycl_ex::property::graph::updatable{}) : - model_sycl_graph.finalize(); - sycl_ctx->exec_graph = std::make_unique< - sycl_ex::command_graph<sycl_ex::graph_state::executable>>(exec_graph); - } else { - try { - sycl_ctx->exec_graph->update(model_sycl_graph); - GGML_SYCL_DEBUG("[SYCL-GRAPH] update success\n"); - } catch (sycl::exception const & e) { - GGML_SYCL_DEBUG("[SYCL-GRAPH] Exception when updating graph, %s\n", e.what()); - auto exec_graph = model_sycl_graph.finalize({sycl_ex::property::graph::updatable{}}); - sycl_ctx->exec_graph = std::make_unique< - sycl_ex::command_graph<sycl_ex::graph_state::executable>>(exec_graph); + sycl_ex::command_graph model_sycl_graph(*(sycl_ctx->stream()), {sycl_ex::property::graph::assume_buffer_outlives_graph{}}); + + model_sycl_graph.begin_recording(*(sycl_ctx->stream())); + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + model_sycl_graph.end_recording(); + + if (!graph->exec_graph || !graph_update_support) { + auto exec_graph = graph_update_support ? model_sycl_graph.finalize(sycl_ex::property::graph::updatable{}) : + model_sycl_graph.finalize(); + graph->exec_graph = std::make_unique< + sycl_ex::command_graph<sycl_ex::graph_state::executable>>(exec_graph); + } else { + try { + graph->exec_graph->update(model_sycl_graph); + GGML_SYCL_DEBUG("[SYCL-GRAPH] update success\n"); + } catch (sycl::exception const & e) { + // the executable graph cannot be updated due to violated constraints, + // so finalize the recorded graph again + GGML_SYCL_DEBUG("[SYCL-GRAPH] Exception when updating graph, %s\n", e.what()); + auto exec_graph = model_sycl_graph.finalize({sycl_ex::property::graph::updatable{}}); + graph->exec_graph = std::make_unique< + sycl_ex::command_graph<sycl_ex::graph_state::executable>>(exec_graph); + } + } } - } - sycl_ctx->stream()->ext_oneapi_graph(*(sycl_ctx->exec_graph)); + sycl_ctx->stream()->ext_oneapi_graph(*(graph->exec_graph)); + } else { + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + } } else #endif { @@ -6468,6 +7290,11 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons return false; } case GGML_OP_CONV_2D: + // WHCN: contiguous [KW,KH,IC,OC] kernel and [W,H,C,N] input + // CWHN: channel-fastest input (C fastest) and kernel with OC fastest + return (ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) || + (op->src[1]->nb[2] == ggml_type_size(op->src[1]->type) && + op->src[0]->nb[3] == ggml_type_size(op->src[0]->type)); case GGML_OP_CONV_2D_DW: case GGML_OP_CONV_TRANSPOSE_2D: return true; @@ -6509,11 +7336,12 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: case GGML_GLU_OP_SWIGLU_CLAMP: - return ggml_is_contiguous_1(op->src[0]); + // the fused GLU kernels only have f32/f16 instantiations + return ggml_is_contiguous_1(op->src[0]) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); default: return false; } - break; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: { @@ -6542,6 +7370,11 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons return false; } + if (op->op == GGML_OP_MUL_MAT && + (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32)) { + return false; + } + return true; } case GGML_OP_OUT_PROD: @@ -6606,7 +7439,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); return res; } - break; case GGML_OP_DSV4_HC_PRE: return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; @@ -6748,7 +7580,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: case GGML_OP_ADD: - case GGML_OP_ADD1: case GGML_OP_ADD_ID: case GGML_OP_SUB: case GGML_OP_COUNT_EQUAL: @@ -6772,7 +7603,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_RMS_NORM_BACK: return ggml_is_contiguous(op->src[0]); case GGML_OP_SCALE: - return true; + // the scale kernel is f32 only, and reads src0 flat + return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); case GGML_OP_CONT: return true; case GGML_OP_TRI: @@ -6782,8 +7614,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons op->type == GGML_TYPE_F32 && ggml_is_contiguous(src0); } - case GGML_OP_DIAG_MASK_INF: - return true; case GGML_OP_SOFT_MAX: return true; case GGML_OP_SOFT_MAX_BACK: { @@ -6805,12 +7635,19 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons #endif ) && op->src[0]->type == op->type; - case GGML_OP_CONV_3D: + case GGML_OP_CONV_3D: { + const int32_t * opts = (const int32_t *) op->op_params; + // zero kernel volume, channels or output makes the im2col GEMM degenerate (k == 0 or m == 0) + // and MKL BLAS rejects it; let those fall back to the CPU backend + const int64_t knl_n_total = op->src[0]->ne[0] * op->src[0]->ne[1] * op->src[0]->ne[2] * opts[9]; + const int64_t patch_total = op->ne[0] * op->ne[1] * op->ne[2] * opts[10]; return op->type == GGML_TYPE_F32 && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && - ggml_is_contiguous(op->src[1]); + ggml_is_contiguous(op->src[1]) && + knl_n_total > 0 && patch_total > 0; + } case GGML_OP_SUM: case GGML_OP_SUM_ROWS: case GGML_OP_MEAN: @@ -6870,7 +7707,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_SOLVE_TRI: return op->src[0]->ne[0] <= SYCL_SOLVE_TRI_MAX_N && op->src[1]->ne[0] <= SYCL_SOLVE_TRI_MAX_K; case GGML_OP_FLASH_ATTN_EXT: - return ggml_sycl_flash_attn_ext_supported(device, op); + // K/V row lists (ggml_flash_attn_ext_rows) are not supported + return op->src[5] == nullptr && ggml_sycl_flash_attn_ext_supported(device, op); default: return false; } @@ -7046,9 +7884,15 @@ struct ggml_backend_sycl_comm_context { // pinned host staging per device context, reused across calls (queues are in-order) uint8_t * host_out[2] = { nullptr, nullptr }; - uint8_t * host_in[2] = { nullptr, nullptr }; + uint8_t * host_in[2] = { nullptr, nullptr }; // only used by the 4-buffer path (separate SYCL contexts) size_t host_bytes = 0; + // SYCL_ENABLE_DEFAULT_CONTEXTS=1 sets comm_exchange_fast on Linux - PR #30956 + using exchange_fn_t = void (*)(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1, + void * dst0, const void * src1, void * dst1, const void * src0, + size_t nbytes); + exchange_fn_t comm_exchange = nullptr; // Moves same context check out of hot path; tests only once + void free_host() { for (int i = 0; i < 2; ++i) { const sycl::context sctx = ((ggml_backend_sycl_context *) backends[i]->context)->stream()->get_context(); @@ -7059,6 +7903,41 @@ struct ggml_backend_sycl_comm_context { } }; +// dst0 <- src1 and dst1 <- src0 through pinned host memory. +static void ggml_sycl_comm_exchange_fast(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1, + void * dst0, const void * src1, void * dst1, const void * src0, + size_t nbytes) { + + // Increases Linux tensor split prompt eval speeds on par with layer split speeds + // Reallocate host buffers if they are too small + if (comm->host_bytes < nbytes) { + q0->wait(); + q1->wait(); + comm->free_host(); + comm->host_out[0] = sycl::malloc_host<uint8_t>(nbytes, q0->get_context()); + comm->host_out[1] = sycl::malloc_host<uint8_t>(nbytes, q1->get_context()); + + GGML_ASSERT(comm->host_out[0] && comm->host_out[1]); + comm->host_bytes = nbytes; + } + + // Step 1: Concurrently copy from GPUs into host buffers (Asynchronous) + sycl::event e_d2h_0 = q0->memcpy(comm->host_out[0], src0, nbytes); + sycl::event e_d2h_1 = q1->memcpy(comm->host_out[1], src1, nbytes); + + // Step 2: Asynchronous Barrier Interlocking (NO HOST BLOCKING), wait() causes MTP conflicts + q0->ext_oneapi_submit_barrier({e_d2h_1}); // q0 waits for q1's device-to-host copy + q1->ext_oneapi_submit_barrier({e_d2h_0}); // q1 waits for q0's device-to-host copy + + // Step 3: Cross-upload directly from the opposing host buffer. + sycl::event e_h2d_0 = q0->memcpy(dst0, comm->host_out[1], nbytes); // q0 pulls from host_out[1] + sycl::event e_h2d_1 = q1->memcpy(dst1, comm->host_out[0], nbytes); // q1 pulls from host_out[0] + + // Step 4: Finalize the exchange before returning to host code + e_h2d_0.wait(); + e_h2d_1.wait(); +} + // dst0 <- src1 and dst1 <- src0 through pinned host memory, with a single host wait static void ggml_sycl_comm_exchange(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1, void * dst0, const void * src1, void * dst1, const void * src0, @@ -7104,6 +7983,15 @@ void * ggml_backend_sycl_comm_init(ggml_backend_t * backends, size_t n_backends) auto * sctx1 = (ggml_backend_sycl_context *) backends[1]->context; ctx->buf0 = std::make_unique<ggml_sycl_pool_alloc<uint8_t>>(sctx0->pool()); ctx->buf1 = std::make_unique<ggml_sycl_pool_alloc<uint8_t>>(sctx1->pool()); + +#if defined(_WIN32) || defined(_WIN64) // Not supported on Windows + ctx->comm_exchange = ggml_sycl_comm_exchange; +#else + // SYCL_ENABLE_DEFAULT_CONTEXTS=1 sets comm_exchange_fast on Linux - PR #30956 + ctx->comm_exchange = (sctx0->stream()->get_context() == sctx1->stream()->get_context()) + ? ggml_sycl_comm_exchange_fast + : ggml_sycl_comm_exchange; +#endif return ctx; } catch (const sycl::exception &) { return nullptr; } @@ -7187,7 +8075,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens sycl::half * f16_tmp0 = (sycl::half *) buf0; sycl::half * f16_tmp1 = (sycl::half *) buf1; - ggml_sycl_comm_exchange(comm_ctx, q0, q1, f16_tmp0, tensors[1]->data, f16_tmp1, tensors[0]->data, nbytes); + comm_ctx->comm_exchange(comm_ctx, q0, q1, f16_tmp0, tensors[1]->data, f16_tmp1, tensors[0]->data, nbytes); q0->submit([&](sycl::handler & h) { h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { @@ -7217,7 +8105,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens float * tmp1 = (float *) buf1; // separate SYCL contexts: a raw peer-USM q->memcpy is a silent no-op, so stage through host memory - ggml_sycl_comm_exchange(comm_ctx, q0, q1, tmp0, tensors[1]->data, tmp1, tensors[0]->data, nbytes); + comm_ctx->comm_exchange(comm_ctx, q0, q1, tmp0, tensors[1]->data, tmp1, tensors[0]->data, nbytes); q0->submit([&](sycl::handler & h) { h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { @@ -7251,7 +8139,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens // Phase B: exchange the compressed bytes (in-order queues keep this after the compress kernels). const size_t bf16_bytes = nelem * sizeof(uint16_t); - ggml_sycl_comm_exchange(comm_ctx, q0, q1, inbox0, outbox1, inbox1, outbox0, bf16_bytes); + comm_ctx->comm_exchange(comm_ctx, q0, q1, inbox0, outbox1, inbox1, outbox0, bf16_bytes); // Phase C: decompress + add into local FP32 partial. q0->submit([&](sycl::handler & h) { @@ -7354,8 +8242,19 @@ ggml_backend_reg_t ggml_backend_sycl_reg() { return ® } +static void set_device_active(int device) { + ggml_sycl_device_info & info = ggml_sycl_info(); + info.devices[device].active = true; + int cnt = 0; + for (int i = 0; i < info.device_count; i++) { + if (info.devices[i].active) + cnt++; + } + info.active_device_count = cnt; +} + ggml_backend_t ggml_backend_sycl_init(int device) { - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_init\n"); + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_init. device=%d\n", device); ggml_check_sycl(); check_allow_gpu_index(device); @@ -7366,6 +8265,7 @@ ggml_backend_t ggml_backend_sycl_init(int device) { return nullptr; }; + set_device_active(device); ggml_backend_t sycl_backend = new ggml_backend { /* .guid = */ ggml_backend_sycl_guid(), /* .iface = */ ggml_backend_sycl_interface, diff --git a/ggml/src/ggml-sycl/gla.cpp b/ggml/src/ggml-sycl/gla.cpp index 879184fdd..379483427 100644 --- a/ggml/src/ggml-sycl/gla.cpp +++ b/ggml/src/ggml-sycl/gla.cpp @@ -1,6 +1,7 @@ #include <sycl/sycl.hpp> #include "common.hpp" +#include "gla.hpp" template <u_int HEAD_SIZE> static void gated_linear_attn_f32_kernel(const dpct::queue_ptr stream, u_int B, u_int T, u_int C, u_int H, float scale, diff --git a/ggml/src/ggml-sycl/im2col.hpp b/ggml/src/ggml-sycl/im2col.hpp index 976d10946..c9eba49ef 100644 --- a/ggml/src/ggml-sycl/im2col.hpp +++ b/ggml/src/ggml-sycl/im2col.hpp @@ -15,8 +15,6 @@ #include "common.hpp" -#define SYCL_IM2COL_BLOCK_SIZE 256 - void ggml_sycl_op_im2col(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_op_im2col_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/mem.cpp b/ggml/src/ggml-sycl/mem.cpp index ad5bfe0ff..8a1c3262c 100644 --- a/ggml/src/ggml-sycl/mem.cpp +++ b/ggml/src/ggml-sycl/mem.cpp @@ -27,7 +27,7 @@ const char * mem_api_int2str(int mem_api) { /* * Depend on to call zesInit(0) before any other Level Zero API calls, otherwise the Level Zero API calls may fail. */ -bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { +static bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using Level Zero API.\n", __func__); free_bytes = 0; @@ -94,7 +94,7 @@ bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & tot } #endif -bool get_memory_size_by_sycl_api(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { +static bool get_memory_size_by_sycl_api(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using SYCL API.\n", __func__); total_bytes = dev.get_info<sycl::info::device::global_mem_size>(); diff --git a/ggml/src/ggml-sycl/memtrace.cpp b/ggml/src/ggml-sycl/memtrace.cpp index 9c4f88539..ecd82ff6d 100644 --- a/ggml/src/ggml-sycl/memtrace.cpp +++ b/ggml/src/ggml-sycl/memtrace.cpp @@ -15,7 +15,6 @@ static const char * mem_type_name(ggml_sycl_mem_type type) { case GGML_SYCL_MEM_POOL_LEG: return "pool_leg"; case GGML_SYCL_MEM_POOL_VMM: return "pool_vmm"; case GGML_SYCL_MEM_ASYNC: return "async"; - case GGML_SYCL_MEM_FATTN_KV: return "fattn_kv"; case GGML_SYCL_MEM_DIRECT: return "direct"; default: GGML_ABORT("[%s] The type value %d is not supported\n", __func__, (int) type); } diff --git a/ggml/src/ggml-sycl/memtrace.hpp b/ggml/src/ggml-sycl/memtrace.hpp index 426d90963..c7da47f31 100644 --- a/ggml/src/ggml-sycl/memtrace.hpp +++ b/ggml/src/ggml-sycl/memtrace.hpp @@ -10,7 +10,6 @@ enum ggml_sycl_mem_type { GGML_SYCL_MEM_POOL_LEG, GGML_SYCL_MEM_POOL_VMM, GGML_SYCL_MEM_ASYNC, - GGML_SYCL_MEM_FATTN_KV, GGML_SYCL_MEM_DIRECT, GGML_SYCL_MEM_TYPE_COUNT, diff --git a/ggml/src/ggml-sycl/mmq.cpp b/ggml/src/ggml-sycl/mmq.cpp index ffb272aa2..68162a8bf 100644 --- a/ggml/src/ggml-sycl/mmq.cpp +++ b/ggml/src/ggml-sycl/mmq.cpp @@ -1340,7 +1340,7 @@ mul_mat_q(const void *__restrict__ vx, const void *__restrict__ vy, #define MMQ_X_Q4_0_RDNA1 64 #define MMQ_Y_Q4_0_RDNA1 64 #define NWARPS_Q4_0_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q4_0_AMPERE 4 #define MMQ_Y_Q4_0_AMPERE 32 #define NWARPS_Q4_0_AMPERE 4 @@ -1384,7 +1384,7 @@ template <bool need_check> static void #define MMQ_X_Q4_1_RDNA1 64 #define MMQ_Y_Q4_1_RDNA1 64 #define NWARPS_Q4_1_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q4_1_AMPERE 4 #define MMQ_Y_Q4_1_AMPERE 32 #define NWARPS_Q4_1_AMPERE 4 @@ -1427,7 +1427,7 @@ template <bool need_check> static void #define MMQ_X_Q5_0_RDNA1 64 #define MMQ_Y_Q5_0_RDNA1 64 #define NWARPS_Q5_0_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q5_0_AMPERE 4 #define MMQ_Y_Q5_0_AMPERE 32 #define NWARPS_Q5_0_AMPERE 4 @@ -1470,7 +1470,7 @@ template <bool need_check> static void #define MMQ_X_Q5_1_RDNA1 64 #define MMQ_Y_Q5_1_RDNA1 64 #define NWARPS_Q5_1_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q5_1_AMPERE 4 #define MMQ_Y_Q5_1_AMPERE 32 #define NWARPS_Q5_1_AMPERE 4 @@ -1513,7 +1513,7 @@ mul_mat_q5_1( #define MMQ_X_Q8_0_RDNA1 64 #define MMQ_Y_Q8_0_RDNA1 64 #define NWARPS_Q8_0_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q8_0_AMPERE 4 #define MMQ_Y_Q8_0_AMPERE 32 #define NWARPS_Q8_0_AMPERE 4 @@ -1556,7 +1556,7 @@ template <bool need_check> static void #define MMQ_X_Q2_K_RDNA1 128 #define MMQ_Y_Q2_K_RDNA1 32 #define NWARPS_Q2_K_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q2_K_AMPERE 4 #define MMQ_Y_Q2_K_AMPERE 32 #define NWARPS_Q2_K_AMPERE 4 @@ -1600,7 +1600,7 @@ mul_mat_q2_K( #define MMQ_X_Q3_K_RDNA1 32 #define MMQ_Y_Q3_K_RDNA1 128 #define NWARPS_Q3_K_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q3_K_AMPERE 4 #define MMQ_Y_Q3_K_AMPERE 32 #define NWARPS_Q3_K_AMPERE 4 @@ -1645,7 +1645,7 @@ mul_mat_q3_K( #define MMQ_X_Q4_K_RDNA1 32 #define MMQ_Y_Q4_K_RDNA1 64 #define NWARPS_Q4_K_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q4_K_AMPERE 4 #define MMQ_Y_Q4_K_AMPERE 32 #define NWARPS_Q4_K_AMPERE 4 @@ -1689,7 +1689,7 @@ template <bool need_check> static void #define MMQ_X_Q5_K_RDNA1 32 #define MMQ_Y_Q5_K_RDNA1 64 #define NWARPS_Q5_K_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q5_K_AMPERE 4 #define MMQ_Y_Q5_K_AMPERE 32 #define NWARPS_Q5_K_AMPERE 4 @@ -1733,7 +1733,7 @@ mul_mat_q5_K( #define MMQ_X_Q6_K_RDNA1 32 #define MMQ_Y_Q6_K_RDNA1 64 #define NWARPS_Q6_K_RDNA1 8 -#if defined(SYCL_USE_XMX) +#if defined(GGML_SYCL_SUPPORT_XMX) && !defined(GGML_SYCL_FORCE_MMQ) #define MMQ_X_Q6_K_AMPERE 4 #define MMQ_Y_Q6_K_AMPERE 32 #define NWARPS_Q6_K_AMPERE 4 @@ -1807,76 +1807,44 @@ static void ggml_mul_mat_q4_0_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:20: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_qs_q4_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<float, 1> tile_x_d_q4_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_0) + mmq_y / QI4_0), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_0<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q4_0_acc_ct1), - get_pointer(tile_x_d_q4_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_qs_q4_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<float, 1> tile_x_d_q4_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI4_0) + mmq_y / QI4_0), + cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q4_0<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_qs_q4_0_acc_ct1), + get_pointer(tile_x_d_q4_0_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:21: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_qs_q4_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<float, 1> tile_x_d_q4_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_0) + mmq_y / QI4_0), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_0<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q4_0_acc_ct1), - get_pointer(tile_x_d_q4_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -1922,76 +1890,44 @@ static void ggml_mul_mat_q4_1_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:22: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_qs_q4_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + +mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q4_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_1) + mmq_y / QI4_1), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_1<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q4_1_acc_ct1), - get_pointer(tile_x_dm_q4_1_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_qs_q4_1_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + +mmq_y), cgh); + sycl::local_accessor<sycl::half2, 1> tile_x_dm_q4_1_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI4_1) + mmq_y / QI4_1), + cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q4_1<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_qs_q4_1_acc_ct1), + get_pointer(tile_x_dm_q4_1_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:23: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_qs_q4_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + +mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q4_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_1) + mmq_y / QI4_1), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_1<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q4_1_acc_ct1), - get_pointer(tile_x_dm_q4_1_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2037,76 +1973,44 @@ static void ggml_mul_mat_q5_0_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:24: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q5_0_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<float, 1> tile_x_d_q5_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_0) + mmq_y / QI5_0), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_0<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_0_acc_ct1), - get_pointer(tile_x_d_q5_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_ql_q5_0_acc_ct1( + sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<float, 1> tile_x_d_q5_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI5_0) + mmq_y / QI5_0), + cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q5_0<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q5_0_acc_ct1), + get_pointer(tile_x_d_q5_0_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:25: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q5_0_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<float, 1> tile_x_d_q5_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_0) + mmq_y / QI5_0), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_0<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_0_acc_ct1), - get_pointer(tile_x_d_q5_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2152,76 +2056,44 @@ static void ggml_mul_mat_q5_1_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:26: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q5_1_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q5_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_1) + mmq_y / QI5_1), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_1<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_1_acc_ct1), - get_pointer(tile_x_dm_q5_1_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_ql_q5_1_acc_ct1( + sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<sycl::half2, 1> tile_x_dm_q5_1_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI5_1) + mmq_y / QI5_1), + cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q5_1<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q5_1_acc_ct1), + get_pointer(tile_x_dm_q5_1_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:27: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q5_1_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q5_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_1) + mmq_y / QI5_1), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_1<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_1_acc_ct1), - get_pointer(tile_x_dm_q5_1_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2267,76 +2139,44 @@ static void ggml_mul_mat_q8_0_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:28: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_qs_q8_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<float, 1> tile_x_d_q8_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI8_0) + mmq_y / QI8_0), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q8_0<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q8_0_acc_ct1), - get_pointer(tile_x_d_q8_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_qs_q8_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<float, 1> tile_x_d_q8_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI8_0) + mmq_y / QI8_0), + cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q8_0<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_qs_q8_0_acc_ct1), + get_pointer(tile_x_d_q8_0_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:29: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_qs_q8_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<float, 1> tile_x_d_q8_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI8_0) + mmq_y / QI8_0), - cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q8_0<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q8_0_acc_ct1), - get_pointer(tile_x_d_q8_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2382,82 +2222,47 @@ static void ggml_mul_mat_q2_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:30: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI2_K) + mmq_y / QI2_K), - cgh); - sycl::local_accessor<int, 1> tile_x_sc_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q2_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q2_K_acc_ct1), - get_pointer(tile_x_dm_q2_K_acc_ct1), - get_pointer(tile_x_sc_q2_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_ql_q2_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<sycl::half2, 1> tile_x_dm_q2_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI2_K) + mmq_y / QI2_K), + cgh); + sycl::local_accessor<int, 1> tile_x_sc_q2_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q2_K<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q2_K_acc_ct1), + get_pointer(tile_x_dm_q2_K_acc_ct1), + get_pointer(tile_x_sc_q2_K_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:31: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI2_K) + mmq_y / QI2_K), - cgh); - sycl::local_accessor<int, 1> tile_x_sc_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q2_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q2_K_acc_ct1), - get_pointer(tile_x_dm_q2_K_acc_ct1), - get_pointer(tile_x_sc_q2_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2505,88 +2310,50 @@ static void ggml_mul_mat_q3_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:32: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI3_K) + mmq_y / QI3_K), - cgh); - sycl::local_accessor<int, 1> tile_x_qh_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 2) + mmq_y / 2), cgh); - sycl::local_accessor<int, 1> tile_x_sc_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q3_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q3_K_acc_ct1), - get_pointer(tile_x_dm_q3_K_acc_ct1), - get_pointer(tile_x_qh_q3_K_acc_ct1), - get_pointer(tile_x_sc_q3_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_ql_q3_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<sycl::half2, 1> tile_x_dm_q3_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI3_K) + mmq_y / QI3_K), + cgh); + sycl::local_accessor<int, 1> tile_x_qh_q3_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 2) + mmq_y / 2), cgh); + sycl::local_accessor<int, 1> tile_x_sc_q3_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q3_K<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q3_K_acc_ct1), + get_pointer(tile_x_dm_q3_K_acc_ct1), + get_pointer(tile_x_qh_q3_K_acc_ct1), + get_pointer(tile_x_sc_q3_K_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:33: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI3_K) + mmq_y / QI3_K), - cgh); - sycl::local_accessor<int, 1> tile_x_qh_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 2) + mmq_y / 2), cgh); - sycl::local_accessor<int, 1> tile_x_sc_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q3_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q3_K_acc_ct1), - get_pointer(tile_x_dm_q3_K_acc_ct1), - get_pointer(tile_x_qh_q3_K_acc_ct1), - get_pointer(tile_x_sc_q3_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } #endif } @@ -2633,82 +2400,47 @@ static void ggml_mul_mat_q4_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:34: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_K) + mmq_y / QI4_K), - cgh); - sycl::local_accessor<int, 1> tile_x_sc_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q4_K_acc_ct1), - get_pointer(tile_x_dm_q4_K_acc_ct1), - get_pointer(tile_x_sc_q4_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_ql_q4_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<sycl::half2, 1> tile_x_dm_q4_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI4_K) + mmq_y / QI4_K), + cgh); + sycl::local_accessor<int, 1> tile_x_sc_q4_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q4_K<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q4_K_acc_ct1), + get_pointer(tile_x_dm_q4_K_acc_ct1), + get_pointer(tile_x_sc_q4_K_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:35: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_K) + mmq_y / QI4_K), - cgh); - sycl::local_accessor<int, 1> tile_x_sc_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q4_K_acc_ct1), - get_pointer(tile_x_dm_q4_K_acc_ct1), - get_pointer(tile_x_sc_q4_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2754,82 +2486,47 @@ static void ggml_mul_mat_q5_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:36: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_K) + mmq_y / QI5_K), - cgh); - sycl::local_accessor<int, 1> tile_x_sc_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_K_acc_ct1), - get_pointer(tile_x_dm_q5_K_acc_ct1), - get_pointer(tile_x_sc_q5_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_ql_q5_K_acc_ct1( + sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<sycl::half2, 1> tile_x_dm_q5_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI5_K) + mmq_y / QI5_K), + cgh); + sycl::local_accessor<int, 1> tile_x_sc_q5_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q5_K<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q5_K_acc_ct1), + get_pointer(tile_x_dm_q5_K_acc_ct1), + get_pointer(tile_x_sc_q5_K_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:37: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_K) + mmq_y / QI5_K), - cgh); - sycl::local_accessor<int, 1> tile_x_sc_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_K_acc_ct1), - get_pointer(tile_x_dm_q5_K_acc_ct1), - get_pointer(tile_x_sc_q5_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2875,82 +2572,47 @@ static void ggml_mul_mat_q6_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:38: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI6_K) + mmq_y / QI6_K), - cgh); - sycl::local_accessor<int, 1> tile_x_sc_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q6_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_acc_ct1), - get_pointer(tile_x_dm_acc_ct1), - get_pointer(tile_x_sc_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<int, 1> tile_x_ql_acc_ct1( + sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor<sycl::half2, 1> tile_x_dm_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI6_K) + mmq_y / QI6_K), + cgh); + sycl::local_accessor<int, 1> tile_x_sc_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); + sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q6_K<decltype(need_check)::value>( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_acc_ct1), + get_pointer(tile_x_dm_acc_ct1), + get_pointer(tile_x_sc_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:39: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor<int, 1> tile_x_ql_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor<sycl::half2, 1> tile_x_dm_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI6_K) + mmq_y / QI6_K), - cgh); - sycl::local_accessor<int, 1> tile_x_sc_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor<int, 1> tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor<sycl::half2, 1> tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q6_K<need_check>( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_acc_ct1), - get_pointer(tile_x_dm_acc_ct1), - get_pointer(tile_x_sc_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index d0f090703..03464bfae 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -6,6 +6,43 @@ #include "quants.hpp" #include "vecdotq.hpp" +#ifdef GGML_SYCL_MMVQ_HAS_XMX +# include "esimd.hpp" +#endif // GGML_SYCL_MMVQ_HAS_XMX + +// vec_dot_q_sycl_t adapters for the IQ vec_dots that take their codebook tables as extra +// arguments: bind the constant tables here (as vec_dot_iq2_s_q8_1 / vec_dot_iq1_m_q8_1 already do +// internally) so they can be used as template arguments of mul_mat_vec_q_moe. +static __dpct_inline__ float vec_dot_iq2_xxs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq2_xxs_q8_1(vbq, bq8_1, iqs, iq2xxs_grid, ksigns_iq2xs, kmask_iq2xs); +} + +static __dpct_inline__ float vec_dot_iq2_xs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq2_xs_q8_1(vbq, bq8_1, iqs, iq2xs_grid); +} + +static __dpct_inline__ float vec_dot_iq3_xxs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq3_xxs_q8_1(vbq, bq8_1, iqs, iq3xxs_grid); +} + +static __dpct_inline__ float vec_dot_iq3_s_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid); +} + +static __dpct_inline__ float vec_dot_iq1_s_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq1_s_q8_1(vbq, bq8_1, iqs, iq1s_grid_gpu); +} + // Minimum weight-row count at which the Q4_K multi-column MMVQ kernel handles two output rows per // subgroup (rows_per_sg == 2) instead of one, when ncols_dst == 2. // @@ -77,7 +114,8 @@ static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __r // With has_fusion, `vgate` is a second weight matrix sharing vx's shape, stride and reorder // layout: one pass computes both row dot products and the epilogue writes glu(gate, up). -template <typename reorder_vec_dot_q_sycl, int ncols_dst, bool has_fusion = false, int rows_per_sg = 1> +template <typename reorder_vec_dot_q_sycl, int ncols_dst, bool has_fusion = false, int rows_per_sg = 1, + bool shared_weights = reorder_vec_dot_shared_weights<reorder_vec_dot_q_sycl::gtype>::value> static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vgate, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, @@ -148,7 +186,7 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void } } } - } else if constexpr (reorder_vec_dot_shared_weights<reorder_vec_dot_q_sycl::gtype>::value) { + } else if constexpr (shared_weights) { const int ibx = row0 * blocks_per_row + i; const auto bx_offset = block_type::get_block_offset(ibx, nblocks); const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); @@ -331,6 +369,45 @@ static void mul_mat_vec_q_ncols( } } +template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl, int assert_qk> +static void launch_mul_mat_vec_q(const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % assert_qk == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + const auto launch = [&](auto ncols_tag) { + constexpr int N = decltype(ncols_tag)::value; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + if constexpr (N == 1) { + mul_mat_vec_q<qk, qi, block_q_t, vdr, vec_dot_q_sycl>( + vx, vy, dst, ncols, nrows, item_ct1); + } else { + mul_mat_vec_q_ncols<qk, qi, block_q_t, vdr, vec_dot_q_sycl, N>( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + } + }); + }); + }; + + switch (ncols_dst) { + case 1: launch(std::integral_constant<int, 1>{}); break; + case 2: launch(std::integral_constant<int, 2>{}); break; + case 3: launch(std::integral_constant<int, 3>{}); break; + case 4: launch(std::integral_constant<int, 4>{}); break; + case 5: launch(std::integral_constant<int, 5>{}); break; + case 6: launch(std::integral_constant<int, 6>{}); break; + case 7: launch(std::integral_constant<int, 7>{}); break; + case 8: launch(std::integral_constant<int, 8>{}); break; + default: GGML_ABORT("unsupported ncols_dst=%d", ncols_dst); + } +} + template <int qk, int qi, typename block_q_t, int vdr> static void mul_mat_vec_q_iq2_xxs_q8_1(const void *__restrict__ vx, const void *__restrict__ vy, @@ -413,7 +490,7 @@ static void mul_mat_vec_q_iq2_xs_q8_1(const void *__restrict__ vx, (item_ct1.get_local_id(2) % (qi / vdr)); // x block quant index when casting the quants to int - tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid, ksigns64); + tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid); } // sum up partial sums and write back result @@ -509,7 +586,7 @@ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx, (item_ct1.get_local_id(2) % (qi / vdr)); // x block quant index when casting the quants to int - tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid, ksigns64); + tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid); } // sum up partial sums and write back result @@ -524,6 +601,11 @@ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx, } } +static __dpct_inline__ float vec_dot_iq3_s_q8_1_adapter( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid); +} + template <int qk, int qi, typename block_q_t, int vdr> static void mul_mat_vec_q_iq3_s_q8_1(const void *__restrict__ vx, const void *__restrict__ vy, @@ -825,364 +907,136 @@ static void reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols( static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1, QK4_0>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q4_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK4_0, QI4_0, block_q4_0, - VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q4_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1, QK4_0>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK4_0, QI4_1, block_q4_1, - VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK4_0, QI4_1, block_q4_1, + VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1, QK4_1>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q4_1_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK4_0, QI4_1, block_q4_1, - VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q4_1_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q4_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q4_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q4_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q4_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q4_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q4_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q4_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q4_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q4_1 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_1 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK4_0, QI4_1, block_q4_1, + VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1, QK4_1>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_MXFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK_MXFP4, QI_MXFP4, block_mxfp4, VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK_MXFP4, QI_MXFP4, block_mxfp4, VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1, QK_MXFP4>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_mxfp4_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_MXFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK_MXFP4, QI_MXFP4, block_mxfp4, - VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_mxfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_mxfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_mxfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_mxfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_mxfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_mxfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_mxfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_mxfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for MXFP4 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for MXFP4 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK_MXFP4, QI_MXFP4, block_mxfp4, VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1, QK_MXFP4>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_nvfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_NVFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK_NVFP4, QI_NVFP4, block_nvfp4, VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK_NVFP4, QI_NVFP4, block_nvfp4, VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1, QK_NVFP4>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_nvfp4_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_NVFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK_NVFP4, QI_NVFP4, block_nvfp4, - VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_nvfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_nvfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_nvfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_nvfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_nvfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_nvfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_nvfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_nvfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for NVFP4 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for NVFP4 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK_NVFP4, QI_NVFP4, block_nvfp4, VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1, QK_NVFP4>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK5_0, QI5_0, block_q5_0, - VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK5_0, QI5_0, block_q5_0, + VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1, QK5_0>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q5_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK5_0, QI5_0, block_q5_0, - VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q5_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q5_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q5_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q5_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q5_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q5_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q5_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q5_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q5_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q5_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK5_0, QI5_0, block_q5_0, + VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1, QK5_0>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK5_1, QI5_1, block_q5_1, - VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK5_1, QI5_1, block_q5_1, + VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1, QK5_1>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q5_1_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK5_1, QI5_1, block_q5_1, - VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q5_1_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q5_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q5_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q5_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q5_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q5_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q5_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q5_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q5_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q5_1 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_1 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK5_1, QI5_1, block_q5_1, + VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1, QK5_1>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q8_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK8_0 == 0); @@ -1251,252 +1105,161 @@ static void reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols( } } -static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK8_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; +// MXFP4 reorder GEMV. Only MoE expert slices are reordered (opt_for_reorder_id); these dense entry +// points serve per-expert ggml_sycl_mul_mat calls from multi-token MUL_MAT_ID after that reorder. +static void reorder_mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_MXFP4 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK8_0, QI8_0, block_q8_0, - VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_MXFP4>>(vx, vy, dst, ncols, nrows, + nd_item); + }); + }); } template <int ncols_dst> -static void mul_mat_vec_q8_0_q8_1_sycl_ncols( +static void reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, + const int stride_col_y_bytes, const int stride_col_dst, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK8_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + GGML_ASSERT(ncols % QK_MXFP4 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK8_0, QI8_0, block_q8_0, - VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); -static void mul_mat_vec_q8_0_q8_1_sycl_switch_ncols( + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_MXFP4>, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, - const int stride_col_y, const int stride_col_dst, + const int stride_col_y_bytes, const int stride_col_dst, dpct::queue_ptr stream) { switch (ncols_dst) { - case 1: mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 multi-col MMVQ", ncols_dst); + case 1: reorder_mul_mat_vec_mxfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for MXFP4 reorder multi-col MMVQ", ncols_dst); } } -static void mul_mat_vec_q1_0_q8_1_sycl(const void * vx, const void * vy, - float * dst, const int ncols, +static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK1_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK1_0, QI1_0, block_q1_0, - VDR_Q1_0_Q8_1_MMVQ, vec_dot_q1_0_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); + launch_mul_mat_vec_q<QK8_0, QI8_0, block_q8_0, + VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1, QK8_0>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q1_0_q8_1_sycl_ncols( + +static void mul_mat_vec_q8_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, + const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK1_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q8_0 multi-col MMVQ", ncols_dst); + } + launch_mul_mat_vec_q<QK8_0, QI8_0, block_q8_0, + VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1, QK8_0>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); +} - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK1_0, QI1_0, block_q1_0, - VDR_Q1_0_Q8_1_MMVQ, vec_dot_q1_0_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); + +static void mul_mat_vec_q1_0_q8_1_sycl(const void * vx, const void * vy, + float * dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + launch_mul_mat_vec_q<QK1_0, QI1_0, block_q1_0, + VDR_Q1_0_Q8_1_MMVQ, vec_dot_q1_0_q8_1, QK1_0>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } + static void mul_mat_vec_q1_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q1_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q1_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK1_0, QI1_0, block_q1_0, + VDR_Q1_0_Q8_1_MMVQ, vec_dot_q1_0_q8_1, QK1_0>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q2_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK2_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK2_0, QI2_0, block_q2_0, - VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); + launch_mul_mat_vec_q<QK2_0, QI2_0, block_q2_0, + VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1, QK2_0>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q2_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK2_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK2_0, QI2_0, block_q2_0, - VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q2_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q2_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK2_0, QI2_0, block_q2_0, + VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1, QK2_0>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK_K, QI2_K, block_q2_K, - VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK_K, QI2_K, block_q2_K, + VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q2_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK_K, QI2_K, block_q2_K, - VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q2_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q2_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q2_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q2_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q2_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q2_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q2_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q2_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q2_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q2_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK_K, QI2_K, block_q2_K, + VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q2_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); @@ -1560,26 +1323,12 @@ static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK_K, QI3_K, block_q3_K, - VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK_K, QI3_K, block_q3_K, + VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } + static void reorder_mul_mat_vec_q3_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); @@ -1639,95 +1388,30 @@ static void reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols( } } -template <int ncols_dst> -static void mul_mat_vec_q3_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK_K, QI3_K, block_q3_K, - VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1, ncols_dst>( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} - static void mul_mat_vec_q3_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q3_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q3_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q3_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q3_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q3_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q3_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q3_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q3_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q3_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK_K, QI3_K, block_q3_K, + VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK_K, QI4_K, block_q4_K, - VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK_K, QI4_K, block_q4_K, + VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q4_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK_K, QI4_K, block_q4_K, - VDR_Q4_K_Q8_1_MMVQ, - vec_dot_q4_K_q8_1, - ncols_dst>( - vx, vy, dst, ncols, nrows, - stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q4_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, @@ -1735,19 +1419,15 @@ static void mul_mat_vec_q4_K_q8_1_sycl_switch_ncols( const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q4_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q4_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q4_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q4_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q4_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q4_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q4_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q4_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK_K, QI4_K, block_q4_K, + VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); @@ -1829,51 +1509,11 @@ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK_K, QI5_K, block_q5_K, - VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK_K, QI5_K, block_q5_K, + VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q5_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK_K, QI5_K, block_q5_K, - VDR_Q5_K_Q8_1_MMVQ, - vec_dot_q5_K_q8_1, - ncols_dst>( - vx, vy, dst, ncols, nrows, - stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q5_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, @@ -1881,19 +1521,15 @@ static void mul_mat_vec_q5_K_q8_1_sycl_switch_ncols( const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q5_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q5_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q5_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q5_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q5_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q5_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q5_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q5_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK_K, QI5_K, block_q5_K, + VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); @@ -1912,8 +1548,8 @@ static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, }); } -template <int ncols_dst> -static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols( +template <int ncols_dst, int rows_per_sg, bool shared_weights> +static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, @@ -1921,20 +1557,35 @@ static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols( GGML_ASSERT(ncols % QK_K == 0); constexpr size_t num_subgroups = WARP_SIZE; - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg); const sycl::range<3> block_nums(1, 1, block_num_y); const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>, ncols_dst>( + mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>, ncols_dst, + /*has_fusion=*/ false, rows_per_sg, shared_weights>( vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); }); }); } +template <int ncols_dst> +static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ggml_sycl_q5_k_mmvq_reuse(ggml_sycl_get_device())) { + constexpr int rows_per_sg = ncols_dst >= 3 ? 2 : 1; + reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl<ncols_dst, rows_per_sg, true>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } else { + reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl<ncols_dst, 1, false>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } +} + static void reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, @@ -2016,51 +1667,11 @@ static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q<QK_K, QI6_K, block_q6_K, - VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q<QK_K, QI6_K, block_q6_K, + VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template <int ncols_dst> -static void mul_mat_vec_q6_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols<QK_K, QI6_K, block_q6_K, - VDR_Q6_K_Q8_1_MMVQ, - vec_dot_q6_K_q8_1, - ncols_dst>( - vx, vy, dst, ncols, nrows, - stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q6_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, @@ -2068,20 +1679,16 @@ static void mul_mat_vec_q6_K_q8_1_sycl_switch_ncols( const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q6_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q6_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q6_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q6_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q6_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q6_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q6_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q6_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q6_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q<QK_K, QI6_K, block_q6_K, + VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1, QK_K>( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -2168,6 +1775,67 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy, } } +template <ggml_type type> +static void reorder_mul_mat_vec_iq3_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S); + GGML_ASSERT(ncols % QK_K == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<type>>( + vx, vy, dst, ncols, nrows, nd_item); + }); + }); +} + +template <ggml_type type, int ncols_dst> +static void reorder_mul_mat_vec_iq3_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S); + GGML_ASSERT(ncols % QK_K == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<type>, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +template <ggml_type type> +static void reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_iq3_q8_1_sycl<type>(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for reordered IQ3 multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -2190,6 +1858,67 @@ static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy, } } +template <int ncols_dst> +static void mul_mat_vec_iq3_s_q8_1_sycl_ncols(const void * vx, + const void * vy, + float * dst, + const int ncols, + const int nrows, + const int stride_col_y, + const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_ncols<QK_K, QI3_S / 2, block_iq3_s, 1, vec_dot_iq3_s_q8_1_moe, ncols_dst>( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + }); + }); +} + +static void mul_mat_vec_iq3_s_q8_1_sycl_switch_ncols(const void * vx, + const void * vy, + float * dst, + const int ncols, + const int nrows, + const int ncols_dst, + const int stride_col_y, + const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: + mul_mat_vec_iq3_s_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); + break; + case 2: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 3: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 4: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 5: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 6: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 7: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 8: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + default: + GGML_ABORT("unsupported ncols_dst=%d for IQ3_S multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -2322,6 +2051,158 @@ static void mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols( } } +#ifdef GGML_SYCL_MMVQ_HAS_XMX +// vgate is the gate weight of the fused gate and up kernel, nullptr for a plain mat-vec +template <template <bool> class Traits> +static void ggml_sycl_mul_mat_vec_q_xmx_run(bool short_rows, + const void * vx, + const void * vgate, + const char * vy, + float * dst, + int ne00, + int64_t nrows, + int64_t ncols_dst, + int stride_y_bytes, + int ldd, + int glu_op, + dpct::queue_ptr stream) { + namespace esimd = ggml_sycl_esimd; + if (vgate) { + if (short_rows) { + esimd::xmx_mul_mat_ncols<Traits<true>, true>(vx, vgate, vy, dst, ne00, nrows, ncols_dst, stride_y_bytes, + ldd, glu_op, stream); + } else { + esimd::xmx_mul_mat_ncols<Traits<false>, true>(vx, vgate, vy, dst, ne00, nrows, ncols_dst, stride_y_bytes, + ldd, glu_op, stream); + } + } else if (short_rows) { + esimd::xmx_mul_mat_ncols<Traits<true>>(vx, nullptr, vy, dst, ne00, nrows, ncols_dst, stride_y_bytes, ldd, 0, + stream); + } else { + esimd::xmx_mul_mat_ncols<Traits<false>>(vx, nullptr, vy, dst, ne00, nrows, ncols_dst, stride_y_bytes, ldd, 0, + stream); + } +} + +static bool ggml_sycl_mul_mat_vec_q_xmx_type(ggml_type type, + bool short_rows, + const void * vx, + const void * vgate, + const char * vy, + float * dst, + int ne00, + int64_t nrows, + int64_t ncols_dst, + int stride_y_bytes, + int ldd, + int glu_op, + dpct::queue_ptr stream) { +# define XMX_RUN(TRAITS) \ + ggml_sycl_mul_mat_vec_q_xmx_run<ggml_sycl_esimd::TRAITS>(short_rows, vx, vgate, vy, dst, ne00, nrows, \ + ncols_dst, stride_y_bytes, ldd, glu_op, stream); \ + return true + switch (type) { + case GGML_TYPE_Q2_K: + XMX_RUN(xmx_traits_q2_k); + case GGML_TYPE_Q3_K: + XMX_RUN(xmx_traits_q3_k); + case GGML_TYPE_Q4_K: + XMX_RUN(xmx_traits_q4_k); + case GGML_TYPE_Q5_K: + XMX_RUN(xmx_traits_q5_k); + case GGML_TYPE_Q6_K: + XMX_RUN(xmx_traits_q6_k); + case GGML_TYPE_Q8_0: + XMX_RUN(xmx_traits_q8_0); + default: + return false; + } +# undef XMX_RUN +} + +// multi-column XMX path for reordered weights, returns false when it does not apply +static bool ggml_sycl_mul_mat_vec_q_xmx(int device, + const ggml_tensor * src0, + const ggml_tensor * dst, + const char * src0_dd_i, + const char * src1_ddq_i, + float * dst_dd_i, + int64_t row_diff, + int64_t src1_ncols, + int64_t src1_padded_col_size, + dpct::queue_ptr stream) { + const auto * extra = static_cast<const ggml_tensor_extra_gpu *>(src0->extra); + if (!ggml_sycl_xmx_enabled(device) || !ggml_sycl_xmx_supports_type(src0->type) || !extra || + !extra->optimized_feature.reorder) { + return false; + } + if (src1_ncols < ggml_sycl_xmx_min_cols(src0->type) || src1_ncols > g_ggml_sycl_xmx_max_cols) { + return false; + } + + const int ne00 = src0->ne[0]; + const int ldd = dst->ne[0]; + if (!ggml_sycl_esimd::xmx_supported(src0_dd_i, ne00)) { + return false; + } + + // rows under 64 bytes use the variant without 2D block loads, it is not built for the widest tile + const bool short_rows = ne00 < 4 * QK_K; + if (short_rows && src1_ncols > 8) { + return false; + } + + const int stride_y_bytes = src1_padded_col_size * sizeof(block_q8_1) / QK8_1; + // the 2D block loads of the activations need a 64 byte aligned base and a pitch of 16 bytes + if ((uintptr_t) src1_ddq_i % 64 != 0 || stride_y_bytes % 16 != 0) { + return false; + } + return ggml_sycl_mul_mat_vec_q_xmx_type(src0->type, short_rows, src0_dd_i, nullptr, src1_ddq_i, dst_dd_i, ne00, + row_diff, src1_ncols, stride_y_bytes, ldd, 0, stream); +} +#endif // GGML_SYCL_MMVQ_HAS_XMX + +bool ggml_sycl_mul_mat_vec_q_glu_xmx(int device, + enum ggml_type src0_type, + enum ggml_glu_op glu_op, + const void * vx, + const void * vgate, + const void * vy, + float * dst, + int ncols, + int nrows, + int ncols_dst, + int stride_col_y_bytes, + int stride_col_dst, + dpct::queue_ptr stream) { +#ifdef GGML_SYCL_MMVQ_HAS_XMX + if (!ggml_sycl_xmx_enabled(device) || !ggml_sycl_xmx_supports_type(src0_type)) { + return false; + } + if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { + return false; + } + if (ncols_dst < ggml_sycl_xmx_min_cols(src0_type) || ncols_dst > g_ggml_sycl_xmx_glu_max_cols) { + return false; + } + if (!ggml_sycl_esimd::xmx_supported(vx, ncols) || !ggml_sycl_esimd::xmx_supported(vgate, ncols) || + (uintptr_t) vy % 64 != 0 || stride_col_y_bytes % 16 != 0) { + return false; + } + // rows under 64 bytes use the variant without 2D block loads, it is not built for the wide tiles + const bool short_rows = ncols < 4 * QK_K; + if (short_rows && ncols_dst > 8) { + return false; + } + return ggml_sycl_mul_mat_vec_q_xmx_type(src0_type, short_rows, vx, vgate, (const char *) vy, dst, ncols, nrows, + ncols_dst, stride_col_y_bytes, stride_col_dst, (int) glu_op, stream); +#else + GGML_UNUSED_VARS(device, src0_type, glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, stride_col_y_bytes, + stride_col_dst, stream); + return false; +#endif // GGML_SYCL_MMVQ_HAS_XMX +} + void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, @@ -2333,6 +2214,13 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens const int64_t ne00 = src0->ne[0]; const int64_t row_diff = row_high - row_low; +#ifdef GGML_SYCL_MMVQ_HAS_XMX + if (ggml_sycl_mul_mat_vec_q_xmx(ctx.device, src0, dst, src0_dd_i, src1_ddq_i, dst_dd_i, row_diff, src1_ncols, + src1_padded_col_size, stream)) { + return; + } +#endif // GGML_SYCL_MMVQ_HAS_XMX + int id; SYCL_CHECK(CHECK_TRY_ERROR(id = get_current_device_id())); const size_t q8_1_ts = sizeof(block_q8_1); @@ -2624,10 +2512,53 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens mul_mat_vec_iq2_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); break; case GGML_TYPE_IQ3_XXS: - mul_mat_vec_iq3_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling reordered IQ3_XXS multi-col MMVQ ncols=%d\n", (int) src1_ncols); + reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_XXS>( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y_bytes, stride_col_dst, stream); + return; + } else { + GGML_SYCL_DEBUG("Calling reordered IQ3_XXS MMVQ\n"); + reorder_mul_mat_vec_iq3_q8_1_sycl<GGML_TYPE_IQ3_XXS>( + src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + } else { + mul_mat_vec_iq3_xxs_q8_1_sycl( + src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } break; case GGML_TYPE_IQ3_S: - mul_mat_vec_iq3_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling reordered IQ3_S multi-col MMVQ ncols=%d\n", (int) src1_ncols); + reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_S>( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y_bytes, stride_col_dst, stream); + return; + } else { + GGML_SYCL_DEBUG("Calling reordered IQ3_S MMVQ\n"); + reorder_mul_mat_vec_iq3_q8_1_sycl<GGML_TYPE_IQ3_S>( + src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y = src1_padded_col_size / QK8_1; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling mul_mat_vec_iq3_s_q8_1_sycl_switch_ncols ncols=%d\n", (int) src1_ncols); + mul_mat_vec_iq3_s_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y, stride_col_dst, stream); + return; + } else if (i == 0 || src1_ncols == 1) { + mul_mat_vec_iq3_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } break; case GGML_TYPE_IQ4_NL: mul_mat_vec_iq4_nl_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); @@ -2646,7 +2577,21 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens } break; case GGML_TYPE_MXFP4: - if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + reorder_mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y_bytes, stride_col_dst, stream); + return; + } else { + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_mxfp4_q8_1_sycl\n"); + reorder_mul_mat_vec_mxfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { const int stride_col_y = src1_padded_col_size / QK8_1; const int stride_col_dst = dst->ne[0]; GGML_SYCL_DEBUG("Calling mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); @@ -2681,34 +2626,6 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens GGML_UNUSED(ctx); } -// vec_dot_q_sycl_t adapters for the IQ vec_dots that take their codebook tables as extra -// arguments: bind the constant tables here (as vec_dot_iq2_s_q8_1 / vec_dot_iq1_m_q8_1 already do -// internally) so they can be used as template arguments of mul_mat_vec_q_moe. -static __dpct_inline__ float vec_dot_iq2_xxs_q8_1_moe(const void * __restrict__ vbq, - const block_q8_1 * __restrict__ bq8_1, const int & iqs) { - return vec_dot_iq2_xxs_q8_1(vbq, bq8_1, iqs, iq2xxs_grid, ksigns_iq2xs, kmask_iq2xs); -} - -static __dpct_inline__ float vec_dot_iq2_xs_q8_1_moe(const void * __restrict__ vbq, - const block_q8_1 * __restrict__ bq8_1, const int & iqs) { - return vec_dot_iq2_xs_q8_1(vbq, bq8_1, iqs, iq2xs_grid, ksigns64); -} - -static __dpct_inline__ float vec_dot_iq3_xxs_q8_1_moe(const void * __restrict__ vbq, - const block_q8_1 * __restrict__ bq8_1, const int & iqs) { - return vec_dot_iq3_xxs_q8_1(vbq, bq8_1, iqs, iq3xxs_grid, ksigns64); -} - -static __dpct_inline__ float vec_dot_iq3_s_q8_1_moe(const void * __restrict__ vbq, - const block_q8_1 * __restrict__ bq8_1, const int & iqs) { - return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid); -} - -static __dpct_inline__ float vec_dot_iq1_s_q8_1_moe(const void * __restrict__ vbq, - const block_q8_1 * __restrict__ bq8_1, const int & iqs) { - return vec_dot_iq1_s_q8_1(vbq, bq8_1, iqs, iq1s_grid_gpu); -} - // src1_row_stride: 0 for shared src1 (gate/up proj), else per-expert stride (down proj). template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl> static void mul_mat_vec_q_moe( @@ -2722,7 +2639,6 @@ static void mul_mat_vec_q_moe( const int expert_idx = item_ct1.get_group(1); const int i02 = ids_dev[expert_idx]; - const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride; const char * vy = (const char *) vy_base + (size_t) expert_idx * src1_row_stride; float * dst = (float *) ((char *) dst_base + (size_t) expert_idx * dst_row_stride); @@ -2732,6 +2648,15 @@ static void mul_mat_vec_q_moe( return; } + if (i02 == -1) { // skipped slot + if (item_ct1.get_local_id(2) == 0) { + dst[row] = 0.0f; + } + return; + } + + const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride; + const int blocks_per_row = ncols / qk; constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi; @@ -2781,6 +2706,50 @@ static void launch_mul_mat_vec_q_moe( }); } +bool ggml_sycl_mul_mat_vec_q_id_supports_type(enum ggml_type src0_type) { + switch (src0_type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return true; + default: + return false; + } +} + +bool ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(enum ggml_type src0_type) { + switch (src0_type) { + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_MXFP4: + return true; + default: + return false; + } +} + bool ggml_sycl_mul_mat_vec_q_id( enum ggml_type src0_type, const void * vx_base, @@ -2794,6 +2763,9 @@ bool ggml_sycl_mul_mat_vec_q_id( size_t dst_row_stride, size_t src1_row_stride, dpct::queue_ptr stream) { + if (!ggml_sycl_mul_mat_vec_q_id_supports_type(src0_type)) { + return false; + } switch (src0_type) { case GGML_TYPE_Q4_0: launch_mul_mat_vec_q_moe<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>( @@ -2881,7 +2853,7 @@ bool ggml_sycl_mul_mat_vec_q_id( expert_weight_stride, dst_row_stride, src1_row_stride, stream); return true; case GGML_TYPE_IQ3_S: - launch_mul_mat_vec_q_moe<QK_K, QI3_S/2, block_iq3_s, VDR_IQ3_S_Q8_1_MMVQ, vec_dot_iq3_s_q8_1_moe>( + launch_mul_mat_vec_q_moe<QK_K, QI3_S/2, block_iq3_s, VDR_IQ3_S_Q8_1_MMVQ, vec_dot_iq3_s_q8_1_adapter>( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, expert_weight_stride, dst_row_stride, src1_row_stride, stream); return true; @@ -2927,7 +2899,6 @@ static void mul_mat_vec_q_moe_reorder( const int expert_idx = item_ct1.get_group(1); const int i02 = ids_dev[expert_idx]; - const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride; const char * vy = (const char *) vy_base + (size_t) expert_idx * src1_row_stride; float * dst = (float *) ((char *) dst_base + (size_t) expert_idx * dst_row_stride); @@ -2938,9 +2909,20 @@ static void mul_mat_vec_q_moe_reorder( const auto sg = item_ct1.get_sub_group(); + if (i02 == -1) { // skipped slot + if (sg.leader()) { + dst[row] = 0.0f; + } + return; + } + + const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride; + + constexpr int vdr_mmvq = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0 ? + VDR_Q8_0_Q8_1_MMVQ : block_traits::vdr_mmvq; const int blocks_per_row = ncols / block_traits::qk; - constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi); - constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq; + constexpr int blocks_per_subgroup = ceil_div(vdr_mmvq * WARP_SIZE, block_traits::qi); + constexpr int block_elements_per_subgroup = block_traits::qi / vdr_mmvq; const int nblocks = nrows * (ncols / block_traits::qk); static_assert(blocks_per_subgroup > 0); @@ -2959,12 +2941,24 @@ static void mul_mat_vec_q_moe_reorder( #pragma unroll for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) { - const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup); - partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + const int iqs = elem + vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup); + if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) { + partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs, vdr_mmvq); + } else { + partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + } } } - auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>()); + float sum = partial_sum; + if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) { + // Keep the accumulation order of the non-reordered MoE kernel. + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + sum += dpct::permute_sub_group_by_xor(sg, sum, mask); + } + } else { + sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>()); + } if (sg.leader()) { dst[row] = sum; } @@ -3004,7 +2998,20 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder( size_t dst_row_stride, size_t src1_row_stride, dpct::queue_ptr stream) { + if (!ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(src0_type)) { + return false; + } switch (src0_type) { + case GGML_TYPE_Q8_0: + launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_Q3_K: + launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q3_K>>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; case GGML_TYPE_Q4_K: launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, @@ -3020,6 +3027,11 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, expert_weight_stride, dst_row_stride, src1_row_stride, stream); return true; + case GGML_TYPE_MXFP4: + launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_MXFP4>>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; default: return false; } @@ -3054,8 +3066,11 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, const ggml_glu_op glu_op, dpct::queue_ptr stream) { + // q4_K pairs rows for 3..4 columns, q5_K for 3..5 + constexpr int row_pair_max = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q5_K ? 5 : 4; constexpr int rows_per_sg = - reorder_vec_dot_shared_activations<reorder_vec_dot_q_sycl::gtype>::value && ncols_dst >= 3 && ncols_dst <= 4 + reorder_vec_dot_shared_activations<reorder_vec_dot_q_sycl::gtype>::value && ncols_dst >= 3 && + ncols_dst <= row_pair_max ? 2 : 1; launch_mul_mat_vec_q_reorder_glu_impl<reorder_vec_dot_q_sycl, ncols_dst, rows_per_sg>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); @@ -3246,55 +3261,48 @@ bool ggml_sycl_mul_mat_vec_q_glu_plain(enum ggml_type gate_type, enum ggml_type return false; } +template <ggml_type type, int... Ns> +static bool mul_mat_vec_q_glu_reorder_ncols(enum ggml_glu_op glu_op, const void * vx, const void * vgate, + const void * vy, float * dst, int ncols, int nrows, int ncols_dst, + int stride_col_y_bytes, int stride_col_dst, dpct::queue_ptr stream) { + using vec_dot = reorder_vec_dot_q_sycl<type>; + + auto launch = [&](auto I) -> bool { + constexpr int n = decltype(I)::value; + if (ncols_dst != n) { + return false; + } + if constexpr (type == GGML_TYPE_Q4_K && n == 2) { + if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { + launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 2>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); + return true; + } + } + launch_mul_mat_vec_q_reorder_glu<vec_dot, n>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, + stride_col_dst, glu_op, stream); + return true; + }; + + // unary fold over launch + return (launch(std::integral_constant<int, Ns>{}) || ...); +} + bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, const void * vgate, const void * vy, float * dst, int ncols, int nrows, int ncols_dst, int stride_col_y_bytes, int stride_col_dst, dpct::queue_ptr stream) { - if (src0_type != GGML_TYPE_Q4_K) { - return false; - } if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { return false; } - using vec_dot = reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>; - - switch (ncols_dst) { - case 1: - launch_mul_mat_vec_q_reorder_glu<vec_dot, 1>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 2: - if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { - launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 2>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); - } else { - launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 1>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); - } - return true; - case 3: - launch_mul_mat_vec_q_reorder_glu<vec_dot, 3>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 4: - launch_mul_mat_vec_q_reorder_glu<vec_dot, 4>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 5: - launch_mul_mat_vec_q_reorder_glu<vec_dot, 5>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 6: - launch_mul_mat_vec_q_reorder_glu<vec_dot, 6>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 7: - launch_mul_mat_vec_q_reorder_glu<vec_dot, 7>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 8: - launch_mul_mat_vec_q_reorder_glu<vec_dot, 8>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; + switch (src0_type) { + case GGML_TYPE_Q4_K: + return mul_mat_vec_q_glu_reorder_ncols<GGML_TYPE_Q4_K, 1, 2, 3, 4, 5, 6, 7, 8>( + glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_Q5_K: + // fusion declines q5_K past 5 columns + return mul_mat_vec_q_glu_reorder_ncols<GGML_TYPE_Q5_K, 1, 2, 3, 4, 5>( + glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, stride_col_y_bytes, stride_col_dst, stream); default: return false; } diff --git a/ggml/src/ggml-sycl/mmvq.hpp b/ggml/src/ggml-sycl/mmvq.hpp index 7fb9cf6f8..e372d50ba 100644 --- a/ggml/src/ggml-sycl/mmvq.hpp +++ b/ggml/src/ggml-sycl/mmvq.hpp @@ -15,6 +15,12 @@ #include "common.hpp" +// q5_K multi-column MMVQ shares weights across columns, pairs rows and fuses gate/up in the reorder +// layout: faster on Xe2 (BMG), so untested archs keep the per-column kernel +inline bool ggml_sycl_q5_k_mmvq_reuse(int device) { + const gpu_arch arch = ggml_sycl_info().devices[device].hw_info.arch; + return arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31; +} void ggml_sycl_op_mul_mat_vec_q( ggml_backend_sycl_context & ctx, @@ -74,6 +80,24 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder( dpct::queue_ptr stream); +// Fused dense-FFN GEMV on the XMX engines for reordered weights of one type, up * act(gate) for up to +// GGML_SYCL_XMX_GLU_MAX_COLS (at most) activation columns. Returns false if it does not apply, the caller falls back. +// vy: src1 quantized with quantize_and_reorder_q8_1_soa. +bool ggml_sycl_mul_mat_vec_q_glu_xmx( + int device, + enum ggml_type src0_type, + enum ggml_glu_op glu_op, + const void * vx, // up weights + const void * vgate, // gate weights + const void * vy, + float * dst, + int ncols, // K, shared by both weights + int nrows, // output rows, i.e. weight ne[1] + int ncols_dst, // activation columns + int stride_col_y_bytes, // bytes between activation columns in vy + int stride_col_dst, // floats between output columns in dst + dpct::queue_ptr stream); + // Fused dense-FFN GEMV + GLU over the standard (non-reorder) layout; the gate and up // weights may carry different block types (q5_K / iq4_xs, mixed included). // vy: src1 quantized with plain quantize_q8_1 (padded rows). stride_col_y is in @@ -93,4 +117,7 @@ bool ggml_sycl_mul_mat_vec_q_glu_plain( int stride_col_dst, // floats between output columns in dst dpct::queue_ptr stream); +bool ggml_sycl_mul_mat_vec_q_id_supports_type(enum ggml_type src0_type); +bool ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(enum ggml_type src0_type); + #endif // GGML_SYCL_MMVQ_HPP diff --git a/ggml/src/ggml-sycl/pad.hpp b/ggml/src/ggml-sycl/pad.hpp index b099e9b73..4bc5a2cfd 100644 --- a/ggml/src/ggml-sycl/pad.hpp +++ b/ggml/src/ggml-sycl/pad.hpp @@ -15,8 +15,6 @@ #include "common.hpp" -#define SYCL_PAD_BLOCK_SIZE 256 - void ggml_sycl_pad(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_op_pad(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/presets.hpp b/ggml/src/ggml-sycl/presets.hpp index 789f3ef0f..e9fa6c8a6 100644 --- a/ggml/src/ggml-sycl/presets.hpp +++ b/ggml/src/ggml-sycl/presets.hpp @@ -33,10 +33,8 @@ #define SYCL_SET_BLOCK_SIZE 256 #define SYCL_CPY_BLOCK_SIZE 32 #define SYCL_SCALE_BLOCK_SIZE 256 -#define SYCL_CLAMP_BLOCK_SIZE 256 #define SYCL_ROPE_BLOCK_SIZE 256 #define SYCL_ALIBI_BLOCK_SIZE 32 -#define SYCL_DIAG_MASK_INF_BLOCK_SIZE 32 #define SYCL_QUANTIZE_BLOCK_SIZE 256 #define SYCL_DEQUANTIZE_BLOCK_SIZE 256 #define SYCL_GET_ROWS_BLOCK_SIZE 256 diff --git a/ggml/src/ggml-sycl/quants.hpp b/ggml/src/ggml-sycl/quants.hpp index a26a6ce6e..9fdcc22f2 100644 --- a/ggml/src/ggml-sycl/quants.hpp +++ b/ggml/src/ggml-sycl/quants.hpp @@ -178,6 +178,53 @@ template <> struct block_q_t<GGML_TYPE_Q6_K> { static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } }; +template <> struct block_q_t<GGML_TYPE_IQ3_XXS> { + struct traits { + static constexpr uint32_t qk = QK_K; + static constexpr uint32_t qi = QI3_XXS / 2; + static constexpr uint32_t qr = QR3_XXS; + static constexpr uint32_t vdr_mmvq = 1; + }; + + // Reordered layout: [96-byte qs records for all blocks][2-byte d records for all blocks]. + static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) { + return { block_index * (3 * QK_K / 8), 0 }; + } + + static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) { + const int nblocks = nrows * (ncols / QK_K); + return { nblocks * (3 * QK_K / 8) + block_index * sizeof(ggml_half), 0 }; + } + + static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } +}; + +template <> struct block_q_t<GGML_TYPE_IQ3_S> { + struct traits { + static constexpr uint32_t qk = QK_K; + static constexpr uint32_t qi = QI3_S / 2; + static constexpr uint32_t qr = QR3_S; + static constexpr uint32_t vdr_mmvq = 1; + }; + + // Reordered layout: [qs][qh][signs][{d, scales} per block]. Keeping the small + // metadata together leaves the generic reordered dot interface at two offset pairs. + static constexpr std::pair<int, int> get_block_offset(const int block_index, const int nblocks) { + return { block_index * (QK_K / 4), + nblocks * (QK_K / 4) + block_index * (QK_K / 32) }; + } + + static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) { + const int nblocks = nrows * (ncols / QK_K); + const int signs_base = nblocks * (QK_K / 4 + QK_K / 32); + const int metadata_base = signs_base + nblocks * (QK_K / 8); + return { signs_base + block_index * (QK_K / 8), + metadata_base + block_index * (sizeof(ggml_half) + IQ3S_N_SCALE) }; + } + + static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } +}; + template <> struct block_q_t<GGML_TYPE_Q8_0> { struct traits { static constexpr uint32_t qk = QK8_0; // 32 @@ -199,6 +246,27 @@ template <> struct block_q_t<GGML_TYPE_Q8_0> { static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } // 1 }; +template <> struct block_q_t<GGML_TYPE_MXFP4> { + struct traits { + static constexpr uint32_t qk = QK_MXFP4; // 32 + static constexpr uint32_t qi = QI_MXFP4; // 4 + static constexpr uint32_t qr = QR_MXFP4; // 2 + static constexpr uint32_t vdr_mmvq = 2; + }; + + // MXFP4 reorder layout: [qs0|qs1|...|qsN][e0|e1|...|eN] + // The 17-byte AoS block leaves qs unaligned; split out, every 16-byte nibble block is aligned. + static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) { + return { block_index * (QK_MXFP4 / 2), 0 }; + } + + static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) { + return { (ncols / 2 * nrows) + block_index, 0 }; + } + + static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } // 1 +}; + } // namespace ggml_sycl_reordered #endif // GGML_SYCL_QUANTS_HPP diff --git a/ggml/src/ggml-sycl/roll.cpp b/ggml/src/ggml-sycl/roll.cpp index 1e0518178..a95cee9a9 100644 --- a/ggml/src/ggml-sycl/roll.cpp +++ b/ggml/src/ggml-sycl/roll.cpp @@ -73,6 +73,8 @@ void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { const ggml_tensor *src = dst->src[0]; GGML_ASSERT(src && src->type == GGML_TYPE_F32); + // the kernel indexes in 32-bit + GGML_ASSERT(dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3] <= INT32_MAX); const int ne0 = (int) dst->ne[0]; const int ne1 = (int) dst->ne[1]; const int ne2 = (int) dst->ne[2]; diff --git a/ggml/src/ggml-sycl/rope.cpp b/ggml/src/ggml-sycl/rope.cpp index b6d22559d..b41c1a80e 100644 --- a/ggml/src/ggml-sycl/rope.cpp +++ b/ggml/src/ggml-sycl/rope.cpp @@ -547,7 +547,7 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst, rope_corr_dims corr_dims; ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, - beta_slow, corr_dims.v); + beta_slow, ggml_rope_get_truncate(dst), corr_dims.v); // compute if (is_neox) { diff --git a/ggml/src/ggml-sycl/rope.hpp b/ggml/src/ggml-sycl/rope.hpp index b95a58580..94a769898 100644 --- a/ggml/src/ggml-sycl/rope.hpp +++ b/ggml/src/ggml-sycl/rope.hpp @@ -15,8 +15,6 @@ #include "common.hpp" -#define SYCL_ROPE_BLOCK_SIZE 256 - void ggml_sycl_rope(ggml_backend_sycl_context & ctx, ggml_tensor *dst); void ggml_sycl_rope_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/topk-moe.cpp b/ggml/src/ggml-sycl/topk-moe.cpp index 78574c4b5..3dd92d261 100644 --- a/ggml/src/ggml-sycl/topk-moe.cpp +++ b/ggml/src/ggml-sycl/topk-moe.cpp @@ -296,7 +296,6 @@ static void ggml_sycl_op_topk_moe(ggml_backend_sycl_context & ctx, break; default: GGML_ASSERT(false && "fatal error"); - break; } } diff --git a/ggml/src/ggml-sycl/upscale.cpp b/ggml/src/ggml-sycl/upscale.cpp index e42cb419d..5f1c16be2 100644 --- a/ggml/src/ggml-sycl/upscale.cpp +++ b/ggml/src/ggml-sycl/upscale.cpp @@ -183,8 +183,8 @@ static void upscale_f32_bilinear_antialias(const float * src0, } namespace bicubic_interpolation { -static float weight1(float x, const float &a) { return ((a + 2) * x - (a + 3)) * x * x + 1; }; -static float weight2(float x, const float &a) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; }; +static float weight1(float x, const float &a) { return ((a + 2) * x - (a + 3)) * x * x + 1; } +static float weight2(float x, const float &a) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; } static float bicubic(float p0, float p1, float p2, float p3, float x, float a) { const float w0 = weight2(x + 1, a); @@ -192,7 +192,7 @@ static float bicubic(float p0, float p1, float p2, float p3, float x, float a) { const float w2 = weight1(1 - x, a); const float w3 = weight2(2 - x, a); return p0 * w0 + p1 * w1 + p2 * w2 + p3 * w3; -}; +} } @@ -358,7 +358,7 @@ static void upscale_f32_bicubic_sycl(const float * x, } } -void ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { +static void ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const float * src0_d = (const float *)src0->data; float * dst_d = (float *)dst->data; diff --git a/ggml/src/ggml-sycl/upscale.hpp b/ggml/src/ggml-sycl/upscale.hpp index c36c1bdc9..2b72ea1b8 100644 --- a/ggml/src/ggml-sycl/upscale.hpp +++ b/ggml/src/ggml-sycl/upscale.hpp @@ -4,6 +4,4 @@ #include "dpct/helper.hpp" #include "common.hpp" -#define SYCL_UPSCALE_BLOCK_SIZE 256 - void ggml_sycl_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index cc6ae6a8d..f547b20e6 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -148,6 +148,28 @@ static __dpct_inline__ sycl::int2 get_int_from_table_16( dpct::byte_level_permute(tmp[0], tmp[1], 0x7531)); } +// Four E2M1 codes (one per byte, bits 0..3) to their kvalues_mxfp4 int8 values. SWAR arithmetic +// replaces get_int_from_table_16 for MXFP4: dpct::byte_level_permute is emulated with 64-bit shifts, +// eight per int, which made the MXFP4 GEMV compute-bound on Intel GPUs. +// Magnitudes 0,1,2,3,4,6,8,12 = m + [m>=5] + [m>=6] + 3*[m>=7]; each byte stays below 256, so the +// byte-wise adds never carry. -0 (code 8) is left as 0 so the two's-complement +1 cannot carry either. +static __dpct_inline__ int mxfp4_codes_to_int8(const uint32_t x) { + const uint32_t m = x & 0x07070707u; + const uint32_t ge5 = ((m + 0x03030303u) >> 3) & 0x01010101u; + const uint32_t ge6 = ((m + 0x02020202u) >> 3) & 0x01010101u; + const uint32_t ge7 = ((m + 0x01010101u) >> 3) & 0x01010101u; + const uint32_t mag = m + ge5 + ge6 + 3u * ge7; + const uint32_t nz = ((mag + 0x7f7f7f7fu) >> 7) & 0x01010101u; + const uint32_t neg = (x >> 3) & nz & 0x01010101u; + return (int) ((mag ^ (neg * 0xffu)) + neg); +} + +// Same result as get_int_from_table_16(q4, kvalues_mxfp4): x = low nibbles, y = high nibbles. +static __dpct_inline__ sycl::int2 get_int_from_mxfp4(const int q4) { + return sycl::int2(mxfp4_codes_to_int8((uint32_t) q4 & 0x0f0f0f0fu), + mxfp4_codes_to_int8(((uint32_t) q4 >> 4) & 0x0f0f0f0fu)); +} + #define VDR_Q2_K_Q8_1_MMVQ 1 // contiguous v/x values @@ -362,6 +384,10 @@ template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_Q4_K> { static constexpr bool value = true; }; +template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_Q5_K> { + static constexpr bool value = true; +}; + template <ggml_type T> struct reorder_vec_dot_shared_activations { static constexpr bool value = false; }; @@ -370,6 +396,10 @@ template <> struct reorder_vec_dot_shared_activations<GGML_TYPE_Q4_K> { static constexpr bool value = true; }; +template <> struct reorder_vec_dot_shared_activations<GGML_TYPE_Q5_K> { + static constexpr bool value = true; +}; + template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0> { static constexpr ggml_type gtype = GGML_TYPE_Q4_0; @@ -412,7 +442,7 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0> { } return vec_dot_q4_0_q8_1_impl(v, u, d, *q8_1_ds); - }; + } }; // Load four contiguous dwords per operand instead of loading each value separately. @@ -453,7 +483,8 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> { __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset, const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr, - const sycl::half2 * q8_1_ds, const int & iqs) { + const sycl::half2 * q8_1_ds, const int & iqs, + const int vdr = q8_0_traits::vdr_mmvq) { const uint8_t * base = static_cast<const uint8_t *>(vbq); const int8_t * qs = reinterpret_cast<const int8_t *>(base + ibx_offset.first); const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.first); @@ -462,14 +493,14 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> { int u[q8_0_traits::vdr_mmvq]; #pragma unroll - for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) { + for (int i = 0; i < vdr; ++i) { v[i] = get_int_from_int8(qs, iqs + i); u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i); } int sumi = 0; #pragma unroll - for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) { + for (int i = 0; i < vdr; ++i) { sumi = dpct::dp4a(v[i], u[i], sumi); } @@ -676,56 +707,72 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K> { using q5_k_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q5_K>; using q5_k_traits = typename q5_k_block::traits; - __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset, - const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr, - const sycl::half2 * q8_1_ds, const int & iqs) { - const uint8_t * base = static_cast<const uint8_t *>(vbq); - const uint8_t * qs = base + ibx_offset.first; // low 4 bits - const uint8_t * qh_base = base + ibx_offset.second; // high bit - const uint8_t * scs = base + d_offset.first; - const ggml_half2 * dms = reinterpret_cast<const ggml_half2 *>(base + d_offset.second); + struct weights { + int vl[2]; + int vh[2]; + uint16_t aux[2]; + ggml_half2 dm; + }; + + // same activation layout as Q4_K + static_assert(QR5_K == QR4_K); + using activations = reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>::activations; + + __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset, + const std::pair<int, int> d_offset, const int & iqs) { + const uint8_t * base = static_cast<const uint8_t *>(vbq); + const uint8_t * qs = base + ibx_offset.first; // low 4 bits + const uint8_t * qh_base = base + ibx_offset.second; // high bit + const uint8_t * scs = base + d_offset.first; + const ggml_half2 * dms = reinterpret_cast<const ggml_half2 *>(base + d_offset.second); const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2)); const int * ql_ptr = (const int *) (qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4)); const int * qh_ptr = (const int *) (qh_base + 4 * ((iqs / 2) % 4)); const uint16_t * scales = (const uint16_t *) scs; - int vl[2]; - int vh[2]; - int u[2 * QR5_K]; - float d8[QR5_K]; - - vl[0] = ql_ptr[0]; - vl[1] = ql_ptr[4]; + weights w; + w.vl[0] = ql_ptr[0]; + w.vl[1] = ql_ptr[4]; - vh[0] = qh_ptr[0] >> bq8_offset; - vh[1] = qh_ptr[4] >> bq8_offset; + w.vh[0] = qh_ptr[0] >> bq8_offset; + w.vh[1] = qh_ptr[4] >> bq8_offset; - uint16_t aux[2]; const int j = (QR5_K * ((iqs / 2) / (QI8_1 / 2))) / 2; if (j < 2) { - aux[0] = scales[j + 0] & 0x3f3f; - aux[1] = scales[j + 2] & 0x3f3f; + w.aux[0] = scales[j + 0] & 0x3f3f; + w.aux[1] = scales[j + 2] & 0x3f3f; } else { - aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); - aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); + w.aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); + w.aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); } - const uint8_t * sc = (const uint8_t *) aux; - const uint8_t * m = sc + 2; + w.dm = *dms; - for (int i = 0; i < QR5_K; ++i) { - const int8_t* quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1; - sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i); + return w; + } + + __dpct_inline__ static activations load_activations(const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + return reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>::load_activations(q8_1_quant_ptr, q8_1_ds, iqs); + } - d8[i] = ds_values[0]; + __dpct_inline__ static float apply(const weights & w, const activations & a) { + const uint8_t * sc = (const uint8_t *) w.aux; + const uint8_t * m = sc + 2; - const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4); - u[2 * i + 0] = q8[0]; - u[2 * i + 1] = q8[4]; - } + return vec_dot_q5_K_q8_1_impl_vmmq(w.vl, w.vh, a.u, sc, m, w.dm, a.d8); + } - return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, *dms, d8); + __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + return apply(w, load_activations(q8_1_quant_ptr, q8_1_ds, iqs)); + } + + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset, + const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs); } }; @@ -771,6 +818,41 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K> { vl, vh, u0, u1, scs[0], scs[4], *d, d80, d81); } }; + +template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_MXFP4> { + static constexpr ggml_type gtype = GGML_TYPE_MXFP4; + + using mxfp4_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_MXFP4>; + using mxfp4_traits = typename mxfp4_block::traits; + + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset, + const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + static_assert(mxfp4_traits::vdr_mmvq == 2, "vector load assumes vdr_mmvq == 2"); + const uint8_t * base = static_cast<const uint8_t *>(vbq); + + // Reordered nibble blocks are 16 contiguous bytes and iqs is 0 or 2, so each lane's two + // weight ints are one aligned 8-byte load (the AoS layout needed eight byte loads). + const sycl::int2 q4 = *reinterpret_cast<const sycl::int2 *>(base + ibx_offset.first + sizeof(int) * iqs); + const uint8_t e = base[d_offset.first]; + + // Low nibbles pair with q8_1 ints iqs..iqs+1, high nibbles with iqs+4..iqs+5. + const sycl::int2 u_lo = *reinterpret_cast<const sycl::int2 *>(q8_1_quant_ptr + sizeof(int) * iqs); + const sycl::int2 u_hi = *reinterpret_cast<const sycl::int2 *>(q8_1_quant_ptr + sizeof(int) * (iqs + 4)); + + const sycl::int2 v0 = get_int_from_mxfp4(q4.x()); + const sycl::int2 v1 = get_int_from_mxfp4(q4.y()); + + int sumi = 0; + sumi = ggml_sycl_dp4a(v0.x(), u_lo.x(), sumi); + sumi = ggml_sycl_dp4a(v0.y(), u_hi.x(), sumi); + sumi = ggml_sycl_dp4a(v1.x(), u_lo.y(), sumi); + sumi = ggml_sycl_dp4a(v1.y(), u_hi.y(), sumi); + + const float d = ggml_sycl_e8m0_to_fp32(e) * 0.5f * static_cast<float>((*q8_1_ds)[0]); + return d * sumi; + } +}; #define VDR_Q4_0_Q8_1_MMVQ 2 #define VDR_Q4_0_Q8_1_MMQ 4 @@ -1100,7 +1182,7 @@ static __dpct_inline__ float vec_dot_mxfp4_q8_1(const void * __restrict__ vbq, #pragma unroll for (int l = 0; l < VDR_MXFP4_Q8_1_MMVQ; ++l) { const int aux_q4 = get_int_b1(bq4->qs, iqs + l); - const sycl::int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4); + const sycl::int2 v = get_int_from_mxfp4(aux_q4); sumi = ggml_sycl_dp4a(v.x(), q8[l + 0], sumi); sumi = ggml_sycl_dp4a(v.y(), q8[l + 4], sumi); } @@ -1433,6 +1515,20 @@ vec_dot_q6_K_q8_1(const void *__restrict__ vbq, // interchangeable and must not be copied across backends. #define VDR_IQ2_XXS_Q8_1_MMVQ 1 +static __dpct_inline__ uint32_t iq_sign_mask(const uint32_t signs) { + const uint32_t bits = + (signs & 0x01) | ((signs & 0x02) << 7) | + ((signs & 0x04) << 14) | ((signs & 0x08) << 21); + + return bits * 0xff; +} + +static __dpct_inline__ uint32_t iq_sign_bits(const uint32_t encoded) { + const uint32_t signs = encoded & 127; + + return signs | ((sycl::popcount(signs) & 1) << 7); +} + static __dpct_inline__ float vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, @@ -1469,7 +1565,7 @@ vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq, static __dpct_inline__ float vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, - const uint64_t *iq2xs_grid, const uint64_t *ksigns64) { + const uint64_t *iq2xs_grid) { #if DPCT_COMPATIBILITY_TEMP >= \ MIN_CC_DP4A // lowest compute capability for integer intrinsics #if QK_K == 256 @@ -1481,25 +1577,27 @@ vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq, const uint8_t ls1 = bq2->scales[ib32] & 0xf; const uint8_t ls2 = bq2->scales[ib32] >> 4; int sumi1 = 0; +#pragma unroll for (int l = 0; l < 2; ++l) { - const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511)); - const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9)); - const int grid_l = dpct::vectorized_binary<sycl::uchar4>( - grid[0] ^ signs[0], signs[0], std::minus<>()); - const int grid_h = dpct::vectorized_binary<sycl::uchar4>( - grid[1] ^ signs[1], signs[1], std::minus<>()); + const uint64_t grid = iq2xs_grid[q2[l] & 511]; + const uint32_t sign_bits = iq_sign_bits(q2[l] >> 9); + const uint32_t signs0 = iq_sign_mask(sign_bits); + const uint32_t signs1 = iq_sign_mask(sign_bits >> 4); + const int grid_l = ((uint32_t)grid ^ signs0) + (signs0 & 0x01010101); + const int grid_h = ((uint32_t)(grid >> 32) ^ signs1) + (signs1 & 0x01010101); sumi1 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi1); sumi1 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi1); q8 += 8; } int sumi2 = 0; +#pragma unroll for (int l = 2; l < 4; ++l) { - const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511)); - const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9)); - const int grid_l = dpct::vectorized_binary<sycl::uchar4>( - grid[0] ^ signs[0], signs[0], std::minus<>()); - const int grid_h = dpct::vectorized_binary<sycl::uchar4>( - grid[1] ^ signs[1], signs[1], std::minus<>()); + const uint64_t grid = iq2xs_grid[q2[l] & 511]; + const uint32_t sign_bits = iq_sign_bits(q2[l] >> 9); + const uint32_t signs0 = iq_sign_mask(sign_bits); + const uint32_t signs1 = iq_sign_mask(sign_bits >> 4); + const int grid_l = ((uint32_t)grid ^ signs0) + (signs0 & 0x01010101); + const int grid_h = ((uint32_t)(grid >> 32) ^ signs1) + (signs1 & 0x01010101); sumi2 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi2); sumi2 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi2); q8 += 8; @@ -1575,7 +1673,7 @@ vec_dot_iq2_s_q8_1(const void *__restrict__ vbq, static __dpct_inline__ float vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, - const uint32_t *iq3xxs_grid, const uint64_t *ksigns64) { + const uint32_t *iq3xxs_grid) { #if DPCT_COMPATIBILITY_TEMP >= \ MIN_CC_DP4A // lowest compute capability for integer intrinsics #if QK_K == 256 @@ -1587,14 +1685,15 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, const int8_t * q8 = bq8_1[ib32].qs; uint32_t aux32 = gas[0] | (gas[1] << 16); int sumi = 0; +#pragma unroll for (int l = 0; l < 4; ++l) { - const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0]; - const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1]; - const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127)); - const int grid_l = dpct::vectorized_binary<sycl::uchar4>( - grid1[0] ^ signs[0], signs[0], std::minus<>()); - const int grid_h = dpct::vectorized_binary<sycl::uchar4>( - grid2[0] ^ signs[1], signs[1], std::minus<>()); + const uint32_t grid1 = iq3xxs_grid[q3[2*l+0]]; + const uint32_t grid2 = iq3xxs_grid[q3[2*l+1]]; + const uint32_t sign_bits = iq_sign_bits(aux32); + const uint32_t signs0 = iq_sign_mask(sign_bits); + const uint32_t signs1 = iq_sign_mask(sign_bits >> 4); + const int grid_l = (grid1 ^ signs0) + (signs0 & 0x01010101); + const int grid_h = (grid2 ^ signs1) + (signs1 & 0x01010101); sumi = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi); sumi = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi); q8 += 8; @@ -1612,6 +1711,50 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, #endif } +template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_IQ3_XXS> { + static constexpr ggml_type gtype = GGML_TYPE_IQ3_XXS; + + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset, + const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { +#if DPCT_COMPATIBILITY_TEMP >= MIN_CC_DP4A && QK_K == 256 + const uint8_t * base = static_cast<const uint8_t *>(vbq); + const uint8_t * qs = base + ibx_offset.first; + const uint8_t * q3 = qs + 8 * iqs; + const uint16_t * gas = reinterpret_cast<const uint16_t *>(qs + QK_K / 4) + 2 * iqs; + const int8_t * q8 = q8_1_quant_ptr + iqs * QK8_1; + uint32_t aux32 = gas[0] | (gas[1] << 16); + + int sumi = 0; +#pragma unroll + for (int l = 0; l < 4; ++l) { + const uint32_t grid1 = iq3xxs_grid[q3[2*l+0]]; + const uint32_t grid2 = iq3xxs_grid[q3[2*l+1]]; + const uint32_t sign_bits = iq_sign_bits(aux32); + const uint32_t signs0 = iq_sign_mask(sign_bits); + const uint32_t signs1 = iq_sign_mask(sign_bits >> 4); + const int grid_l = (grid1 ^ signs0) + (signs0 & 0x01010101); + const int grid_h = (grid2 ^ signs1) + (signs1 & 0x01010101); + sumi = dpct::dp4a(grid_l, *((const int *) q8 + 0), sumi); + sumi = dpct::dp4a(grid_h, *((const int *) q8 + 1), sumi); + q8 += 8; + aux32 >>= 7; + } + + const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.first); + return (float) d * (0.5f + aux32) * q8_1_ds[iqs][0] * 0.5f * sumi; +#else + GGML_UNUSED(vbq); + GGML_UNUSED(ibx_offset); + GGML_UNUSED(d_offset); + GGML_UNUSED(q8_1_quant_ptr); + GGML_UNUSED(q8_1_ds); + GGML_UNUSED(iqs); + GGML_ABORT("IQ3_XXS reorder vec dot requires dp4a and QK_K == 256"); +#endif + } +}; + #define VDR_IQ3_S_Q8_1_MMVQ 1 static __dpct_inline__ float @@ -1626,18 +1769,13 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq, const int8_t * q8 = bq8_1[ib32].qs; int sumi = 0; for (int l = 0; l < 4; ++l) { - const uint32_t * grid1 = iq3s_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256)); - const uint32_t * grid2 = iq3s_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256)); - uint32_t signs0 = dpct::vectorized_binary<sycl::uchar4>( - ((bq2->signs[4 * ib32 + l] & 0xf) * 0x01010101) & 0x08040201, - 0x08040201, std::equal_to<>()); - uint32_t signs1 = dpct::vectorized_binary<sycl::uchar4>( - ((bq2->signs[4 * ib32 + l] >> 4) * 0x01010101) & 0x08040201, - 0x08040201, std::equal_to<>()); - const int grid_l = dpct::vectorized_binary<sycl::uchar4>( - grid1[0] ^ signs0, signs0, std::minus<>()); - const int grid_h = dpct::vectorized_binary<sycl::uchar4>( - grid2[0] ^ signs1, signs1, std::minus<>()); + const uint32_t grid1 = iq3s_grid[qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256)]; + const uint32_t grid2 = iq3s_grid[qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256)]; + const uint32_t signs = bq2->signs[4 * ib32 + l]; + const uint32_t signs0 = iq_sign_mask(signs); + const uint32_t signs1 = iq_sign_mask(signs >> 4); + const int grid_l = (grid1 ^ signs0) + (signs0 & 0x01010101); + const int grid_h = (grid2 ^ signs1) + (signs1 & 0x01010101); sumi = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi); sumi = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi); q8 += 8; @@ -1652,6 +1790,50 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq, #endif } +template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_IQ3_S> { + static constexpr ggml_type gtype = GGML_TYPE_IQ3_S; + + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset, + const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { +#if QK_K == 256 + const uint8_t * base = static_cast<const uint8_t *>(vbq); + const uint8_t * qs = base + ibx_offset.first + 8 * iqs; + const uint8_t * qh = base + ibx_offset.second; + const uint8_t * signs = base + d_offset.first + 4 * iqs; + const uint8_t * metadata = base + d_offset.second; + const ggml_half d = *reinterpret_cast<const ggml_half *>(metadata); + const uint8_t * scales = metadata + sizeof(ggml_half); + const int8_t * q8 = q8_1_quant_ptr + iqs * QK8_1; + + int sumi = 0; +#pragma unroll + for (int l = 0; l < 4; ++l) { + const uint32_t grid1 = iq3s_grid[qs[2*l+0] | ((qh[iqs] << (8 - 2*l)) & 256)]; + const uint32_t grid2 = iq3s_grid[qs[2*l+1] | ((qh[iqs] << (7 - 2*l)) & 256)]; + const uint32_t signs0 = iq_sign_mask(signs[l]); + const uint32_t signs1 = iq_sign_mask(signs[l] >> 4); + const int grid_l = (grid1 ^ signs0) + (signs0 & 0x01010101); + const int grid_h = (grid2 ^ signs1) + (signs1 & 0x01010101); + sumi = dpct::dp4a(grid_l, *((const int *) q8 + 0), sumi); + sumi = dpct::dp4a(grid_h, *((const int *) q8 + 1), sumi); + q8 += 8; + } + + const float scale = 1 + 2 * ((scales[iqs / 2] >> (4 * (iqs % 2))) & 0xf); + return (float) d * scale * q8_1_ds[iqs][0] * sumi; +#else + GGML_UNUSED(vbq); + GGML_UNUSED(ibx_offset); + GGML_UNUSED(d_offset); + GGML_UNUSED(q8_1_quant_ptr); + GGML_UNUSED(q8_1_ds); + GGML_UNUSED(iqs); + GGML_ABORT("IQ3_S reorder vec dot not supported for QK_K != 256"); +#endif + } +}; + #define VDR_IQ1_S_Q8_1_MMVQ 1 static __dpct_inline__ float diff --git a/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp b/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp index b6c561cd6..271be9a40 100644 --- a/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +++ b/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp @@ -106,6 +106,9 @@ const ggml_backend_buffer_i ggml_backend_remoting_buffer_interface = { /* .cpy_tensor = */ ggml_backend_remoting_buffer_cpy_tensor, /* .clear = */ ggml_backend_remoting_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; const ggml_backend_buffer_i ggml_backend_remoting_buffer_from_ptr_interface = { @@ -120,4 +123,7 @@ const ggml_backend_buffer_i ggml_backend_remoting_buffer_from_ptr_interface = { /* .cpy_tensor = */ ggml_backend_remoting_buffer_cpy_tensor, /* .clear = */ ggml_backend_remoting_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; diff --git a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp index 987ce9dd1..86df4cc5f 100644 --- a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +++ b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp @@ -30,11 +30,14 @@ static void ggml_backend_remoting_device_get_memory(ggml_backend_dev_t dev, size } static bool ggml_backend_remoting_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + if ((op->op == GGML_OP_ROPE || op->op == GGML_OP_ROPE_BACK) && + !ggml_rope_get_truncate(op)) { + return false; + } #if USE_ALWAYS_TRUE_SUPPORTS_OP == 1 /* ggml-rpc cheats it like this */ /* with the current implementation of serialize_tensor, the src/view aren't properly passed */ UNUSED(dev); - UNUSED(op); return true; #else diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txt index d7dec92a6..682b90e16 100644 --- a/ggml/src/ggml-vulkan/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/CMakeLists.txt @@ -66,6 +66,7 @@ if (Vulkan_FOUND) ggml-vulkan-push-constants.h ggml-vulkan-common.h ggml-vulkan-buffers.cpp + ggml-vulkan-comm.cpp ggml-vulkan-debug.cpp ) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp index f7a21dd27..9c6edb569 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp @@ -1,5 +1,30 @@ #include "ggml-vulkan-common.h" +#include <cstdint> +#include <cstdlib> +#include <mutex> +#include <shared_mutex> + +static void * portable_aligned_malloc(size_t align, size_t size) { +#ifdef _WIN32 + return _aligned_malloc(size, align); +#else + void * ptr = nullptr; + if (posix_memalign(&ptr, align, size) != 0) { + return nullptr; + } + return ptr; +#endif +} + +static void portable_aligned_free(void * ptr) { +#ifdef _WIN32 + _aligned_free(ptr); +#else + std::free(ptr); +#endif +} + ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = { /* .get_name = */ ggml_backend_vk_buffer_type_name, /* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer, @@ -204,7 +229,7 @@ vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) { buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, vk::MemoryPropertyFlagBits::eDeviceLocal, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); - } else if (device->disable_host_visible_vidmem) { + } else if (!device->rebar || device->disable_host_visible_vidmem) { if (device->allow_sysmem_fallback) { buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); @@ -243,67 +268,120 @@ void ggml_vk_destroy_buffer(vk_buffer& buf) { buf.reset(); } -void * ggml_vk_host_malloc(vk_device& device, size_t size) { +// caller must hold host_allocations_mutex +static size_t ggml_vk_host_allocation_find(const void * ptr) { + for (size_t i = 0; i < vk_instance.host_allocations.size(); i++) { + const uint8_t * base = (const uint8_t *) vk_instance.host_allocations[i].base; + if (ptr >= base && ptr < base + vk_instance.host_allocations[i].size) { + return i; + } + } + return SIZE_MAX; +} + +void * ggml_vk_host_malloc(size_t size) { VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")"); - vk_buffer buf = ggml_vk_create_buffer(device, size, - {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); - - if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) { - fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n", - size/1024.0/1024.0); - device->device.freeMemory(buf->device_memory); - device->device.destroyBuffer(buf->buffer); + + ggml_vk_instance_init(); + + if (vk_instance.multi_device) { + const size_t align = std::max<size_t>(vk_instance.host_import_alignment, TENSOR_ALIGNMENT); + const size_t alloc_size = (size + align - 1) / align * align; + + void * ptr = portable_aligned_malloc(align, alloc_size); + if (ptr == nullptr) { + return nullptr; + } + vk_host_allocation entry; + entry.base = ptr; + entry.size = alloc_size; + + std::unique_lock<std::shared_mutex> guard(vk_instance.host_allocations_mutex); + vk_instance.host_allocations.push_back(std::move(entry)); + return ptr; + } + + vk_device dev0 = ggml_vk_get_device(0); + vk_buffer buf; + try { + buf = ggml_vk_create_buffer(dev0, size, + {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); + } catch (vk::SystemError&) { return nullptr; } - std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex); - device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf)); + if (!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) { + fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n", size / 1024.0 / 1024.0); + ggml_vk_destroy_buffer(buf); + return nullptr; + } + vk_host_allocation entry; + entry.base = buf->ptr; + entry.size = size; + entry.views[dev0->idx] = buf; + + std::unique_lock<std::shared_mutex> guard(vk_instance.host_allocations_mutex); + vk_instance.host_allocations.push_back(std::move(entry)); return buf->ptr; } -void ggml_vk_host_free(vk_device& device, void* ptr) { +void ggml_vk_host_free(void * ptr) { if (ptr == nullptr) { return; } VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")"); - std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex); - vk_buffer buf; - size_t index; - for (size_t i = 0; i < device->pinned_memory.size(); i++) { - const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]); - const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]); - if (ptr >= addr && ptr < endr) { - buf = std::get<2>(device->pinned_memory[i]); - index = i; - break; + vk_host_allocation entry; + { + std::unique_lock<std::shared_mutex> guard(vk_instance.host_allocations_mutex); + const size_t index = ggml_vk_host_allocation_find(ptr); + if (index == SIZE_MAX) { + fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n"); + return; } + entry = std::move(vk_instance.host_allocations[index]); + vk_instance.host_allocations.erase(vk_instance.host_allocations.begin() + index); } - if (buf == nullptr) { - fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n"); - return; - } - - ggml_vk_destroy_buffer(buf); - device->pinned_memory.erase(device->pinned_memory.begin() + index); + for (auto & view : entry.views) { + ggml_vk_destroy_buffer(view); + } + if (vk_instance.multi_device) { + portable_aligned_free(entry.base); + } } void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) { - std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex); buf = nullptr; buf_offset = 0; - for (size_t i = 0; i < device->pinned_memory.size(); i++) { - const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]); - const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]); - if (ptr >= addr && ptr < endr) { - buf = std::get<2>(device->pinned_memory[i]); - buf_offset = ((const uint8_t *)ptr) - addr; - break; + + std::unique_lock<std::shared_mutex> guard(vk_instance.host_allocations_mutex); + + const size_t index = ggml_vk_host_allocation_find(ptr); + if (index == SIZE_MAX) { + return; + } + vk_host_allocation & entry = vk_instance.host_allocations[index]; + + if (entry.views[device->idx] == nullptr) { + const uint32_t bit = 1u << device->idx; + if (!vk_instance.multi_device || !vk_instance.device_host_buffer_import[device->idx] || (vk_instance.host_import_failed & bit)) { + return; + } + GGML_ASSERT(device->external_memory_host); + // const only propagates from the const context; the pointee is mutable + entry.views[device->idx] = ggml_vk_buffer_from_host_ptr(const_cast<vk_device&>(device), entry.base, entry.size); + if (entry.views[device->idx] == nullptr) { + vk_instance.host_import_failed |= bit; + GGML_LOG_WARN("ggml_vulkan: %s: host memory import failed, using staging\n", device->name.c_str()); + return; } } + + buf = entry.views[device->idx]; + buf_offset = (const uint8_t *) ptr - (const uint8_t *) entry.base; } void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size) { @@ -326,6 +404,33 @@ void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t si } } +static vk_context ggml_vk_begin_sync_ctx(vk_device & device, vk_command_pool & pool) { + vk_context subctx = ggml_vk_create_temporary_context(pool); + ggml_vk_ctx_begin(device, subctx); + return subctx; +} + +static void ggml_vk_submit_sync(vk_device & device, vk_context & subctx) { + ggml_vk_ctx_end(subctx); + + for (auto & cpy : subctx->in_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + + for (auto & mset : subctx->memsets) { + memset(mset.dst, mset.val, mset.n); + } + + ggml_vk_submit(subctx, device->fence); + VK_CHECK(device->device.waitForFences({ device->fence }, true, UINT64_MAX), "vk_submit_sync waitForFences", device); + device->device.resetFences({ device->fence }); + ggml_vk_queue_command_pools_cleanup(device); + + for (auto & cpy : subctx->out_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } +} + static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) { VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")"); GGML_ASSERT(!ggml_is_contiguous(tensor)); @@ -512,24 +617,10 @@ void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, si } else { std::lock_guard<std::recursive_mutex> guard(dst->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(dst->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(dst->device, dst->device->transfer_queue->cmd_pool); bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true); GGML_ASSERT(ret); - ggml_vk_ctx_end(subctx); - - for (auto& cpy : subctx->in_memcpys) { - memcpy(cpy.dst, cpy.src, cpy.n); - } - - for (auto& mset : subctx->memsets) { - memset(mset.dst, mset.val, mset.n); - } - - ggml_vk_submit(subctx, dst->device->fence); - VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device); - dst->device->device.resetFences({ dst->device->fence }); - ggml_vk_queue_command_pools_cleanup(dst->device); + ggml_vk_submit_sync(dst->device, subctx); } } @@ -617,18 +708,24 @@ static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging); } -void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) { - VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")"); - +static bool ggml_vk_buffer_host_read_direct(const vk_buffer & buf) { // If the device is not an UMA device the memory is host-accessible through rebar. While writing // through PCIe is sufficient fast reading back data from PCIe is slower than going through // the HW device to host copy path. - if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma) { + // AMD UMA: uncached host-visible memory is write-combined, CPU reads are slow + const bool slow_host_read = buf->device->vendor_id == VK_VENDOR_ID_AMD && + !(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); + return (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) && buf->device->uma && !slow_host_read; +} + +void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) { + VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")"); + + if (ggml_vk_buffer_host_read_direct(src)) { GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); std::lock_guard<std::recursive_mutex> guard(src->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->compute_queue->cmd_pool); subctx->s->buffer->buf.pipelineBarrier( vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost, @@ -636,12 +733,7 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite, vk::AccessFlagBits::eHostRead } }, {}, {}); - ggml_vk_ctx_end(subctx); - ggml_vk_submit(subctx, src->device->fence); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), - "vk_buffer_read_2d uma waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - ggml_vk_queue_command_pools_cleanup(src->device); + ggml_vk_submit_sync(src->device, subctx); if (width == spitch && width == dpitch) { memcpy(dst, (const uint8_t *) src->ptr + offset, width * height); @@ -653,20 +745,10 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp } else { std::lock_guard<std::recursive_mutex> guard(src->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->transfer_queue->cmd_pool); bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true); GGML_ASSERT(ret); - ggml_vk_ctx_end(subctx); - - ggml_vk_submit(subctx, src->device->fence); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - ggml_vk_queue_command_pools_cleanup(src->device); - - for (auto& cpy : subctx->out_memcpys) { - memcpy(cpy.dst, cpy.src, cpy.n); - } + ggml_vk_submit_sync(src->device, subctx); } } @@ -690,14 +772,9 @@ void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size std::lock_guard<std::recursive_mutex> guard(src->device->mutex); VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")"); // Copy within the device - vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->transfer_queue->cmd_pool); ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size); - ggml_vk_ctx_end(subctx); - ggml_vk_submit(subctx, src->device->fence); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - ggml_vk_queue_command_pools_cleanup(src->device); + ggml_vk_submit_sync(src->device, subctx); } else { VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")"); // Copy device to device @@ -733,15 +810,222 @@ void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t siz } std::lock_guard<std::recursive_mutex> guard(dst->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(dst->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(dst->device, dst->device->transfer_queue->cmd_pool); subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c); - ggml_vk_ctx_end(subctx); + ggml_vk_submit_sync(dst->device, subctx); +} + +static size_t ggml_vk_staging_size_n(const vk_device & device, const std::vector<vk_buffer_region> & regions) { + size_t total = 0; + size_t max = 0; + for (const auto & r : regions) { + vk_buffer pinned; + size_t pinned_offset; + ggml_vk_host_get(device, r.host, pinned, pinned_offset); + if (pinned != nullptr) { + continue; + } + total += r.size; + max = std::max(max, r.size); + } + return std::max(max, std::min(total, (size_t) device->suballocation_block_size)); +} + +// returns number of regions recorded, stops when staging is full +size_t ggml_vk_buffer_write_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging, size_t * staging_offset) { + VK_LOG_DEBUG("ggml_vk_buffer_write_n_async(" << n << ")"); + ggml_vk_sync_buffers(nullptr, subctx); + + size_t i = 0; + for (; i < n; i++) { + const vk_buffer_region & r = regions[i]; + + vk_buffer pinned = nullptr; + size_t pinned_offset = 0; + ggml_vk_host_get(r.buf->device, r.host, pinned, pinned_offset); + + if (pinned != nullptr) { + subctx->s->buffer->buf.copyBuffer(pinned->buffer, r.buf->buffer, { vk::BufferCopy{ pinned_offset, r.offset, r.size } }); + continue; + } + + if (staging == nullptr || *staging_offset + r.size > (*staging)->size) { + break; + } + + subctx->s->buffer->buf.copyBuffer((*staging)->buffer, r.buf->buffer, { vk::BufferCopy{ *staging_offset, r.offset, r.size } }); + deferred_memcpy((uint8_t *) (*staging)->ptr + *staging_offset, r.host, r.size, &subctx->in_memcpys); + *staging_offset += r.size; + } + return i; +} + +// returns number of regions recorded, stops when staging is full +size_t ggml_vk_buffer_read_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging, size_t * staging_offset) { + VK_LOG_DEBUG("ggml_vk_buffer_read_n_async(" << n << ")"); + ggml_vk_sync_buffers(nullptr, subctx); + + size_t i = 0; + for (; i < n; i++) { + const vk_buffer_region & r = regions[i]; + + vk_buffer pinned = nullptr; + size_t pinned_offset = 0; + ggml_vk_host_get(r.buf->device, r.host, pinned, pinned_offset); + + if (pinned != nullptr) { + subctx->s->buffer->buf.copyBuffer(r.buf->buffer, pinned->buffer, { vk::BufferCopy{ r.offset, pinned_offset, r.size } }); + continue; + } + + if (staging == nullptr || *staging_offset + r.size > (*staging)->size) { + break; + } + + subctx->s->buffer->buf.copyBuffer(r.buf->buffer, (*staging)->buffer, { vk::BufferCopy{ r.offset, *staging_offset, r.size } }); + deferred_memcpy(r.host, (const uint8_t *) (*staging)->ptr + *staging_offset, r.size, &subctx->out_memcpys); + *staging_offset += r.size; + } + return i; +} + +void ggml_vk_buffer_copy_n_async(vk_context& subctx, const vk_buffer_copy_region * regions, size_t n) { + VK_LOG_DEBUG("ggml_vk_buffer_copy_n_async(" << n << ")"); + ggml_vk_sync_buffers(nullptr, subctx); + + for (size_t i = 0; i < n; i++) { + const vk_buffer_copy_region & r = regions[i]; + GGML_ASSERT(r.src->device == r.dst->device); + subctx->s->buffer->buf.copyBuffer(r.src->buffer, r.dst->buffer, { vk::BufferCopy{ r.src_offset, r.dst_offset, r.size } }); + } +} + +void ggml_vk_buffer_write_n(const vk_buffer_region * regions, size_t n) { + VK_LOG_DEBUG("ggml_vk_buffer_write_n(" << n << ")"); + std::vector<vk_buffer_region> pending; + + for (size_t i = 0; i < n; i++) { + const vk_buffer_region & r = regions[i]; + if (r.size == 0) { + continue; + } + if (r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { + GGML_ASSERT(r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); + memcpy((uint8_t *) r.buf->ptr + r.offset, r.host, r.size); + continue; + } + GGML_ASSERT(pending.empty() || r.buf->device == pending[0].buf->device); + pending.push_back(r); + } + + if (pending.empty()) { + return; + } + + vk_device device = pending[0].buf->device; + std::lock_guard<std::recursive_mutex> guard(device->mutex); + const size_t staging_size = ggml_vk_staging_size_n(device, pending); + if (staging_size > 0) { + ggml_vk_ensure_sync_staging_buffer(device, staging_size); + } + + for (size_t i = 0; i < pending.size();) { + vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool); + size_t staging_offset = 0; + const size_t recorded = ggml_vk_buffer_write_n_async(subctx, pending.data() + i, pending.size() - i, &device->sync_staging, &staging_offset); + GGML_ASSERT(recorded > 0); + ggml_vk_submit_sync(device, subctx); + i += recorded; + } +} + +void ggml_vk_buffer_read_n(const vk_buffer_region * regions, size_t n) { + VK_LOG_DEBUG("ggml_vk_buffer_read_n(" << n << ")"); + std::vector<vk_buffer_region> direct; + std::vector<vk_buffer_region> pending; + + for (size_t i = 0; i < n; i++) { + const vk_buffer_region & r = regions[i]; + if (r.size == 0) { + continue; + } + GGML_ASSERT((direct.empty() && pending.empty()) || r.buf->device == (direct.empty() ? pending[0] : direct[0]).buf->device); + if (ggml_vk_buffer_host_read_direct(r.buf)) { + GGML_ASSERT(r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); + direct.push_back(r); + } else { + pending.push_back(r); + } + } + + if (direct.empty() && pending.empty()) { + return; + } + + vk_device device = (direct.empty() ? pending[0] : direct[0]).buf->device; + std::lock_guard<std::recursive_mutex> guard(device->mutex); + + if (!direct.empty()) { + vk_context subctx = ggml_vk_begin_sync_ctx(device, device->compute_queue->cmd_pool); + subctx->s->buffer->buf.pipelineBarrier( + vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, + {}, + { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite, + vk::AccessFlagBits::eHostRead } }, + {}, {}); + ggml_vk_submit_sync(device, subctx); + + for (const auto & r : direct) { + memcpy(r.host, (const uint8_t *) r.buf->ptr + r.offset, r.size); + } + } + + if (pending.empty()) { + return; + } + + const size_t staging_size = ggml_vk_staging_size_n(device, pending); + if (staging_size > 0) { + ggml_vk_ensure_sync_staging_buffer(device, staging_size); + } + + for (size_t i = 0; i < pending.size();) { + vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool); + size_t staging_offset = 0; + const size_t recorded = ggml_vk_buffer_read_n_async(subctx, pending.data() + i, pending.size() - i, &device->sync_staging, &staging_offset); + GGML_ASSERT(recorded > 0); + ggml_vk_submit_sync(device, subctx); + i += recorded; + } +} + +void ggml_vk_buffer_copy_n(const vk_buffer_copy_region * regions, size_t n) { + VK_LOG_DEBUG("ggml_vk_buffer_copy_n(" << n << ")"); + std::vector<vk_buffer_copy_region> same_device; + + for (size_t i = 0; i < n; i++) { + vk_buffer_copy_region r = regions[i]; + if (r.size == 0) { + continue; + } + if (r.src->device != r.dst->device) { + ggml_vk_buffer_copy(r.dst, r.dst_offset, r.src, r.src_offset, r.size); + continue; + } + GGML_ASSERT(same_device.empty() || r.dst->device == same_device[0].dst->device); + same_device.push_back(std::move(r)); + } + + if (same_device.empty()) { + return; + } - ggml_vk_submit(subctx, dst->device->fence); - VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device); - dst->device->device.resetFences({ dst->device->fence }); - ggml_vk_queue_command_pools_cleanup(dst->device); + vk_device device = same_device[0].dst->device; + std::lock_guard<std::recursive_mutex> guard(device->mutex); + vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool); + ggml_vk_buffer_copy_n_async(subctx, same_device.data(), same_device.size()); + ggml_vk_submit_sync(device, subctx); } ggml_backend_buffer_i ggml_backend_vk_buffer_interface = { @@ -756,6 +1040,9 @@ ggml_backend_buffer_i ggml_backend_vk_buffer_interface = { /* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor, /* .clear = */ ggml_backend_vk_buffer_clear, /* .reset = */ NULL, + /* .set_tensor_n = */ ggml_backend_vk_buffer_set_tensor_n, + /* .get_tensor_n = */ ggml_backend_vk_buffer_get_tensor_n, + /* .cpy_tensor_n = */ ggml_backend_vk_buffer_cpy_tensor_n, }; vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) { diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp new file mode 100644 index 000000000..84af0661a --- /dev/null +++ b/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp @@ -0,0 +1,591 @@ +#include "ggml-vulkan-common.h" + +#if !defined(_WIN32) +#include <fcntl.h> +#include <unistd.h> +#if defined(__linux__) +#include <pthread.h> +#endif + +static constexpr size_t ggml_vk_comm_f16_min_size = 1024 * 1024; + +// follows ops ggml-backend-meta may delay a reduction across +// TODO: the required precision should come from the meta backend instead. +static bool ggml_vk_comm_needs_f32(const ggml_tensor * t, int depth = 0) { + if (t == nullptr || depth > 64) { + return false; + } + switch (t->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + return ggml_prec_get_acc(t) == GGML_PREC_BF16 || ggml_prec_get_acc(t) == GGML_PREC_F32; + case GGML_OP_ADD: + return ggml_vk_comm_needs_f32(t->src[0], depth + 1) || ggml_vk_comm_needs_f32(t->src[1], depth + 1); + case GGML_OP_MUL: + case GGML_OP_ADD_ID: + return ggml_vk_comm_needs_f32(t->src[0], depth + 1); + case GGML_OP_VIEW: + return ggml_vk_comm_needs_f32(t->src[0], depth + 1); + default: + return false; + } +} + +// host_bridge: device i signals done[i], a host thread signals go[i] once all peers of i are done +// sync_fd: a host thread exchanges SYNC_FD semaphores between peers, the wait resolves in the kernel +enum class vk_comm_sync { + host_bridge, + sync_fd, +}; + +struct vk_comm_device { + ggml_backend_t backend = nullptr; + ggml_backend_vk_context * ctx = nullptr; + void * host_ptr = nullptr; + std::vector<ggml_backend_buffer_t> host_buf; // host_ptr imported into each peer + + struct { + PFN_vkGetSemaphoreFdKHR get_fd = nullptr; + PFN_vkImportSemaphoreFdKHR import_fd = nullptr; + vk::Semaphore sem; + std::vector<vk::Semaphore> imp; // [peer], pooled, reused only after ggml_vk_graph_cleanup + } sfd; + + struct { + vk::Semaphore done; + vk::Semaphore go; + } bridge; +}; + +struct ggml_backend_vk_comm_context { + std::vector<vk_comm_device> devs; + vk_comm_sync mode = vk_comm_sync::host_bridge; + size_t align = 4096; + size_t cap = 0; + uint64_t round = 0; + ggml_context * tctx = nullptr; + ggml_tensor * slot = nullptr; + bool disabled = false; + + std::thread thread; + std::chrono::nanoseconds spin{ 200'000 }; + std::atomic<uint64_t> submitted{ 0 }; + std::atomic<uint64_t> imported{ 0 }; + std::atomic<bool> sleeping{ false }; + std::atomic<bool> stop{ false }; + std::mutex mutex; + std::condition_variable cv; +}; + +static void ggml_vk_comm_free_buffers(ggml_backend_vk_comm_context * comm) { + for (vk_comm_device & d : comm->devs) { + for (ggml_backend_buffer_t & buf : d.host_buf) { + ggml_backend_buffer_free(buf); + buf = nullptr; + } + std::free(d.host_ptr); + d.host_ptr = nullptr; + } + comm->cap = 0; +} + +static bool ggml_backend_vk_comm_ensure(ggml_backend_vk_comm_context * comm, size_t nbytes) { + if (comm->disabled) { + return false; + } + if (nbytes <= comm->cap) { + return true; + } + auto fail = [comm]() { + GGML_LOG_WARN("ggml_vulkan: shared host buffer allocation failed, using generic all-reduce\n"); + comm->disabled = true; + return false; + }; + for (vk_comm_device & d : comm->devs) { + ggml_backend_synchronize(d.backend); + } + ggml_vk_comm_free_buffers(comm); + + const size_t n = comm->devs.size(); + const size_t cap = GGML_PAD(nbytes, comm->align); + for (size_t i = 0; i < n; i++) { + vk_comm_device & d = comm->devs[i]; + if (posix_memalign(&d.host_ptr, comm->align, 2 * cap) != 0) { + d.host_ptr = nullptr; + return fail(); + } + for (size_t j = 0; j < n; j++) { + d.host_buf[j] = ggml_backend_vk_device_buffer_from_host_ptr( + ggml_backend_get_device(comm->devs[j].backend), d.host_ptr, 2 * cap, 2 * cap); + if (!d.host_buf[j]) { + return fail(); + } + } + } + comm->cap = cap; + return true; +} + +static void ggml_vk_comm_set_slot(ggml_tensor * slot, const ggml_tensor * src, ggml_type type, ggml_backend_buffer_t buf, size_t offs) { + slot->type = type; + slot->nb[0] = ggml_type_size(type); + for (int d = 0; d < GGML_MAX_DIMS; d++) { + slot->ne[d] = src->ne[d]; + if (d > 0) { + slot->nb[d] = slot->nb[d - 1] * slot->ne[d - 1]; + } + } + slot->buffer = buf; + slot->data = (char *) ggml_backend_buffer_get_base(buf) + offs; +} + +static void ggml_vk_comm_submit(ggml_backend_vk_context * ctx, vk_context & c, vk::Semaphore signal, uint64_t value) { + ggml_vk_ctx_end(c); + for (auto & cpy : c->in_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + for (auto & mset : c->memsets) { + memset(mset.dst, mset.val, mset.n); + } + c->in_memcpys.clear(); + c->memsets.clear(); + if (signal) { + c->seqs.back().back().signal_semaphores.push_back({ signal, value }); + } + ggml_vk_submit(c, {}); + ctx->submit_pending = true; + ctx->compute_ctx.reset(); +} + +static void ggml_vk_comm_notify_submitted(ggml_backend_vk_comm_context * comm, uint64_t round) { + comm->submitted.store(round); + if (comm->sleeping.load()) { + std::lock_guard<std::mutex> lock(comm->mutex); + comm->cv.notify_one(); + } +} + +// false on stop +static bool ggml_vk_comm_wait_submitted(ggml_backend_vk_comm_context * comm, uint64_t r) { + using clock = std::chrono::steady_clock; + for (const auto t0 = clock::now(); comm->submitted.load() < r;) { + if (comm->stop.load()) { + return false; + } + if (clock::now() - t0 < comm->spin) { + YIELD(); + continue; + } + comm->sleeping.store(true); + if (comm->submitted.load() < r) { + std::unique_lock<std::mutex> lock(comm->mutex); + comm->cv.wait(lock, [&] { return comm->submitted.load() >= r || comm->stop.load(); }); + } + comm->sleeping.store(false); + } + return true; +} + +// signals go[i] once all peers of i are done; false on stop +static bool ggml_vk_comm_bridge_release(ggml_backend_vk_comm_context * comm, uint64_t r, std::vector<uint64_t> & seen, std::vector<bool> & sent) { + using clock = std::chrono::steady_clock; + const size_t n = comm->devs.size(); + std::fill(sent.begin(), sent.end(), false); + size_t left = n; + for (const auto t0 = clock::now();;) { + size_t pending = n; + for (size_t i = 0; i < n; i++) { + if (sent[i]) { + continue; + } + bool ready = true; + for (size_t j = 0; j < n && ready; j++) { + if (j != i && seen[j] < r) { + const vk_comm_device & p = comm->devs[j]; + seen[j] = p.ctx->device->device.getSemaphoreCounterValue(p.bridge.done); + ready = seen[j] >= r; + pending = ready ? pending : j; + } + } + if (ready) { + const vk_comm_device & d = comm->devs[i]; + d.ctx->device->device.signalSemaphore({ d.bridge.go, r }); + sent[i] = true; + left--; + } + } + if (left == 0) { + return true; + } + if (clock::now() - t0 < comm->spin) { + YIELD(); + continue; + } + const vk_comm_device & p = comm->devs[pending]; + const vk::SemaphoreWaitInfo wi{ {}, 1, &p.bridge.done, &r }; + while (p.ctx->device->device.waitSemaphores(wi, 100'000'000) != vk::Result::eSuccess) { + if (comm->stop.load()) { + return false; + } + } + } +} + +static void ggml_vk_comm_bridge_thread(ggml_backend_vk_comm_context * comm) { +#if defined(__linux__) + pthread_setname_np(pthread_self(), "ggml-vk-comm"); +#endif + std::vector<uint64_t> seen(comm->devs.size(), 0); + std::vector<bool> sent(comm->devs.size()); + try { + for (uint64_t r = 1;; r++) { + if (!ggml_vk_comm_wait_submitted(comm, r) || !ggml_vk_comm_bridge_release(comm, r, seen, sent)) { + return; + } + } + } catch (const vk::SystemError & e) { + GGML_ABORT("ggml_vulkan: all-reduce bridge failed: %s", e.what()); + } +} + +static void ggml_vk_comm_bridge_init(ggml_backend_vk_comm_context * comm) { + vk::SemaphoreTypeCreateInfo stci{ vk::SemaphoreType::eTimeline, 0 }; + vk::SemaphoreCreateInfo sci{}; + sci.pNext = &stci; + for (vk_comm_device & d : comm->devs) { + d.bridge.done = d.ctx->device->device.createSemaphore(sci); + d.bridge.go = d.ctx->device->device.createSemaphore(sci); + } + comm->thread = std::thread(ggml_vk_comm_bridge_thread, comm); +} + +static void ggml_vk_comm_bridge_free(vk_comm_device & d) { + if (d.bridge.done) { + d.ctx->device->device.destroySemaphore(d.bridge.done); + } + if (d.bridge.go) { + d.ctx->device->device.destroySemaphore(d.bridge.go); + } +} + +static void ggml_vk_comm_barrier_to_host(vk_context & c) { + const vk::MemoryBarrier to_host{ vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eShaderWrite, vk::AccessFlagBits::eHostRead }; + c->s->buffer->buf.pipelineBarrier(c->p->q->stage_flags, vk::PipelineStageFlagBits::eHost, {}, { to_host }, {}, {}); +} + +// the peers' slots were made available to the host before the wait resolved, the submit predates that +static void ggml_vk_comm_barrier_from_host(vk_context & c) { + const vk::MemoryBarrier from_host{ vk::AccessFlagBits::eHostWrite, vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eTransferRead }; + c->s->buffer->buf.pipelineBarrier(vk::PipelineStageFlagBits::eHost, c->p->q->stage_flags, {}, { from_host }, {}, {}); +} + +static void ggml_vk_comm_bridge_signal(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) { + const vk_comm_device & d = comm->devs[i]; + ggml_vk_comm_barrier_to_host(c); + ggml_vk_comm_submit(d.ctx, c, d.bridge.done, round); +} + +static void ggml_vk_comm_bridge_wait(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) { + c->s->wait_semaphores.push_back({ comm->devs[i].bridge.go, round }); + ggml_vk_comm_barrier_from_host(c); +} + +static bool ggml_vk_comm_sfd_supported(vk_comm_device & d) { + const vk_device & dev = d.ctx->device; + const bool mesa = dev->driver_id == vk::DriverId::eMesaRadv || + dev->driver_id == vk::DriverId::eIntelOpenSourceMESA || + dev->driver_id == vk::DriverId::eMesaNvk; + if (!mesa || !dev->external_semaphore) { + return false; + } + d.sfd.get_fd = (PFN_vkGetSemaphoreFdKHR) dev->device.getProcAddr("vkGetSemaphoreFdKHR"); + d.sfd.import_fd = (PFN_vkImportSemaphoreFdKHR) dev->device.getProcAddr("vkImportSemaphoreFdKHR"); + if (!d.sfd.get_fd || !d.sfd.import_fd) { + return false; + } + vk::PhysicalDeviceExternalSemaphoreInfo esi{ vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd }; + vk::ExternalSemaphoreProperties esp = dev->physical_device.getExternalSemaphoreProperties(esi); + const vk::ExternalSemaphoreFeatureFlags need = + vk::ExternalSemaphoreFeatureFlagBits::eExportable | vk::ExternalSemaphoreFeatureFlagBits::eImportable; + return (esp.externalSemaphoreFeatures & need) == need && + (esp.compatibleHandleTypes & vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd); +} + +// Mesa moves a queue to its submit thread on the first wait-before-signal, taking the CS ioctls off this thread +static void ggml_vk_comm_sfd_enable_submit_thread(const vk_device & dev) { + vk::SemaphoreTypeCreateInfo stci{ vk::SemaphoreType::eTimeline, 0 }; + vk::SemaphoreCreateInfo sci{}; + sci.pNext = &stci; + const vk::Semaphore sem = dev->device.createSemaphore(sci); + const vk::Fence fence = dev->device.createFence({}); + const uint64_t one = 1; + vk::TimelineSemaphoreSubmitInfo tsi{ 1, &one, 0, nullptr }; + const vk::PipelineStageFlags stage = vk::PipelineStageFlagBits::eAllCommands; + vk::SubmitInfo si{ 1, &sem, &stage, 0, nullptr, 0, nullptr }; + si.setPNext(&tsi); + dev->compute_queue->handle->submit(si, fence); + dev->device.signalSemaphore({ sem, 1 }); + VK_CHECK(dev->device.waitForFences({ fence }, true, UINT64_MAX), "ggml_vk_comm_sfd_enable_submit_thread", dev); + dev->device.destroyFence(fence); + dev->device.destroySemaphore(sem); +} + +// Requires a submitted signal on sem; the export resets it. +static int ggml_vk_comm_sfd_export(vk_comm_device & d) { + int fd = -1; + VkSemaphoreGetFdInfoKHR gi{}; + gi.sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR; + gi.semaphore = d.sfd.sem; + gi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; + const VkResult result = d.sfd.get_fd((VkDevice) d.ctx->device->device, &gi, &fd); + if (result != VK_SUCCESS) { + GGML_ABORT("ggml_vulkan: vkGetSemaphoreFdKHR failed: %d", (int) result); + } + return fd; +} + +// fd -1 imports an already signalled payload +static void ggml_vk_comm_sfd_import(vk_comm_device & d, vk::Semaphore sem, int fd) { + VkImportSemaphoreFdInfoKHR isi{}; + isi.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR; + isi.semaphore = sem; + isi.flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT; + isi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; + isi.fd = fd; + const VkResult result = d.sfd.import_fd((VkDevice) d.ctx->device->device, &isi); + if (result != VK_SUCCESS) { + GGML_ABORT("ggml_vulkan: vkImportSemaphoreFdKHR failed: %d", (int) result); + } +} + +static void ggml_vk_comm_sfd_exchange(ggml_backend_vk_comm_context * comm, uint64_t r) { + const size_t n = comm->devs.size(); + for (size_t j = 0; j < n; j++) { + const int fd = ggml_vk_comm_sfd_export(comm->devs[j]); + const size_t last = j == n - 1 ? n - 2 : n - 1; + for (size_t i = 0; i < n; i++) { + if (i == j) { + continue; + } + const int imp_fd = fd < 0 || i == last ? fd : fcntl(fd, F_DUPFD_CLOEXEC, 0); + GGML_ASSERT(imp_fd >= 0 || fd < 0); + ggml_vk_comm_sfd_import(comm->devs[i], comm->devs[i].sfd.imp[j], imp_fd); + } + } + comm->imported.store(r, std::memory_order_release); +} + +static void ggml_vk_comm_sfd_thread(ggml_backend_vk_comm_context * comm) { +#if defined(__linux__) + pthread_setname_np(pthread_self(), "ggml-vk-comm"); +#endif + for (uint64_t r = 1; ggml_vk_comm_wait_submitted(comm, r); r++) { + ggml_vk_comm_sfd_exchange(comm, r); + } +} + +static void ggml_vk_comm_sfd_init(ggml_backend_vk_comm_context * comm) { + vk::ExportSemaphoreCreateInfo esci{ vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd }; + vk::SemaphoreCreateInfo sci{}; + sci.pNext = &esci; + for (vk_comm_device & d : comm->devs) { + d.sfd.sem = d.ctx->device->device.createSemaphore(sci); + d.sfd.imp.assign(comm->devs.size(), {}); + ggml_vk_comm_sfd_enable_submit_thread(d.ctx->device); + } + comm->thread = std::thread(ggml_vk_comm_sfd_thread, comm); +} + +static void ggml_vk_comm_sfd_free(vk_comm_device & d) { + if (d.sfd.sem) { + d.ctx->device->device.destroySemaphore(d.sfd.sem); + } +} + +static void ggml_vk_comm_sfd_signal(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c) { + vk_comm_device & d = comm->devs[i]; + ggml_vk_comm_barrier_to_host(c); + ggml_vk_comm_submit(d.ctx, c, d.sfd.sem, 0); + for (size_t j = 0; j < comm->devs.size(); j++) { + d.sfd.imp[j] = j == i ? vk::Semaphore{} : ggml_vk_create_binary_semaphore(d.ctx); + } +} + +static void ggml_vk_comm_sfd_wait(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) { + for (size_t j = 0; j < comm->devs.size(); j++) { + if (j != i) { + c->s->wait_semaphores.push_back({ comm->devs[i].sfd.imp[j], 0 }); + } + } + c->s->ready = &comm->imported; + c->s->ready_value = round; + ggml_vk_comm_barrier_from_host(c); +} + +void ggml_backend_vk_comm_free(void * comm_ctx) { + ggml_backend_vk_comm_context * comm = static_cast<ggml_backend_vk_comm_context *>(comm_ctx); + // pending reductions need the thread + for (vk_comm_device & d : comm->devs) { + ggml_backend_synchronize(d.backend); + } + if (comm->thread.joinable()) { + { + std::lock_guard<std::mutex> lock(comm->mutex); + comm->stop.store(true); + } + comm->cv.notify_all(); + comm->thread.join(); + } + for (vk_comm_device & d : comm->devs) { + ggml_vk_comm_sfd_free(d); + ggml_vk_comm_bridge_free(d); + } + ggml_vk_comm_free_buffers(comm); + if (comm->tctx) { + ggml_free(comm->tctx); + } + delete comm; +} + +void * ggml_backend_vk_comm_init(ggml_backend_t * backends, size_t n_backends) { + if (vk_instance.allreduce_disabled || n_backends < 2) { + return nullptr; + } + for (size_t i = 0; i < n_backends; i++) { + if (!ggml_backend_is_vk(backends[i])) { + return nullptr; + } + } + ggml_backend_vk_comm_context * comm = new ggml_backend_vk_comm_context; + comm->devs.resize(n_backends); + for (size_t i = 0; i < n_backends; i++) { + vk_comm_device & d = comm->devs[i]; + d.backend = backends[i]; + d.ctx = (ggml_backend_vk_context *) backends[i]->context; + d.host_buf.assign(n_backends, nullptr); + } + + for (size_t i = 0; i < n_backends; i++) { + const vk_device & dev = comm->devs[i].ctx->device; + comm->align = std::max(comm->align, (size_t) dev->min_imported_host_pointer_alignment); + if (!dev->external_memory_host) { + GGML_LOG_WARN("ggml_vulkan: device %zu: VK_EXT_external_memory_host missing, using generic all-reduce\n", i); + ggml_backend_vk_comm_free(comm); + return nullptr; + } + if (dev->serialize_submissions) { + GGML_LOG_WARN("ggml_vulkan: device %zu: GGML_VK_SERIALIZE_SUBMISSIONS set, using generic all-reduce\n", i); + ggml_backend_vk_comm_free(comm); + return nullptr; + } + } + + if (std::all_of(comm->devs.begin(), comm->devs.end(), ggml_vk_comm_sfd_supported)) { + comm->mode = vk_comm_sync::sync_fd; + } + + if (const char * spin = getenv("GGML_VK_ALLREDUCE_SPIN_US")) { + comm->spin = std::chrono::microseconds(std::max(0, atoi(spin))); + } + const ggml_init_params ip = { ggml_tensor_overhead(), nullptr, true }; + comm->tctx = ggml_init(ip); + comm->slot = ggml_new_tensor_1d(comm->tctx, GGML_TYPE_F32, 1); + + try { + if (comm->mode == vk_comm_sync::sync_fd) { + ggml_vk_comm_sfd_init(comm); + } else { + ggml_vk_comm_bridge_init(comm); + } + } catch (const vk::SystemError & e) { + GGML_LOG_WARN("ggml_vulkan: semaphore initialization failed: %s, using generic all-reduce\n", e.what()); + ggml_backend_vk_comm_free(comm); + return nullptr; + } + return comm; +} + +// A: partial to own host slot (round & 1), signal; B: wait on peers' A, add their slots; all A precede any B +// slot reuse: A(k) follows own B(k-1), which waited on peers' A(k-1), submitted after their B(k-2), the last reader +bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tensors) { + ggml_backend_vk_comm_context * comm = static_cast<ggml_backend_vk_comm_context *>(comm_ctx); + const size_t n = comm->devs.size(); + + const int64_t ne = ggml_nelements(tensors[0]); + for (size_t i = 0; i < n; i++) { + const ggml_tensor * t = tensors[i]; + if (t->type != GGML_TYPE_F32 || ggml_nelements(t) != ne || !ggml_is_contiguous(t) || !ggml_backend_buffer_is_vk(t->buffer)) { + return false; + } + const ggml_backend_vk_context * ctx = comm->devs[i].ctx; + if (!ctx->compute_ctx.expired() && !ctx->compute_ctx.lock()->out_memcpys.empty()) { + return false; + } + } + if (ne == 0) { + return true; + } + const size_t nbytes = ggml_nbytes(tensors[0]); + if (!ggml_backend_vk_comm_ensure(comm, nbytes)) { + return false; + } + + const bool f32_wire = nbytes < ggml_vk_comm_f16_min_size || ggml_vk_comm_needs_f32(tensors[0]); + const ggml_type wire = f32_wire ? GGML_TYPE_F32 : GGML_TYPE_F16; + const uint64_t round = ++comm->round; + const size_t slot_offs = (round & 1) * comm->cap; + const size_t slot_size = ne * ggml_type_size(wire); + + for (size_t i = 0; i < n; i++) { + const vk_comm_device & d = comm->devs[i]; + ggml_backend_vk_context * ctx = d.ctx; + ggml_tensor * t = tensors[i]; + vk_buffer & tbuf = ((ggml_backend_vk_buffer_context *) t->buffer->context)->dev_buffer; + vk_buffer & hbuf = ((ggml_backend_vk_buffer_context *) d.host_buf[i]->context)->dev_buffer; + const size_t toffs = vk_tensor_offset(t) + t->view_offs; + + vk_context c = ggml_vk_get_compute_ctx(ctx); + ggml_vk_sync_buffers(ctx, c); + if (!(t->flags & GGML_TENSOR_FLAG_COMPUTE)) { + c->s->buffer->buf.fillBuffer(hbuf->buffer, slot_offs, GGML_PAD(slot_size, 4), 0); + c->s->buffer->buf.fillBuffer(tbuf->buffer, toffs, nbytes, 0); + } else if (wire == GGML_TYPE_F32) { + ggml_vk_buffer_copy_async(c, hbuf, slot_offs, tbuf, toffs, nbytes); + } else { + ggml_vk_comm_set_slot(comm->slot, t, wire, d.host_buf[i], slot_offs); + ggml_vk_cpy(ctx, c, t, comm->slot); + } + if (comm->mode == vk_comm_sync::sync_fd) { + ggml_vk_comm_sfd_signal(comm, i, c); + } else { + ggml_vk_comm_bridge_signal(comm, i, c, round); + } + } + ggml_vk_comm_notify_submitted(comm, round); + + for (size_t i = 0; i < n; i++) { + ggml_backend_vk_context * ctx = comm->devs[i].ctx; + vk_context c = ggml_vk_get_compute_ctx(ctx); + ctx->do_add_rms_partials = false; + if (comm->mode == vk_comm_sync::sync_fd) { + ggml_vk_comm_sfd_wait(comm, i, c, round); + } else { + ggml_vk_comm_bridge_wait(comm, i, c, round); + } + ggml_vk_sync_buffers(ctx, c); + for (size_t j = 0; j < n; j++) { + if (j == i) { + continue; + } + ggml_vk_comm_set_slot(comm->slot, tensors[i], wire, comm->devs[j].host_buf[i], slot_offs); + ggml_vk_add(ctx, c, tensors[i], comm->slot, tensors[i]); + ggml_vk_sync_buffers(ctx, c); + } + // small reductions go out with the next graph's first submit + if (wire != GGML_TYPE_F32 || vk_perf_logger_enabled) { + ggml_vk_comm_submit(ctx, c, {}, 0); + } + } + return true; +} +#endif diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h index a642c5516..4e05fb818 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h @@ -1,6 +1,10 @@ #pragma once #include "ggml-vulkan-push-constants.h" +inline std::ostream & operator<<(std::ostream & os, vk::Buffer buffer) { + return os << static_cast<VkBuffer>(buffer); +} + // shared globals extern ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface; extern bool vk_memory_logger_enabled; @@ -41,8 +45,8 @@ bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_ vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)); vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size); void ggml_vk_destroy_buffer(vk_buffer& buf); -void * ggml_vk_host_malloc(vk_device& device, size_t size); -void ggml_vk_host_free(vk_device& device, void* ptr); +void * ggml_vk_host_malloc(size_t size); +void ggml_vk_host_free(void * ptr); void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset); void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size); void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size); @@ -55,6 +59,12 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size); void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size); void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size); +size_t ggml_vk_buffer_write_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging = nullptr, size_t * staging_offset = nullptr); +size_t ggml_vk_buffer_read_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging = nullptr, size_t * staging_offset = nullptr); +void ggml_vk_buffer_copy_n_async(vk_context& subctx, const vk_buffer_copy_region * regions, size_t n); +void ggml_vk_buffer_write_n(const vk_buffer_region * regions, size_t n); +void ggml_vk_buffer_read_n(const vk_buffer_region * regions, size_t n); +void ggml_vk_buffer_copy_n(const vk_buffer_copy_region * regions, size_t n); void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size); void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size); vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size); @@ -82,6 +92,7 @@ void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name); void ggml_vk_ctx_end(vk_context& ctx); void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx); vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx); +vk::Semaphore ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx); vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx); bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx); size_t ggml_vk_align_size(size_t width, size_t align); @@ -90,7 +101,7 @@ void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_stagi // matmul vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type); -void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n); +void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes); bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor); vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to); vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type); @@ -161,7 +172,6 @@ void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const gg void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); -void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst); void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx); @@ -222,6 +232,9 @@ void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_ten void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size); void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst); +void ggml_backend_vk_buffer_set_tensor_n(ggml_backend_buffer_t buffer, ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); +void ggml_backend_vk_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); +bool ggml_backend_vk_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * srcs, ggml_tensor * const * dsts, size_t n_tensors); void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value); const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft); ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size); @@ -229,9 +242,23 @@ size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft); size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor); void ggml_backend_vk_free(ggml_backend_t backend); +ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size); ggml_backend_reg_t ggml_backend_vk_reg(); +// comm +#if !defined(_WIN32) +void * ggml_backend_vk_comm_init(ggml_backend_t * backends, size_t n_backends); +void ggml_backend_vk_comm_free(void * comm_ctx); +bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tensors); +#endif + // debug +void ggml_vk_count_fusion(const vk_device & device, const char * label); +void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev); +void ggml_backend_vk_fusion_stats_init(void * finfo); +void ggml_backend_vk_fusion_stats_reset(void * finfo); +int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n); +void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled); int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op); #ifdef GGML_VULKAN_RUN_TESTS template <typename X_TYPE, typename Y_TYPE> diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp index 8d8fe6592..2c9c09c06 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp @@ -12,6 +12,58 @@ uint32_t vk_perf_logger_frequency = 1; std::string vk_pipeline_stats_filter; +void ggml_vk_count_fusion(const vk_device & device, const char * label) { + if (!device->fusion_stats_enabled.load(std::memory_order_relaxed)) { + return; + } + std::lock_guard<std::mutex> lock(device->fusion_mutex); + device->fusion_counts[label]++; +} + +void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev) { + auto * ctx = (ggml_backend_vk_device_context *) dev->context; + return ggml_vk_get_device(ctx->device).get(); +} + +void ggml_backend_vk_fusion_stats_init(void * finfo) { + auto * device = (vk_device_struct *) finfo; + device->fusion_stats_enabled.store(true, std::memory_order_relaxed); +} + +void ggml_backend_vk_fusion_stats_reset(void * finfo) { + auto * device = (vk_device_struct *) finfo; + std::lock_guard<std::mutex> lock(device->fusion_mutex); + // Keep labels and their order stable between fused and unfused runs. + for (auto & entry : device->fusion_counts) { + entry.second = 0; + } +} + +int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n) { + auto * device = (vk_device_struct *) finfo; + std::lock_guard<std::mutex> lock(device->fusion_mutex); + if (labels == nullptr) { + return (int) device->fusion_counts.size(); + } + int i = 0; + for (const auto & entry : device->fusion_counts) { + if (i >= n) { + break; + } + labels[i] = entry.first.c_str(); + if (counts) { + counts[i] = entry.second; + } + i++; + } + return i; +} + +void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled) { + auto * device = (vk_device_struct *) finfo; + device->disable_fusion = !enabled; +} + void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) { if (!vk_memory_logger_enabled) { return; @@ -173,7 +225,8 @@ void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k), m, n, k, k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1, n + split_k, batch, batch, batch, 1, 1, n, + k * sizeof(Y_TYPE), m * sizeof(float) ); } ggml_vk_ctx_end(subctx); @@ -640,7 +693,8 @@ void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k }, m, n, k, k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1, n + split_k, batch, batch, batch, 1, 1, n, + (uint64_t)(k / ggml_blck_size(GGML_TYPE_Q8_1)) * ggml_type_size(GGML_TYPE_Q8_1), m * sizeof(float) ); } } else { @@ -649,7 +703,8 @@ void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k }, m, n, k, k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1, n + split_k, batch, batch, batch, 1, 1, n, + k * sizeof(float), m * sizeof(float) ); } } @@ -980,8 +1035,6 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph } else if (tensor->op == GGML_OP_SCALE) { const float * params = (const float *)tensor->op_params; tensor_clone = ggml_scale_bias(ggml_ctx, src_clone[0], params[0], params[1]); - } else if (tensor->op == GGML_OP_ADD1) { - tensor_clone = ggml_add1(ggml_ctx, src_clone[0], src_clone[1]); } else if (tensor->op == GGML_OP_ARANGE) { const float start = ggml_get_op_params_f32(tensor, 0); const float stop = ggml_get_op_params_f32(tensor, 1); @@ -1046,12 +1099,9 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph } } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) { tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); - } else if (tensor->op == GGML_OP_DIAG_MASK_INF) { - tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], tensor->op_params[0]); } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) { const int n_dims = ((int32_t *) tensor->op_params)[1]; const int mode = ((int32_t *) tensor->op_params)[2]; - //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3]; const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4]; const float freq_base = ((float *) tensor->op_params)[5]; const float freq_scale = ((float *) tensor->op_params)[6]; @@ -1073,6 +1123,7 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); } } + ggml_rope_set_truncate(tensor_clone, ggml_rope_get_truncate(tensor)); const int n_offs = ((int32_t *) tensor->op_params)[15]; if (n_offs != 0) { tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h index f1f66a628..1ef00477b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h @@ -63,7 +63,7 @@ struct vk_mat_mat_id_push_constants { uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11; uint32_t n_experts; - uint32_t hoist_row_ids; + uint32_t row_ids_offset; }; struct vk_mat_vec_id_push_constants { @@ -124,22 +124,30 @@ struct vk_flash_attn_push_constants { static_assert(sizeof(vk_flash_attn_push_constants) <= 128, "sizeof(vk_flash_attn_push_constants) must be <= 128"); -struct vk_fa_xe_opt_push_constants { - uint32_t kv_seq_len; - uint32_t activation_length; - uint32_t q_head; - uint32_t kv_head; - uint32_t qk_ratio; - uint32_t qk_sub_groups; +struct vk_xe_fa_push_constants { + uint32_t neq1; + uint32_t neq2; + uint32_t neq3; + uint32_t nek1; + uint32_t nek2; + uint32_t nek3; + uint32_t nev2; + uint32_t nev3; + uint32_t nem1; + uint32_t nem2; + uint32_t nem3; + uint32_t nbq1; + uint32_t nbq2; + uint32_t nbk1; + uint32_t nbk2; + uint32_t nbv1; + uint32_t nbv2; + uint32_t nbq3; + uint32_t nbk3; + uint32_t nbv3; + uint32_t nb3; uint32_t flag; - uint32_t nbkv_tok; - uint32_t nbkv_head; - uint32_t batch_stride_q; - uint32_t batch_stride_k; - uint32_t batch_stride_v; - uint32_t batch_stride_m; - uint32_t batch_stride_o; - float softmax_scale; + float scale; }; struct vk_op_push_constants { @@ -218,9 +226,13 @@ struct vk_op_count_experts_push_constants { uint32_t nb01; uint32_t a_offset; uint32_t n_experts; - uint32_t hoist_row_ids; uint32_t ne00mp; uint32_t ne00L; + uint32_t row_tile_size; + uint32_t row_ids_offset; + uint32_t ne0_dst; + uint32_t stride_dst; + uint32_t batch_stride_dst; }; struct vk_op_glu_push_constants { @@ -450,12 +462,6 @@ struct vk_op_add_id_push_constants { uint32_t s21; }; -struct vk_op_diag_mask_push_constants { - uint32_t ncols; - uint32_t rows_per_channel; - int32_t n_past; -}; - struct vk_op_rope_push_constants { uint32_t rope_mode; uint32_t nrows; @@ -825,7 +831,7 @@ struct vk_op_upscale_push_constants { struct vk_op_sum_rows_push_constants { uint32_t n_cols; - uint32_t ne01, ne02; + uint32_t ne01, ne02, ne03; uint32_t nb01, nb02, nb03; uint32_t nb11, nb12, nb13; float weight; @@ -840,6 +846,7 @@ static vk_op_sum_rows_push_constants vk_op_sum_rows_push_constants_init(const gg p.n_cols = (uint32_t)n_cols; p.ne01 = (uint32_t)src->ne[1]; p.ne02 = (uint32_t)src->ne[2]; + p.ne03 = (uint32_t)src->ne[3]; p.nb01 = (uint32_t)src->nb[1] / type_size; p.nb02 = (uint32_t)src->nb[2] / type_size; p.nb03 = (uint32_t)src->nb[3] / type_size; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index 4158ff779..81fd5b8a8 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -48,43 +48,24 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV { #endif #include <algorithm> - #include <cmath> - #include <functional> - #include <iomanip> - #include <iostream> - #include <tuple> - #include <vector> - #include <deque> - #include <sstream> - #include <utility> - #include <memory> - #include <limits> - #include <map> - #include <set> - #include <unordered_map> - #include <shared_mutex> - #include <mutex> - #include <future> - #include <condition_variable> - #include <thread> #if defined(_MSC_VER) @@ -110,9 +91,7 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV { #endif #include "ggml-impl.h" - #include "ggml-backend-impl.h" - #include "ggml-vulkan-shaders.hpp" #if !defined(VK_KHR_shader_bfloat16) @@ -403,6 +382,7 @@ enum vk_conv_shapes { CONV_SHAPE_64x32, CONV_SHAPE_32x256, CONV_SHAPE_64x128, + CONV_SHAPE_256x128, CONV_SHAPE_COUNT, }; @@ -418,6 +398,7 @@ inline vk_conv_block_size vk_conv_block_sizes[CONV_SHAPE_COUNT] = { { 64, 32, 32 }, // CONV_SHAPE_64x32 { 32, 256, 16 }, // CONV_SHAPE_32x256 { 64, 128, 16 }, // CONV_SHAPE_64x128 + { 256, 128, 16 }, // CONV_SHAPE_256x128 }; enum dmmv_wg_sizes { @@ -452,6 +433,21 @@ struct vk_fa_pipeline_state { } }; +struct vk_xe_fa_pipeline_state { + uint32_t HSK, HSV; + uint32_t qk_ratio, pv_ratio; + uint32_t decode_n; + bool unaligned_seq_len; + bool unaligned_q; + bool partial_row_mask; + bool multi_head_mask; + + bool operator<(const vk_xe_fa_pipeline_state &b) const { + return std::tie(HSK, HSV, qk_ratio, pv_ratio, decode_n, unaligned_seq_len, unaligned_q, partial_row_mask, multi_head_mask) < + std::tie(b.HSK, b.HSV, b.qk_ratio, b.pv_ratio, b.decode_n, b.unaligned_seq_len, b.unaligned_q, b.partial_row_mask, b.multi_head_mask); + } +}; + struct vk_conv2d_pipeline_state { vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned) : s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {} @@ -505,6 +501,15 @@ static constexpr uint32_t num_topk_moe_pipelines = 10; static constexpr uint32_t num_topk_pipelines = 11; +// FWHT block widths with a dedicated pipeline. The subgroup shader keeps n/subgroup_size +// values per invocation, so widths above fwht_max_subgroup_n, or a subgroup narrow enough to +// exceed fwht_max_subgroup_el_w values, use the shared-memory shader with one row per +// workgroup instead. +static constexpr uint32_t num_fwht_pipelines = 8; +static constexpr uint32_t fwht_max_subgroup_n = 512; +static constexpr uint32_t fwht_max_subgroup_el_w = 64; +static constexpr uint32_t fwht_rows = 4; + static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, @@ -680,7 +685,6 @@ class vk_memory_logger; struct vk_device_struct { std::recursive_mutex mutex; std::mutex queue_submit_mutex; - mutable std::shared_mutex pinned_memory_mutex; // Guards compile_pending, all_pipelines, and the dynamic pipeline maps // (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile @@ -699,6 +703,7 @@ struct vk_device_struct { uint64_t suballocation_block_size; uint64_t min_imported_host_pointer_alignment; bool external_memory_host {}; + bool external_semaphore {}; bool fp16; bool bf16; bool pipeline_robustness; @@ -717,6 +722,7 @@ struct vk_device_struct { uint32_t subgroup_size_log2; uint32_t shader_core_count; bool uma; + bool rebar; bool prefer_host_memory; bool float_controls_rte_fp16; bool float_controls_denorm_preserve_fp16; @@ -772,6 +778,9 @@ struct vk_device_struct { bool dot2_f16 {}; bool ocp_fp4 {}; + bool workgroup_memory_explicit_layout {}; + bool workgroup_memory_explicit_layout_16bit {}; + bool pipeline_executable_properties_support {}; bool device_fault {}; @@ -846,6 +855,7 @@ struct vk_device_struct { vk_pipeline pipeline_add_id_f32; vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64; + vk_pipeline pipeline_concat_transpose_i8, pipeline_concat_transpose_i16, pipeline_concat_transpose_i32; vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32; vk_pipeline pipeline_scale_f32; vk_pipeline pipeline_log[2]; @@ -857,8 +867,8 @@ struct vk_device_struct { vk_pipeline pipeline_roll_f32; vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32; vk_pipeline pipeline_repeat_i16; - vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32; - vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32; + vk_pipeline pipeline_cpy_32, pipeline_cpy_f32_f16, pipeline_cpy_16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_bf16_f16, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32; + vk_pipeline pipeline_contig_cpy_32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_bf16_f16, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32; vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT]; vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT]; vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32; @@ -868,18 +878,18 @@ struct vk_device_struct { vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT]; vk_pipeline pipeline_norm_f32; vk_pipeline pipeline_group_norm_f32; - vk_pipeline pipeline_rms_norm_f32; - vk_pipeline pipeline_rms_norm_mul_f32; - vk_pipeline pipeline_rms_norm_mul_add_f32; - vk_pipeline pipeline_rms_norm_mul_add_mul_f32; + std::array<vk_pipeline, 3> pipeline_rms_norm_f32; + std::array<vk_pipeline, 3> pipeline_rms_norm_mul_f32; + std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_f32; + std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_mul_f32; vk_pipeline pipeline_rms_norm_mul_add_partials_f32; vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32; vk_pipeline pipeline_rms_norm_set_rows_f32_f32; vk_pipeline pipeline_rms_norm_set_rows_f32_f16; vk_pipeline pipeline_rms_norm_partials_f32; vk_pipeline pipeline_rms_norm_mul_partials_f32; - vk_pipeline pipeline_rms_norm_mul_rope_f32_f32; - vk_pipeline pipeline_rms_norm_mul_rope_f32_f16; + std::array<vk_pipeline, 3> pipeline_rms_norm_mul_rope_f32_f32; + std::array<vk_pipeline, 3> pipeline_rms_norm_mul_rope_f32_f16; vk_pipeline pipeline_rms_norm_back_f32; vk_pipeline pipeline_l2_norm_f32; @@ -933,7 +943,6 @@ struct vk_device_struct { vk_pipeline pipeline_leaky_relu[2]; vk_pipeline pipeline_silu_back_f32; - vk_pipeline pipeline_diag_mask_inf_f32; vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16; vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512; vk_pipeline pipeline_soft_max_back_f32; @@ -954,7 +963,9 @@ struct vk_device_struct { vk_pipeline pipeline_sum_rows_f32; vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512; vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512; - vk_pipeline pipeline_fwht_f32[4]; + vk_pipeline pipeline_fwht_f32[num_fwht_pipelines]; + vk_pipeline pipeline_fwht_f16[num_fwht_pipelines]; + uint32_t fwht_rows_per_wg[num_fwht_pipelines] = {}; vk_pipeline pipeline_cumsum_f32; vk_pipeline pipeline_cumsum_small_f32; vk_pipeline pipeline_cumsum_multipass1_f32; @@ -980,6 +991,7 @@ struct vk_device_struct { vk_pipeline pipeline_snake_bf16; vk_pipeline pipeline_pool1d_f32; vk_pipeline pipeline_pool2d_f32; + vk_pipeline pipeline_pool2d_subgroup_f32; vk_pipeline pipeline_rwkv_wkv6_f32; vk_pipeline pipeline_rwkv_wkv7_f32; vk_pipeline pipeline_gated_linear_attn_f32; @@ -1011,7 +1023,8 @@ struct vk_device_struct { bool fa_sparse_compact_use_subgroups; vk_pipeline pipeline_flash_attn_split_k_reduce; - std::map<std::tuple<uint32_t, uint32_t, uint32_t, uint32_t>, std::pair<vk_pipeline, vk_pipeline>> pipeline_xe_fa_decode_dual_phases; + std::map<vk_xe_fa_pipeline_state, std::pair<vk_pipeline, vk_pipeline>> pipeline_xe_fa_decode_dual_phases; + std::map<vk_xe_fa_pipeline_state, vk_pipeline> pipeline_xe_fa_prefill; vk_pipeline pipeline_count_experts; // [2] is for whether to take n_experts from spec constant (0) or push constant (1) @@ -1019,19 +1032,21 @@ struct vk_device_struct { std::vector<vk_pipeline_ref> all_pipelines; - std::vector<std::tuple<void*, size_t, vk_buffer>> pinned_memory; - vk::Fence fence; vk_buffer sync_staging; ggml_backend_buffer_type buffer_type; bool disable_fusion; + std::atomic<bool> fusion_stats_enabled {false}; + std::mutex fusion_mutex; + std::map<std::string, uint64_t> fusion_counts; bool disable_descriptor_reuse; std::atomic<uint64_t> buffer_destroy_count {}; bool disable_host_visible_vidmem; bool allow_sysmem_fallback; bool disable_graph_optimize; + bool xe_fa_support {}; std::unique_ptr<vk_memory_logger> memory_logger; @@ -1108,6 +1123,9 @@ struct vk_submission { vk_command_buffer* buffer = nullptr; std::vector<vk_semaphore> wait_semaphores; std::vector<vk_semaphore> signal_semaphores; + // submit waits until *ready >= ready_value + const std::atomic<uint64_t> * ready = nullptr; + uint64_t ready_value = 0; }; typedef std::vector<vk_submission> vk_sequence; @@ -1136,6 +1154,21 @@ struct vk_staging_memset { size_t n; }; +struct vk_buffer_region { + vk_buffer buf; + size_t offset; + void * host; + size_t size; +}; + +struct vk_buffer_copy_region { + vk_buffer dst; + size_t dst_offset; + vk_buffer src; + size_t src_offset; + size_t size; +}; + struct vk_context_struct { vk_submission * s; std::vector<vk_sequence> seqs; @@ -1246,7 +1279,7 @@ struct ggml_backend_vk_context { vk_device device; - size_t semaphore_idx, event_idx; + size_t semaphore_idx, binary_semaphore_idx, event_idx; ggml_vk_garbage_collector gc; size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset; vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging; @@ -1333,6 +1366,12 @@ struct ggml_backend_vk_buffer_context { }; +struct vk_host_allocation { + void * base = nullptr; + size_t size = 0; + vk_buffer views[GGML_VK_MAX_DEVICES]; // indexed by device->idx; null = not (yet) mapped here +}; + struct vk_instance_t { vk::Instance instance; @@ -1346,7 +1385,17 @@ struct vk_instance_t { std::vector<size_t> device_indices; std::vector<bool> device_supports_membudget; + std::vector<bool> device_host_buffer_import; vk_device devices[GGML_VK_MAX_DEVICES]; + + std::vector<vk_host_allocation> host_allocations; + std::shared_mutex host_allocations_mutex; + + size_t host_import_alignment = 0; + uint32_t host_import_failed = 0; // bitmask by device->idx, guarded by host_allocations_mutex + + bool multi_device = false; + bool allreduce_disabled = false; }; typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); @@ -1377,6 +1426,34 @@ struct vk_fa_tuning_params { } }; +struct vk_xe_fa_tuning_params { + size_t shmem_sz; + std::array<uint32_t, 2> workgroup_size; + std::array<uint32_t, 2> subgroup_size; + uint32_t tm; + uint32_t tn; + uint32_t tk; + + struct { + uint32_t wg_row; + uint32_t gqa_fold; + uint32_t q_in_reg; + uint32_t p_per_iter; + uint32_t sg_col; + uint32_t acc_count; + uint32_t pipeline_flag; + } prefill_params; + + struct { + uint32_t p_chunk_ph1; + uint32_t out_per_wg_ph1; + uint32_t split_p_per_iter_ph2; + uint32_t out_row_per_wg_ph2; + uint32_t out_col_per_wg_ph2; + uint32_t pipeline_flag_ph2; + } decode_params; +}; + struct GpuPipelineConfig { // GPU architecture identifier. // Example: vk_device_architecture::AMD_GCN diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 5587dfc28..22641f5fb 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1,10 +1,5 @@ #include "ggml-vulkan-common.h" -namespace { -inline std::ostream & operator<<(std::ostream & os, vk::Buffer buffer) { - return os << static_cast<VkBuffer>(buffer); -} -} static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& device) { vk::PhysicalDeviceProperties props = device.getProperties(); @@ -166,11 +161,13 @@ void ggml_vk_print_device_fault_info(const vk_device& device) { std::vector<VkDeviceFaultAddressInfoEXT> address_infos(fault_counts.addressInfoCount); std::vector<VkDeviceFaultVendorInfoEXT> vendor_infos(fault_counts.vendorInfoCount); + std::vector<uint8_t> vendor_binary(fault_counts.vendorBinarySize); VkDeviceFaultInfoEXT fault_info {}; fault_info.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT; fault_info.pAddressInfos = address_infos.data(); fault_info.pVendorInfos = vendor_infos.data(); + fault_info.pVendorBinaryData = vendor_binary.empty() ? nullptr : vendor_binary.data(); res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, &fault_info); if (res != VK_SUCCESS) { @@ -939,6 +936,11 @@ void ggml_vk_submit(vk_context& ctx, vk::Fence fence) { for (const auto& sequence : ctx->seqs) { for (const auto& submission : sequence) { + if (submission.ready) { + while (submission.ready->load(std::memory_order_acquire) < submission.ready_value) { + YIELD(); + } + } stage_flags.push_back({}); idx++; tl_wait_vals.push_back({}); @@ -1087,14 +1089,16 @@ vk_context ggml_vk_create_temporary_context(vk_command_pool& p) { return result; } -static vk_semaphore * ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) { - VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()"); - vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 }; - vk::SemaphoreCreateInfo ci{}; - ci.setPNext(&tci); - vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); - ctx->gc.semaphores.push_back({ semaphore, 0 }); - return &ctx->gc.semaphores[ctx->gc.semaphores.size() - 1]; +vk::Semaphore ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) { + VK_LOG_DEBUG("ggml_vk_create_binary_semaphore()"); + if (ctx->binary_semaphore_idx >= ctx->gc.semaphores.size()) { + vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 }; + vk::SemaphoreCreateInfo ci{}; + ci.setPNext(&tci); + vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); + ctx->gc.semaphores.push_back({ semaphore, 0 }); + } + return ctx->gc.semaphores[ctx->binary_semaphore_idx++].s; } static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context * ctx) { @@ -1258,6 +1262,12 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, result.block_rows /= 2; } + // With head sizes above 128, 16 rows per workgroup spill registers on Qualcomm and run about + // 40x slower; at large KV a single dispatch can then exceed the Windows GPU timeout. + if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && (hsk > 128 || hsv > 128)) { + result.block_rows = std::min(result.block_rows, 8u); + } + // On AMD RDNA, for small head sizes and big batch size the shader uses few registers, so too many subgroups get scheduled // at once and end up thrashing the cache. Fix this by setting a large (unused) shmem buffer that reduces occupancy. // This targets an occupancy of 4 subgroups per SIMD. @@ -1424,6 +1434,186 @@ static std::vector<uint32_t> get_fa_spec_constants(const vk_fa_pipeline_state& s }; } +static vk_xe_fa_tuning_params get_xe_fa_tuning_params(const vk_device_struct * device, const vk_xe_fa_pipeline_state& pipeline_state, bool use_prefill) { + vk_xe_fa_tuning_params params {}; + size_t shmem_sz = 0; + uint32_t sg_sz = 32u; + std::array<uint32_t, 3> matrix_mnk = { 16u, 16u, 16u }; + bool intel_gpu = device->vendor_id == VK_VENDOR_ID_INTEL; + + if (device->architecture == INTEL_XE1) { + sg_sz = 8u; + matrix_mnk = { 8u, 8u, 16u }; + } else if (device->architecture == INTEL_XE2) { + sg_sz = 16u; + matrix_mnk = { 8u, 16u, 16u }; + } + + uint32_t tm = matrix_mnk[0]; + uint32_t tn = matrix_mnk[1]; + uint32_t tk = matrix_mnk[2]; + uint32_t hsk = pipeline_state.HSK; + uint32_t hsv = pipeline_state.HSV; + uint32_t qk_ratio = pipeline_state.qk_ratio; + uint32_t pv_ratio = pipeline_state.pv_ratio; + uint32_t sg_per_gw; + + params.tm = tm; + params.tn = tn; + params.tk = tk; + GGML_ASSERT(hsv % tm == 0); + + if (use_prefill) { + sg_per_gw = sg_sz == 8 ? 32 : 16; + uint32_t wg_sz = sg_per_gw * sg_sz; + uint32_t output_div = hsk > 256 ? 8 : hsk > 160 ? 4 : hsk > 64 ? 2 : 1; + uint32_t p_per_iter = intel_gpu ? (hsk > 256 ? 16 : 32) : 16; + uint32_t q_in_reg = hsk > 512 ? hsk - 384 : std::min(128u, hsk); + uint32_t q_in_shmem = hsk - q_in_reg; + uint32_t wg_row = 256u / output_div; + uint32_t gqa_fold = (pipeline_state.multi_head_mask || qk_ratio != pv_ratio) ? 1 : 16; + uint32_t max_fold_rows = sg_sz <= 16 ? 16 : 8; + + while (gqa_fold > 1 && (qk_ratio % gqa_fold != 0 || gqa_fold * output_div > max_fold_rows)) { + gqa_fold = gqa_fold / 2; + } + + uint32_t mask_row = wg_row / gqa_fold; + uint32_t k_col = (hsk % 64 == 0 && hsk <= 512 && wg_sz <= 256) ? 8 : 4; + uint32_t k_loops = CEIL_DIV(hsk * p_per_iter, k_col * wg_sz); + + bool k_in_shmem = hsk <= 576u; + bool v_in_shmem = hsk <= 128u && hsv <= 128u; + bool emf_in_shmem = wg_row / matrix_mnk[1] != sg_per_gw; + bool mask_in_shmem = mask_row <= 64; + size_t shmem_q_sz = q_in_shmem * wg_row * sizeof(ggml_fp16_t); + size_t shmem_oob_staging_sz = sg_per_gw * tm * tk * sizeof(ggml_fp16_t); + size_t shmem_mask_sz = mask_in_shmem ? (mask_row * p_per_iter * sizeof(ggml_fp16_t)) : 0; + size_t shmem_k_sz = k_in_shmem ? (k_loops * k_col * wg_sz * sizeof(ggml_fp16_t)) : shmem_oob_staging_sz; + size_t shmem_v_sz = v_in_shmem ? (hsv * p_per_iter * sizeof(ggml_fp16_t)) : 0; + size_t shmem_p_sz = wg_row * p_per_iter * sizeof(float); + size_t shmem_emf_sz = emf_in_shmem ? (wg_row * sizeof(float)) : 0; + size_t shmem_pv_sz = intel_gpu ? 0 : (sg_per_gw * tm * tn * sizeof(float)); + + // Reuse section sits at tail of shmem, reused with k + mask/kv oob staging/pv staging + size_t shmem_reuse_sz = shmem_k_sz + shmem_mask_sz; + // When v is not in shmem, its staging reuses the k + mask region; size the reuse section to the larger of the two + shmem_reuse_sz = v_in_shmem ? shmem_reuse_sz : std::max(shmem_reuse_sz, shmem_oob_staging_sz); + shmem_reuse_sz = std::max(shmem_pv_sz, shmem_reuse_sz); + + // Q shmem size check for prologue staging is safe to bypass: + // 1. Prologue q staging shmem is released right after prologue, so it never overlaps other shmem sections + // 2. Extreme large hsk one-time q staging is capped within shader (max_q_reg_staging = shmem_size / 2 / Wr) + shmem_sz = shmem_q_sz + shmem_reuse_sz + shmem_v_sz + shmem_p_sz + shmem_emf_sz; + + uint32_t sg_col = hsv / tm; + uint32_t pipeline_flag = 0; + sg_col = (sg_col + output_div - 1) / output_div; + sg_col = sg_col * tm; + if (pipeline_state.unaligned_seq_len) { + pipeline_flag |= 1 << 0; + } + + if (pipeline_state.unaligned_q) { + pipeline_flag |= 1 << 1; + } + + if (pipeline_state.partial_row_mask) { + pipeline_flag |= 1 << 2; + } + + if (intel_gpu) { + pipeline_flag |= 1 << 3; + } + + params.workgroup_size = { wg_sz }; + params.subgroup_size = { sg_sz }; + params.shmem_sz = shmem_sz; + params.prefill_params.pipeline_flag = pipeline_flag; + params.prefill_params.wg_row = wg_row; + params.prefill_params.gqa_fold = gqa_fold; + params.prefill_params.q_in_reg = q_in_reg; + params.prefill_params.p_per_iter = p_per_iter; + params.prefill_params.sg_col = sg_col; + params.prefill_params.acc_count = tm == 8 ? (sg_col % 16 == 0 ? 2 : 1) : 1; + } else { + auto upper_power_of_2 = [&](uint32_t in) { + GGML_ASSERT(in != 0); + if (in <= 1) return 1u; + uint32_t ret = in - 1; + ret |= ret >> 1; + ret |= ret >> 2; + ret |= ret >> 4; + ret |= ret >> 8; + ret |= ret >> 16; + return ret + 1; + }; + + size_t shmem_ph1_sz = 0; + size_t shmem_ph2_sz = 0; + + // Phase 1 shmem size calculation + const uint32_t p_chunk_ph1 = 32; + // Phase 1 k shmem staging for extreme large hsk is capped within shader, bypass that check + uint32_t pmat_row = CEIL_DIV(qk_ratio, tn); + uint32_t aligned_q_len = upper_power_of_2(pipeline_state.decode_n); + uint32_t group_sz_ph1 = std::min(std::max(aligned_q_len * sg_sz, 64u), 256u); + uint32_t out_per_wg_ph1 = std::min(pipeline_state.decode_n, 256u / sg_sz); + size_t shmem_k_sz = group_sz_ph1 / sg_sz * tm * tk * sizeof(ggml_fp16_t); + size_t shmem_mask_sz = out_per_wg_ph1 * p_chunk_ph1 * sizeof(float); + size_t shmem_p_sz = p_chunk_ph1 * pmat_row * tn * sizeof(float); + // K shmem is reused with mask/p, calculate the max size + shmem_ph1_sz = std::max(shmem_k_sz, shmem_mask_sz + shmem_p_sz); + + // Phase 2 shmem size calculation + uint32_t aligned_pv_ratio = upper_power_of_2(pv_ratio); + uint32_t split_p_per_iter_ph2 = (pv_ratio <= 32) ? 256 : 128; + uint32_t out_col_per_wg_ph2 = intel_gpu ? (pv_ratio > 16 ? 8 : 16) : 16; + uint32_t split_p_per_warp = 16; + uint32_t group_sz_ph2 = (split_p_per_iter_ph2 / split_p_per_warp) * sg_sz; + sg_per_gw = group_sz_ph2 / sg_sz; + uint32_t out_row_per_wg_ph2 = std::min(std::max(16u / aligned_pv_ratio, 1u), pipeline_state.decode_n); + uint32_t o_row = pv_ratio * out_row_per_wg_ph2; + uint32_t omat_row = CEIL_DIV(o_row, tn) * tn; + uint32_t global_max_loops = CEIL_DIV(o_row, sg_per_gw); + size_t shmem_global_max_sz = CEIL_DIV(global_max_loops * sg_per_gw, sg_sz) * sg_sz * sizeof(float); + + uint32_t size_max = o_row * split_p_per_iter_ph2 / p_chunk_ph1; + size_t shmem_group_max_sz = CEIL_DIV(size_max, group_sz_ph2) * group_sz_ph2 * sizeof(float); + + uint32_t p_loop_col = group_sz_ph2 * 4; + uint32_t p_loops = CEIL_DIV(o_row * split_p_per_iter_ph2, p_loop_col); + shmem_p_sz = std::max(p_loop_col * p_loops * sizeof(ggml_fp16_t), omat_row * split_p_per_iter_ph2 * sizeof(ggml_fp16_t)); + + size_t shmem_lf_sz = CEIL_DIV(sg_per_gw * p_loops, sg_sz) * sg_sz * sizeof(float); + size_t shmem_oob_staging_sz = sg_per_gw * tm * tk * sizeof(ggml_fp16_t); + size_t shmem_o_sz = sg_per_gw * out_col_per_wg_ph2 * omat_row * sizeof(float); + size_t shmem_reuse_sz = 2 * shmem_group_max_sz + std::max(shmem_oob_staging_sz, 2 * shmem_p_sz); + + shmem_ph2_sz = shmem_lf_sz + shmem_global_max_sz; + // Shmem of p/group max is reused with output accumulation, calculate the max size + shmem_ph2_sz += std::max(shmem_reuse_sz, shmem_o_sz); + shmem_sz = std::max(shmem_ph1_sz, shmem_ph2_sz); + + params.workgroup_size = { group_sz_ph1, group_sz_ph2 }; + params.subgroup_size = { sg_sz, sg_sz }; + params.shmem_sz = shmem_sz; + params.decode_params.p_chunk_ph1 = p_chunk_ph1; + params.decode_params.out_per_wg_ph1 = out_per_wg_ph1; + params.decode_params.split_p_per_iter_ph2 = split_p_per_iter_ph2; + params.decode_params.out_row_per_wg_ph2 = out_row_per_wg_ph2; + params.decode_params.out_col_per_wg_ph2 = out_col_per_wg_ph2; + params.decode_params.pipeline_flag_ph2 = pipeline_state.unaligned_seq_len ? (1u << 0) : 0; + } + + if (shmem_sz > device->properties.limits.maxComputeSharedMemorySize) { + VK_LOG_DEBUG("get_xe_fa_tuning_params(HSK=" << hsk << ", HSV=" << hsv << ", qk_ratio=" << qk_ratio << ", pv_ratio=" << pv_ratio << "), " + "shmem_sz = " << shmem_sz << ", exceeding max shmem size = " << device->properties.limits.maxComputeSharedMemorySize); + } + + return params; +} + static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) { uint32_t lut_size = 0; @@ -1455,8 +1645,8 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec lut_size = 4*16; break; case GGML_TYPE_NVFP4: - // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[128] (types.glsl, DATA_A_NVFP4). - lut_size = 4*16 + 128u * (uint32_t)sizeof(float); + // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[256] (types.glsl, DATA_A_NVFP4). + lut_size = 4*16 + 256u * (uint32_t)sizeof(float); break; default: break; @@ -1473,9 +1663,8 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec const uint32_t load_bufs = (warptile[1] + warptile[2]) * (warptile[3] + bank_conflict_offset) * type_size; const uint32_t mmid_row_ids = mul_mat_id ? (warptile[2] * 2 * sizeof(uint16_t)) : 0; const uint32_t coopmat_stage = device->coopmat_support ? warptile[7] * warptile[8] / warps * sizeof(float) : 0; - const uint32_t ballots_sh = mul_mat_id ? (warps * 4 * sizeof(uint32_t)) : 0; - const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size + ballots_sh; + const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size; const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; VK_LOG_DEBUG("ggml_vk_matmul_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), " @@ -1540,10 +1729,7 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std: const uint32_t buf_b_size = BN * BK_STEP * block_b_size; const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u; - const uint32_t warps = warptile[0] / warptile[10]; - const uint32_t ballots_sh = mul_mat_id ? (warps * 4u * (uint32_t)sizeof(uint32_t)) : 0u; - - const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + ballots_sh + lut_size; + const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size; const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), " @@ -1573,10 +1759,8 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, const return false; } - const uint32_t BLOCK_SIZE = warptile[0]; const uint32_t BM = warptile[1]; const uint32_t BN = warptile[2]; - const uint32_t WARP = warptile[10]; const uint32_t BK = 32; const uint32_t BK_STEP = mul_mat_id ? 2u : 4u; @@ -1596,12 +1780,10 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, const total += 16u * (uint32_t)sizeof(int8_t); // cm1_kvalues[16] } if (src0_type == GGML_TYPE_NVFP4 && !device->ocp_fp4) { - total += 128u * (uint32_t)sizeof(float); // ue4m3_fp32_lut[128] + total += 256u * (uint32_t)sizeof(float); // ue4m3_fp32_lut[256] } if (mul_mat_id) { total += BN * 2u * (uint32_t)sizeof(uint16_t); // row_ids[BN] (u16vec2) - const uint32_t num_warps = BLOCK_SIZE / std::max(WARP, 1u); - total += num_warps * 4u * (uint32_t)sizeof(uint32_t); // ballots_sh[NUM_WARPS] (uvec4) } const bool supported = total <= device->properties.limits.maxComputeSharedMemorySize; @@ -1684,6 +1866,17 @@ static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type #endif } +// AMD proprietary drivers before 2.0.416 (Adrenalin 26.10.1) run the large coopmat matmul tile much slower than the medium one. +static bool ggml_vk_amd_large_tile_supported(const vk_device& device) { + if (device->driver_id != vk::DriverId::eAmdProprietary) { + return true; + } + // AMD encodes driverVersion as major [31:22], build [21:0], not VK_MAKE_API_VERSION. + const uint32_t major = device->properties.driverVersion >> 22; + const uint32_t build = device->properties.driverVersion & 0x3fffff; + return major > 2 || (major == 2 && build >= 416); +} + void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")"); @@ -1703,6 +1896,11 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const uint32_t mul_mat_mm_warp_8 = std::min(mul_mat_subgroup_size_8, 64u); const uint32_t mul_mat_mm_warp_16 = std::min(mul_mat_subgroup_size_16, 64u); + // small warptiles split BM=32 between BLOCK_SIZE/WARP subgroups, so each subgroup needs its own warp tile + const uint32_t s_warptile_wm_8 = 32 * subgroup_size_8 / subgroup_size_32; + const uint32_t mul_mat_s_warptile_wm_8 = 32 * mul_mat_subgroup_size_8 / mul_mat_subgroup_size_32; + const uint32_t mul_mat_s_warptile_wm_16 = 32 * mul_mat_subgroup_size_16 / mul_mat_subgroup_size_32; + const bool subgroup_min_size_16 = (!device->subgroup_size_control && device->subgroup_size >= 16) || (device->subgroup_size_control && device->subgroup_max_size >= 16); @@ -1797,20 +1995,18 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const uint32_t itn = device->coopmat_int_n; const uint32_t itk = device->coopmat_int_k; - const uint32_t s_warptile_wm = device->subgroup_size == 8 ? 8 : 32; - l_warptile = { 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mm_warp_8 }; m_warptile = { 128, 64, 64, 16, mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mm_warp_8 }; - s_warptile = { subgroup_size_32, 32, 32, 16, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 }; + s_warptile = { subgroup_size_32, 32, 32, 16, s_warptile_wm_8, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 }; l_warptile_mmq = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mm_warp_8 }; m_warptile_mmq = { 128, 64, 64, 32, mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mm_warp_8 }; - s_warptile_mmq = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 }; + s_warptile_mmq = { subgroup_size_32, 32, 32, 32, s_warptile_wm_8, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 }; // Integer MMQ has a smaller shared memory profile, but heavier register use l_warptile_mmq_int = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }; m_warptile_mmq_int = { 128, 64, 64, 32, mm_warp_8, 32, 2, 2, 2, 1, mm_warp_8 }; - s_warptile_mmq_int = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, 2, 1, 1, subgroup_size_8 }; + s_warptile_mmq_int = { subgroup_size_32, 32, 32, 32, s_warptile_wm_8, 32, 2, 2, 1, 1, subgroup_size_8 }; const auto cm1_bs = [cm1_sg](uint32_t bm, uint32_t bn) { return cm1_sg * (bm / std::min(cm1_sg, bm)) * (bn / 32); @@ -1831,29 +2027,29 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // K-quants use even more registers, mitigate by setting WMITER to 1 l_warptile_mmq_int_k = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 1, 4, 4, 1, mm_warp_8 }; m_warptile_mmq_int_k = { 128, 64, 64, 32, mm_warp_8, 32, 1, 2, 2, 1, mm_warp_8 }; - s_warptile_mmq_int_k = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 1, 2, 1, 1, subgroup_size_8 }; + s_warptile_mmq_int_k = { subgroup_size_32, 32, 32, 32, s_warptile_wm_8, 32, 1, 2, 1, 1, subgroup_size_8 }; l_warptile_id = { 128, 128, 128, 16, mul_mat_mm_warp_16 * 2, 64, 2, tm_l, tn_l, tk_l, mul_mat_mm_warp_16 }; m_warptile_id = { 128, 64, 64, 16, mul_mat_mm_warp_16, 32, 2, tm_m, tn_m, tk_m, mul_mat_mm_warp_16 }; - s_warptile_id = { mul_mat_subgroup_size_16, 32, 32, 16, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, mul_mat_subgroup_size_16 }; + s_warptile_id = { mul_mat_subgroup_size_16, 32, 32, 16, 32, 32, 2, tm_s, tn_s, tk_s, mul_mat_subgroup_size_16 }; l_warptile_mmqid = { 128, 128, 128, 32, mul_mat_mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mul_mat_mm_warp_8 }; m_warptile_mmqid = { 128, 64, 64, 32, mul_mat_mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mul_mat_mm_warp_8 }; - s_warptile_mmqid = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, mul_mat_subgroup_size_8 }; + s_warptile_mmqid = { mul_mat_subgroup_size_32, 32, 32, 32, mul_mat_s_warptile_wm_8, 32, 2, tm_s, tn_s, tk_s, mul_mat_subgroup_size_8 }; l_warptile_mmqid_int = { 128, 128, 128, 32, mul_mat_mm_warp_8 * 2, 64, 2, 4, 4, 1, mul_mat_mm_warp_8 }; m_warptile_mmqid_int = { 128, 64, 64, 32, mul_mat_mm_warp_8, 32, 2, 2, 2, 1, mul_mat_mm_warp_8 }; - s_warptile_mmqid_int = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, 2, 1, 1, mul_mat_subgroup_size_8 }; + s_warptile_mmqid_int = { mul_mat_subgroup_size_32, 32, 32, 32, mul_mat_s_warptile_wm_8, 32, 2, 2, 1, 1, mul_mat_subgroup_size_8 }; l_warptile_mmqid_int_k = { 128, 128, 128, 32, mul_mat_mm_warp_16 * 2, 64, 1, 4, 4, 1, mul_mat_mm_warp_16 }; m_warptile_mmqid_int_k = { 128, 64, 64, 32, mul_mat_mm_warp_16, 32, 1, 2, 2, 1, mul_mat_mm_warp_16 }; - s_warptile_mmqid_int_k = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 1, 2, 1, 1, mul_mat_subgroup_size_16 }; + s_warptile_mmqid_int_k = { mul_mat_subgroup_size_32, 32, 32, 32, mul_mat_s_warptile_wm_16, 32, 1, 2, 1, 1, mul_mat_subgroup_size_16 }; // chip specific tuning if ((device->architecture == AMD_GCN) && (device->driver_id != vk::DriverId::eAmdProprietary)) { m_warptile_mmq = m_warptile_mmq_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 }; m_warptile_mmqid = m_warptile_mmqid_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 }; - } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->coopmat_support && device->driver_id != vk::DriverId::eAmdProprietary) { + } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->coopmat_support && ggml_vk_amd_large_tile_supported(device)) { // This is intentionally using tx_m values, slight performance increase l_warptile = { 256, 128, 128, 16, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; l_warptile_mmq = l_warptile_mmq_int = { 256, 128, 128, 32, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; @@ -2806,9 +3002,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { && !device->coopmat_bf16_support #endif ) { - const uint32_t s_warptile_wm_bf16 = device->subgroup_size == 8 ? 8 : 32; std::vector<vk_tile_config> tc_bf16_fb = { - {{ subgroup_size_32, 32, 32, 16, s_warptile_wm_bf16, 32, 2, 2, 2, 1, subgroup_size_8 }, {32, 32, 1}, s_align}, + {{ subgroup_size_32, 32, 32, 16, s_warptile_wm_8, 32, 2, 2, 2, 1, subgroup_size_8 }, {32, 32, 1}, s_align}, {{ 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 }, {64, 64, 1}, m_align}, {{ 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }, {128, 128, 1}, l_align}, }; @@ -2892,8 +3087,14 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // RDNA3/4: above four columns, static 4 rows for all types bench faster than the default const bool is_rdna3_or_4 = device->vendor_id == VK_VENDOR_ID_AMD && (device->architecture == AMD_RDNA3 || device->architecture == AMD_RDNA4); auto const &rm_int_n = [&](uint32_t rows, uint32_t i) { return (is_rdna3_or_4 && i >= 4) ? 4u : rows; }; - // RDNA3/4: Static 4 rows for all types bench faster than the default - auto const &rm_id = [&](uint32_t rows) { return is_rdna3_or_4 ? 4u : rows; }; + // RDNA3/4 and NVIDIA except pre-Turing: use 4 rows for MUL_MAT_ID MMVQ. + auto const &rm_id = [&](uint32_t rows) { + if (device->vendor_id == VK_VENDOR_ID_NVIDIA && + device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) { + return 4u; + } + return is_rdna3_or_4 ? 4u : rows; + }; uint32_t rm_iq = 2 * rm_kq; const bool use_subgroups = device->subgroup_arithmetic; @@ -3181,43 +3382,36 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_flash_attn_split_k_reduce, "fa_split_k_reduce", fa_split_k_reduce_len, fa_split_k_reduce_data, "main", 3, sizeof(vk_op_flash_attn_split_k_reduce_push_constants), {1, device->subgroup_size, 1}, {device->subgroup_size}, 1, true); #if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (device->vendor_id == VK_VENDOR_ID_INTEL && (device->architecture == INTEL_XE2 || (device->architecture == INTEL_XE1 && device->coopmat_support && device->uma))) { - auto upper_power_of_2 = [&](uint32_t in) { - GGML_ASSERT(in != 0); - if (in <= 1) return 1u; - uint32_t ret = in - 1; - ret |= ret >> 1; - ret |= ret >> 2; - ret |= ret >> 4; - ret |= ret >> 8; - ret |= ret >> 16; - return ret + 1; - }; - - uint32_t xe_native_sub_group_size = 16; - if (device->architecture == INTEL_XE1) { - xe_native_sub_group_size = 8; - } + if (device->xe_fa_support) { + uint32_t shmem_sz = (uint32_t)device->properties.limits.maxComputeSharedMemorySize; + vk_xe_fa_tuning_params xe_fa_params {}; for (auto& it : device->pipeline_xe_fa_decode_dual_phases) { - const uint32_t split_p_chunk = 32; - auto HdQk = it.first; + auto pipeline_state = it.first; + auto& pipelines = it.second; + xe_fa_params = get_xe_fa_tuning_params(device.get(), pipeline_state, false); + std::vector<uint32_t> sc_ph1 = { xe_fa_params.workgroup_size[0], pipeline_state.qk_ratio, pipeline_state.HSK, + xe_fa_params.subgroup_size[0], xe_fa_params.decode_params.p_chunk_ph1, + xe_fa_params.decode_params.out_per_wg_ph1, xe_fa_params.decode_params.p_chunk_ph1 / xe_fa_params.tm, + xe_fa_params.tm, xe_fa_params.tn, shmem_sz }; + std::vector<uint32_t> sc_ph2 = { xe_fa_params.workgroup_size[1], pipeline_state.pv_ratio, pipeline_state.HSV, + xe_fa_params.decode_params.out_row_per_wg_ph2, xe_fa_params.subgroup_size[1], xe_fa_params.decode_params.split_p_per_iter_ph2, + xe_fa_params.decode_params.p_chunk_ph1, xe_fa_params.decode_params.out_col_per_wg_ph2, + xe_fa_params.tm, xe_fa_params.tn, xe_fa_params.decode_params.pipeline_flag_ph2 }; + ggml_vk_create_pipeline(device, pipelines.first, "xe_fa_decode_ph1", fa_decode_ph1_cm1_len, fa_decode_ph1_cm1_data, "main", 5, sizeof(vk_xe_fa_push_constants), { 1, xe_fa_params.decode_params.p_chunk_ph1, 1 }, sc_ph1, 1, false, true, xe_fa_params.subgroup_size[0]); + ggml_vk_create_pipeline(device, pipelines.second, "xe_fa_decode_ph2", fa_decode_ph2_cm1_len, fa_decode_ph2_cm1_data, "main", 5, sizeof(vk_xe_fa_push_constants), { 1, 1, 1 }, sc_ph2, 1, false, true, xe_fa_params.subgroup_size[0]); + } + + for (auto& it : device->pipeline_xe_fa_prefill) { + auto pipeline_state = it.first; auto& pipelines = it.second; - uint32_t head_dim_qk = std::get<0>(HdQk); - uint32_t head_dim_pv = std::get<1>(HdQk); - uint32_t gqa_ratio = std::get<2>(HdQk); - uint32_t q_len = std::get<3>(HdQk); - const uint32_t out_dim_per_wg = gqa_ratio > 16 ? 8 : 16; - uint32_t aligned_q_len = upper_power_of_2(q_len); - uint32_t group_sz_ph1 = std::min(std::max(aligned_q_len * xe_native_sub_group_size, 64u), 256u); - uint32_t out_per_wg_ph1 = std::min(q_len, 256u / xe_native_sub_group_size); - uint32_t aligned_gqa_ratio = upper_power_of_2(gqa_ratio); - uint32_t split_p_per_iter_ph2 = 256; - uint32_t split_p_per_warp = 16; - uint32_t group_sz_ph2 = (split_p_per_iter_ph2 / split_p_per_warp) * xe_native_sub_group_size; - uint32_t out_per_wg_ph2 = std::min(std::max(16u / aligned_gqa_ratio, 1u), q_len); - ggml_vk_create_pipeline(device, pipelines.first, "xe_fa_decode_ph1", fa_decode_ph1_cm1_len, fa_decode_ph1_cm1_data, "main", 5, sizeof(vk_fa_xe_opt_push_constants), { 1, 32, 1 }, { group_sz_ph1, gqa_ratio, head_dim_qk, xe_native_sub_group_size, split_p_chunk, out_per_wg_ph1 }, 1, false, true, xe_native_sub_group_size); - ggml_vk_create_pipeline(device, pipelines.second, "xe_fa_decode_ph2", fa_decode_ph2_cm1_len, fa_decode_ph2_cm1_data, "main", 5, sizeof(vk_fa_xe_opt_push_constants), { 1, 1, 1 }, { group_sz_ph2, gqa_ratio, head_dim_pv, out_per_wg_ph2, xe_native_sub_group_size, split_p_per_iter_ph2, split_p_chunk, out_dim_per_wg }, 1, false, true, xe_native_sub_group_size); + xe_fa_params = get_xe_fa_tuning_params(device.get(), pipeline_state, true); + + std::vector<uint32_t> sc = { xe_fa_params.workgroup_size[0], xe_fa_params.prefill_params.wg_row, xe_fa_params.prefill_params.sg_col, + pipeline_state.HSK, pipeline_state.HSV, pipeline_state.qk_ratio, xe_fa_params.prefill_params.p_per_iter, + xe_fa_params.tm, xe_fa_params.tn, xe_fa_params.prefill_params.q_in_reg, shmem_sz, xe_fa_params.prefill_params.acc_count, + xe_fa_params.prefill_params.pipeline_flag, xe_fa_params.subgroup_size[0], xe_fa_params.prefill_params.gqa_fold }; + ggml_vk_create_pipeline(device, pipelines, "xe_fa_prefill", fa_prefill_cm1_len, fa_prefill_cm1_data, "main", 6, sizeof(vk_xe_fa_push_constants), { xe_fa_params.prefill_params.gqa_fold, xe_fa_params.prefill_params.wg_row / xe_fa_params.prefill_params.gqa_fold, 1 }, sc, 1, false, true, xe_fa_params.subgroup_size[0]); } } #endif @@ -3258,43 +3452,58 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_f32, "rms_norm_mul_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_f32, "rms_norm_mul_add_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_f32, "rms_norm_mul_add_mul_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true); + const bool rms_subgroups = (device->vendor_id == VK_VENDOR_ID_INTEL || device->vendor_id == VK_VENDOR_ID_NVIDIA || device->vendor_id == VK_VENDOR_ID_AMD) && device->subgroup_arithmetic && device->subgroup_require_full_support; + const uint32_t rms_set_rows_block_size = 128; +#define RMS_SHADER(name) (rms_subgroups ? name##_subgroup_len : name##_len), (rms_subgroups ? name##_subgroup_data : name##_data) + // 128/256/512 block sizes, picked by row length + for (uint32_t i = 0; i < 3; ++i) { + const uint32_t block_size = 128u << i; + const std::string suffix = "_" + std::to_string(block_size); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_f32[i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_f32[i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_f32[i], "rms_norm_mul_add_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_mul_f32[i], "rms_norm_mul_add_mul_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1, block_size}, 1, true, rms_subgroups); + } ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_partials_f32, "rms_norm_mul_add_mul_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", rms_norm_set_rows_f32_f32_len, rms_norm_set_rows_f32_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", rms_norm_set_rows_f32_f16_len, rms_norm_set_rows_f32_f16_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", RMS_SHADER(rms_norm_set_rows_f32_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, rms_set_rows_block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", RMS_SHADER(rms_norm_set_rows_f32_f16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, rms_set_rows_block_size}, 1, true, rms_subgroups); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_partials_f32, "rms_norm_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_partials_f32, "rms_norm_mul_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true); if (sizeof(vk_op_rms_norm_mul_rope_push_constants) <= device->properties.limits.maxPushConstantsSize) { - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f32, "rms_norm_mul_rope_f32_f32", rms_norm_mul_rope_f32_f32_len, rms_norm_mul_rope_f32_f32_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f16, "rms_norm_mul_rope_f32_f16", rms_norm_mul_rope_f32_f16_len, rms_norm_mul_rope_f32_f16_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true); + for (uint32_t i = 0; i < 3; ++i) { + const uint32_t block_size = 128u << i; + const std::string suffix = "_" + std::to_string(block_size); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_rope_f32_f32[i], "rms_norm_mul_rope_f32_f32" + suffix, RMS_SHADER(rms_norm_mul_rope_f32_f32), "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_rope_f32_f16[i], "rms_norm_mul_rope_f32_f16" + suffix, RMS_SHADER(rms_norm_mul_rope_f32_f16), "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + } } +#undef RMS_SHADER ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_32, "cpy_32", cpy_32_len, cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f16, "cpy_f16_f16", cpy_f16_f16_len, cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_16, "cpy_16", cpy_16_len, cpy_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f32, "cpy_f16_f32", cpy_f16_f32_len, cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_bf16,"cpy_f32_bf16",cpy_f32_bf16_len,cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f32,"cpy_bf16_f32",cpy_bf16_f32_len,cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f16,"cpy_bf16_f16",cpy_bf16_f16_len,cpy_bf16_f16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_i32_f32, "cpy_i32_f32", cpy_i32_f32_len, cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_i32, "cpy_f32_i32", cpy_f32_i32_len, cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f32, "contig_cpy_f32_f32", contig_cpy_f32_f32_len, contig_cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_32, "contig_cpy_32", contig_cpy_32_len, contig_cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f16, "contig_cpy_f32_f16", contig_cpy_f32_f16_len, contig_cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f16, "contig_cpy_f16_f16", contig_cpy_f16_f16_len, contig_cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_16, "contig_cpy_16", contig_cpy_16_len, contig_cpy_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f32, "contig_cpy_f16_f32", contig_cpy_f16_f32_len, contig_cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_bf16,"contig_cpy_f32_bf16",contig_cpy_f32_bf16_len,contig_cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f32,"contig_cpy_bf16_f32",contig_cpy_bf16_f32_len,contig_cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f16,"contig_cpy_bf16_f16",contig_cpy_bf16_f16_len,contig_cpy_bf16_f16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_i32_f32, "contig_cpy_i32_f32", contig_cpy_i32_f32_len, contig_cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_i32, "contig_cpy_f32_i32", contig_cpy_f32_i32_len, contig_cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); @@ -3382,6 +3591,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_concat_i8, "concat_i8", concat_i8_len, concat_i8_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_concat_i16, "concat_i16", concat_i16_len, concat_i16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_concat_i32, "concat_i32", concat_i32_len, concat_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i8, "concat_transpose_i8", concat_transpose_i8_len, concat_transpose_i8_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i16, "concat_transpose_i16", concat_transpose_i16_len, concat_transpose_i16_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i32, "concat_transpose_i32", concat_transpose_i32_len, concat_transpose_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_concat_i64, "concat_i64", concat_i64_len, concat_i64_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_upscale_nearest_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_NEAREST}, 1); @@ -3464,10 +3676,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_UNARY_MUL(softplus, 3) #undef CREATE_UNARY_MUL - ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_arange_f32, "arange_f32", arange_f32_len, arange_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); @@ -3488,8 +3696,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {1, 512, 1}, {}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_wg512, "soft_max_f32_wg512", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 512 }, 1); ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_f16, "soft_max_f32_f16", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); @@ -3556,7 +3762,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1, true); ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32, "cross_entropy_loss_f32", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32_wg512, "cross_entropy_loss_f32_wg512", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { 512 }, 1); ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_back_f32, "cross_entropy_loss_back_f32", cross_entropy_loss_back_f32_len, cross_entropy_loss_back_f32_data, "main", 4, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); @@ -3564,21 +3770,31 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // Intel Windows driver in range [32.0.101.8509, 32.0.101.8860) will crash when using fwht kernels so we gate that here const bool can_use_fwht = device->driver_id != vk::DriverId::eIntelProprietaryWindows || !ggml_vk_intel_windows_driver_in_range(device->properties.driverVersion, 101, 8509, 101, 8860); - if (can_use_fwht && device->subgroup_basic && device->subgroup_shuffle) { + if (can_use_fwht) { + const bool use_subgroup = device->subgroup_basic && device->subgroup_shuffle; + const uint32_t sg = std::max(device->subgroup_size, 1u); int idx = 0; - for (uint32_t n : {64, 128, 256, 512}) { - if (device->subgroup_size <= n) { - ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_f32", fwht_f32_len, fwht_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n }, 1, true, true, device->subgroup_size); + for (uint32_t n : {64, 128, 256, 512, 1024, 2048, 4096, 8192}) { + const bool wide = n > fwht_max_subgroup_n || n / sg > fwht_max_subgroup_el_w; + if (use_subgroup && !wide) { + if (device->subgroup_size <= n) { + ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_f32", fwht_f32_len, fwht_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n, fwht_rows }, 1, true, true, device->subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_fwht_f16[idx], "fwht_f16", fwht_f16_len, fwht_f16_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n, fwht_rows }, 1, true, true, device->subgroup_size); + device->fwht_rows_per_wg[idx] = fwht_rows; + } + } else { + // wide blocks trade rows per workgroup for a smaller register block + const uint32_t block_size = wide ? std::min(n, 256u) : std::min(sg, n); + const uint32_t rows = wide ? 1u : fwht_rows; + if ((uint64_t)rows * n * sizeof(float) <= device->properties.limits.maxComputeSharedMemorySize && + block_size * rows <= device->properties.limits.maxComputeWorkGroupInvocations) { + ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_shmem_f32", fwht_shmem_f32_len, fwht_shmem_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n, rows }, 1); + ggml_vk_create_pipeline(device, device->pipeline_fwht_f16[idx], "fwht_shmem_f16", fwht_shmem_f16_len, fwht_shmem_f16_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n, rows }, 1); + device->fwht_rows_per_wg[idx] = rows; + } } ++idx; } - } else if (can_use_fwht) { - int idx = 0; - for (uint32_t n : {64, 128, 256, 512}) { - const uint32_t block_size = std::min(device->subgroup_size, n); - ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_shmem_f32", fwht_shmem_f32_len, fwht_shmem_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n }, 1); - ++idx; - } } const uint32_t cumsum_elem_per_thread = (device->vendor_id == VK_VENDOR_ID_AMD || device->vendor_id == VK_VENDOR_ID_INTEL) ? 2 : 4; @@ -3589,10 +3805,10 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_count_equal_i32, "count_equal_i32", count_equal_i32_len, count_equal_i32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, { device->subgroup_size }, 1); - if (device->subgroup_arithmetic && device->subgroup_require_full_support) { - ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true); + if (device->subgroup_arithmetic && device->subgroup_vote && device->subgroup_require_full_support) { + ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 3, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true); } else { - ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 3, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true); } // comb holds a token's 4x4 matrix in one 16-lane slice of a subgroup, so it @@ -3646,7 +3862,10 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_snake_bf16, "snake_bf16", snake_bf16_len, snake_bf16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_pool1d_f32, "pool1d_f32", pool1d_f32_len, pool1d_f32_data, "main", 2, sizeof(vk_op_pool1d_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_pool2d_f32, "pool2d_f32", pool2d_f32_len, pool2d_f32_data, "main", 2, sizeof(vk_op_pool2d_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_pool2d_f32, "pool2d_f32", pool2d_f32_len, pool2d_f32_data, "main", 2, sizeof(vk_op_pool2d_push_constants), {1, 1, 1}, {512}, 1); + if (device->subgroup_arithmetic && device->subgroup_require_full_support) { + ggml_vk_create_pipeline(device, device->pipeline_pool2d_subgroup_f32, "pool2d_subgroup_f32", pool2d_subgroup_f32_len, pool2d_subgroup_f32_data, "main", 2, sizeof(vk_op_pool2d_push_constants), {1, 1, 1}, {128}, 1, false, true, device->subgroup_size); + } ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv6_f32, "rwkv_wkv6_f32", rwkv_wkv6_f32_len, rwkv_wkv6_f32_data, "main", 7, sizeof(vk_op_rwkv_wkv6_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); @@ -3810,7 +4029,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { auto shmem_req = [&](uint32_t pad, bool csh_store, bool fp16_shmem) { const uint32_t elem_size = fp16_shmem ? (uint32_t)sizeof(uint16_t) : (uint32_t)sizeof(float); - const uint32_t csh_elems = csh_store ? conv2d_BS.K * conv2d_BS.NPQ : 0u; + const uint32_t csh_elems = csh_store ? conv2d_BS.K * conv2d_BS.NPQ : (fp16_shmem ? 8u : 0u); return (conv2d_BS.K * (conv2d_BS.CRS + pad) + conv2d_BS.CRS * (conv2d_BS.NPQ + pad) + csh_elems) * elem_size; }; @@ -3838,9 +4057,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } // stage cm2 accumulator through shmem for coalesced global stores; - // skipped on 128x128 where the extra Csh footprint hurts occupancy. + // skipped on large tiles where the extra Csh footprint hurts occupancy. // cm1 always uses the staged path. - uint32_t conv2d_csh_store = (device->coopmat2 && s != CONV_SHAPE_128x128) ? 1u : 0u; + uint32_t conv2d_csh_store = (device->coopmat2 && s != CONV_SHAPE_128x128 && s != CONV_SHAPE_256x128) ? 1u : 0u; if (conv2d_use_cm1) { conv2d_csh_store = 1; } @@ -3989,13 +4208,44 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } +static bool ggml_vk_external_memory_host_usable(vk::DriverId driver_id) { + // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622 + // is available in the Vulkan SDK. + return driver_id != vk::DriverId::eMoltenvk; +} + +static bool ggml_vk_host_buffer_import_usable(uint32_t vendor_id) { + // amdgpu re-faults all pages of every imported host allocation on each submission + return vendor_id != VK_VENDOR_ID_AMD; +} + +static bool ggml_vk_has_rebar(const vk::PhysicalDevice & physical_device) { + const vk::PhysicalDeviceMemoryProperties mem_props = physical_device.getMemoryProperties(); + const vk::MemoryPropertyFlags hvv = vk::MemoryPropertyFlagBits::eDeviceLocal | + vk::MemoryPropertyFlagBits::eHostVisible | + vk::MemoryPropertyFlagBits::eHostCoherent; + uint32_t vram_heap = UINT32_MAX; + for (uint32_t i = 0; i < mem_props.memoryTypeCount; i++) { + const vk::MemoryType & t = mem_props.memoryTypes[i]; + if (vram_heap == UINT32_MAX && (t.propertyFlags & vk::MemoryPropertyFlagBits::eDeviceLocal)) { + vram_heap = t.heapIndex; + } + if ((t.propertyFlags & hvv) == hvv && t.heapIndex == vram_heap) { + return true; + } + } + return false; +} + vk_device ggml_vk_get_device(size_t idx) { VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")"); + static std::mutex mutex; + std::lock_guard<std::mutex> lock(mutex); + if (vk_instance.devices[idx] == nullptr) { VK_LOG_DEBUG("Initializing new vk_device"); vk_device device = std::make_shared<vk_device_struct>(); - vk_instance.devices[idx] = device; device->memory_logger = std::unique_ptr<vk_memory_logger>(new vk_memory_logger()); @@ -4042,6 +4292,7 @@ vk_device ggml_vk_get_device(size_t idx) { bool dot2_f16_support = false; bool ocp_microscaling_extension = false; bool shader_float8_extension = false; + bool workgroup_memory_explicit_layout_support = false; for (const auto& properties : ext_props) { if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { @@ -4097,11 +4348,17 @@ vk_device ggml_vk_get_device(size_t idx) { dot2_f16_support = true; } else if (strcmp("VK_KHR_pipeline_executable_properties", properties.extensionName) == 0) { pipeline_executable_properties_support = true; +#if defined(VK_KHR_workgroup_memory_explicit_layout) + } else if (strcmp("VK_KHR_workgroup_memory_explicit_layout", properties.extensionName) == 0) { + workgroup_memory_explicit_layout_support = true; +#endif } else if (strcmp("VK_EXT_memory_priority", properties.extensionName) == 0 && getenv("GGML_VK_ENABLE_MEMORY_PRIORITY")) { device->memory_priority = true; } else if (strcmp("VK_EXT_external_memory_host", properties.extensionName) == 0) { device->external_memory_host = true; + } else if (strcmp("VK_KHR_external_semaphore_fd", properties.extensionName) == 0) { + device->external_semaphore = true; #if defined(VK_EXT_shader_64bit_indexing) } else if (strcmp("VK_EXT_shader_64bit_indexing", properties.extensionName) == 0) { device->shader_64b_indexing = true; @@ -4174,11 +4431,7 @@ vk_device ggml_vk_get_device(size_t idx) { device->vendor_id = device->properties.vendorID; device->driver_id = driver_props.driverID; - if (device->driver_id == vk::DriverId::eMoltenvk) { - // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622 - // is available in the Vulkan SDK. - device->external_memory_host = false; - } + device->external_memory_host = device->external_memory_host && ggml_vk_external_memory_host_usable(device->driver_id); // Implementing the async backend interfaces seems broken on older Intel HW, // see https://github.com/ggml-org/llama.cpp/issues/17302. @@ -4223,6 +4476,7 @@ vk_device ggml_vk_get_device(size_t idx) { device->subgroup_size = subgroup_props.subgroupSize; device->subgroup_size_log2 = uint32_t(log2f(float(device->subgroup_size))); device->uma = device->properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; + device->rebar = ggml_vk_has_rebar(device->physical_device); if (sm_builtins) { device->shader_core_count = sm_props.shaderSMCount; } else if (amd_shader_core_properties2) { @@ -4450,9 +4704,22 @@ vk_device ggml_vk_get_device(size_t idx) { device_extensions.push_back("VK_KHR_pipeline_executable_properties"); } +#if defined(VK_KHR_workgroup_memory_explicit_layout) + VkPhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR wg_explicit_layout_features {}; + wg_explicit_layout_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_FEATURES_KHR; + if (workgroup_memory_explicit_layout_support) { + last_struct->pNext = (VkBaseOutStructure *)&wg_explicit_layout_features; + last_struct = (VkBaseOutStructure *)&wg_explicit_layout_features; + device_extensions.push_back("VK_KHR_workgroup_memory_explicit_layout"); + } +#endif + if (device->external_memory_host) { device_extensions.push_back("VK_EXT_external_memory_host"); } + if (device->external_semaphore) { + device_extensions.push_back("VK_KHR_external_semaphore_fd"); + } #if defined(VK_EXT_shader_64bit_indexing) VkPhysicalDeviceShader64BitIndexingFeaturesEXT shader_64bit_indexing_features {}; @@ -4508,6 +4775,16 @@ vk_device ggml_vk_get_device(size_t idx) { shader_float8_extension && shader_float8_features.shaderFloat8 && !getenv("GGML_VK_DISABLE_OCP_FP4"); +#if defined(VK_KHR_workgroup_memory_explicit_layout) + device->workgroup_memory_explicit_layout = workgroup_memory_explicit_layout_support && + wg_explicit_layout_features.workgroupMemoryExplicitLayout; + device->workgroup_memory_explicit_layout_16bit = device->workgroup_memory_explicit_layout && + wg_explicit_layout_features.workgroupMemoryExplicitLayout16BitAccess; +#else + device->workgroup_memory_explicit_layout = false; + device->workgroup_memory_explicit_layout_16bit = false; +#endif + device->pipeline_robustness = pl_robustness_features.pipelineRobustness; device->multi_add = vk12_props.shaderRoundingModeRTEFloat16 && @@ -4782,7 +5059,7 @@ vk_device ggml_vk_get_device(size_t idx) { switch (device->vendor_id) { #ifndef GGML_VULKAN_RUN_TESTS case VK_VENDOR_ID_AMD: - device->mul_mat_l[i] = device->coopmat_support && device->driver_id != vk::DriverId::eAmdProprietary; + device->mul_mat_l[i] = device->coopmat_support && ggml_vk_amd_large_tile_supported(device); device->mul_mat_m[i] = true; device->mul_mat_s[i] = true; device->mul_mat_id_l[i] = false; @@ -4873,6 +5150,13 @@ vk_device ggml_vk_get_device(size_t idx) { descriptor_set_layout_create_info.setPNext(&dslbfci); device->dsl = device->device.createDescriptorSetLayout(descriptor_set_layout_create_info); +#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + device->xe_fa_support = device->coopmat_support && device->properties.limits.maxComputeSharedMemorySize >= 48 * 1024 && + device->workgroup_memory_explicit_layout && device->workgroup_memory_explicit_layout_16bit && + ((device->vendor_id == VK_VENDOR_ID_INTEL && (device->architecture == INTEL_XE2 || (device->architecture == INTEL_XE1 && device->uma))) || + (device->subgroup_size == 32 && device->coopmat_support_16x16x16_f32acc && (getenv("GGML_VK_FORCE_XE_FA") != nullptr))); +#endif + ggml_vk_load_shaders(device); // Prefer a dedicated transfer queue on AMD dGPUs (non-GCN) when graphics queue use is disabled. @@ -4893,14 +5177,14 @@ vk_device ggml_vk_get_device(size_t idx) { device->async_use_transfer_queue = false; } + device->fence = device->device.createFence({}); + device->buffer_type = { /* .iface = */ ggml_backend_vk_buffer_type_interface, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_vk_reg(), idx), /* .context = */ new ggml_backend_vk_buffer_type_context{ device->name, device }, }; - device->fence = device->device.createFence({}); - device->idx = idx; device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr; @@ -4909,8 +5193,7 @@ vk_device ggml_vk_get_device(size_t idx) { device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr; - device->add_rms_fusion = !device->disable_fusion && - device->subgroup_arithmetic && + device->add_rms_fusion = device->subgroup_arithmetic && device->vendor_id != VK_VENDOR_ID_INTEL; device->partials_binding_alignment = std::max(4u, (uint32_t)device->properties.limits.minStorageBufferOffsetAlignment); @@ -4922,6 +5205,9 @@ vk_device ggml_vk_get_device(size_t idx) { device->mmvq_mode = 1; } + // publish the device only when it is fully built + vk_instance.devices[idx] = device; + return device; } @@ -5119,6 +5405,7 @@ static void ggml_vk_print_gpu_info(size_t idx) { uint32_t default_subgroup_size = get_subgroup_size("", device_architecture); const size_t subgroup_size = (default_subgroup_size != 0) ? default_subgroup_size : subgroup_props.subgroupSize; const bool uma = props2.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; + const bool rebar = ggml_vk_has_rebar(physical_device); integer_dot_product = integer_dot_product && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated @@ -5165,8 +5452,8 @@ static void ggml_vk_print_gpu_info(size_t idx) { #endif std::string device_name = props2.properties.deviceName.data(); - GGML_LOG_DEBUG("ggml_vulkan: %zu = %s (%s) | uma: %d | fp16: %s | bf16: %d | fp4: %d | warp size: %zu | shared memory: %d | int dot: %d | matrix cores: %s\n", - idx, device_name.c_str(), driver_props.driverName.data(), uma, fp16_str, bf16, fp4, subgroup_size, + GGML_LOG_DEBUG("ggml_vulkan: %zu = %s (%s) | uma: %d | rebar: %d | fp16: %s | bf16: %d | fp4: %d | warp size: %zu | shared memory: %d | int dot: %d | matrix cores: %s\n", + idx, device_name.c_str(), driver_props.driverName.data(), uma, rebar, fp16_str, bf16, fp4, subgroup_size, props2.properties.limits.maxComputeSharedMemorySize, integer_dot_product, matrix_cores.c_str()); if (props2.properties.deviceType == vk::PhysicalDeviceType::eCpu) { @@ -5189,10 +5476,16 @@ void ggml_vk_instance_init() { // See https://github.com/KhronosGroup/Vulkan-Hpp?tab=readme-ov-file#extensions--per-device-function-pointers- ggml_vk_default_dispatcher_instance.init(vkGetInstanceProcAddr); + // vkEnumerateInstanceVersion is Vulkan 1.1. A null value indicated Vulkan 1.0. + if (ggml_vk_default_dispatcher_instance.vkEnumerateInstanceVersion == nullptr) { + GGML_LOG_ERROR("ggml_vulkan: Error: Vulkan 1.2 required."); + throw vk::SystemError(vk::Result::eErrorFeatureNotPresent, "Vulkan 1.2 required"); + } + uint32_t api_version = vk::enumerateInstanceVersion(); if (api_version < VK_API_VERSION_1_2) { - std::cerr << "ggml_vulkan: Error: Vulkan 1.2 required." << std::endl; + GGML_LOG_ERROR("ggml_vulkan: Error: Vulkan 1.2 required."); throw vk::SystemError(vk::Result::eErrorFeatureNotPresent, "Vulkan 1.2 required"); } @@ -5256,6 +5549,7 @@ void ggml_vk_instance_init() { vk_perf_logger_concurrent = getenv("GGML_VK_PERF_LOGGER_CONCURRENT") != nullptr; vk_enable_sync_logger = getenv("GGML_VK_SYNC_LOGGER") != nullptr; vk_memory_logger_enabled = getenv("GGML_VK_MEMORY_LOGGER") != nullptr; + vk_instance.allreduce_disabled = getenv("GGML_VK_DISABLE_ALLREDUCE") != nullptr; const char* GGML_VK_PIPELINE_STATS = getenv("GGML_VK_PIPELINE_STATS"); if (GGML_VK_PIPELINE_STATS != nullptr) { vk_pipeline_stats_filter = GGML_VK_PIPELINE_STATS; @@ -5412,22 +5706,44 @@ void ggml_vk_instance_init() { } GGML_LOG_DEBUG("ggml_vulkan: Found %zu Vulkan devices:\n", vk_instance.device_indices.size()); + size_t host_buffer_import_count = 0; for (size_t i = 0; i < vk_instance.device_indices.size(); i++) { vk::PhysicalDevice vkdev = devices[vk_instance.device_indices[i]]; std::vector<vk::ExtensionProperties> extensionprops = vkdev.enumerateDeviceExtensionProperties(); bool membudget_supported = false; + bool external_memory_host_supported = false; for (const auto & ext : extensionprops) { if (strcmp(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME, ext.extensionName) == 0) { membudget_supported = true; - break; + } else if (strcmp("VK_EXT_external_memory_host", ext.extensionName) == 0) { + external_memory_host_supported = true; } } vk_instance.device_supports_membudget.push_back(membudget_supported); + bool host_buffer_import = false; + if (external_memory_host_supported) { + vk::PhysicalDeviceProperties2 props2; + vk::PhysicalDeviceDriverProperties driver_props; + vk::PhysicalDeviceExternalMemoryHostPropertiesEXT ext_props; + props2.pNext = &driver_props; + driver_props.pNext = &ext_props; + vkdev.getProperties2(&props2); + host_buffer_import = ggml_vk_external_memory_host_usable(driver_props.driverID) && + ggml_vk_host_buffer_import_usable(props2.properties.vendorID); + if (host_buffer_import) { + vk_instance.host_import_alignment = std::max<size_t>(vk_instance.host_import_alignment, ext_props.minImportedHostPointerAlignment); + host_buffer_import_count++; + } + } + vk_instance.device_host_buffer_import.push_back(host_buffer_import); + ggml_vk_print_gpu_info(i); } + + vk_instance.multi_device = host_buffer_import_count > 1 && vk_instance.device_host_buffer_import[0]; } void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { @@ -5440,6 +5756,7 @@ void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { ctx->device = ggml_vk_get_device(idx); ctx->semaphore_idx = 0; + ctx->binary_semaphore_idx = 0; ctx->event_idx = 0; ctx->prealloc_size_x = 0; @@ -5867,8 +6184,18 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, return 1; } + const bool rdna = ctx->device->architecture == AMD_RDNA3 || ctx->device->architecture == AMD_RDNA4; uint32_t split_k = 1; - if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) { + if (rdna && ctx->device->shader_core_count != 0 && k >= 2048 && n <= 32) { + // Small n gives too few workgroups to hide latency, aim for 8 per CU. Small tiles are one subgroup each. + const uint32_t tiles = CEIL_DIV(m, pipeline->wg_denoms[0]) * CEIL_DIV(n, pipeline->wg_denoms[1]); + if (tiles < 4 * ctx->device->shader_core_count) { + split_k = std::min(8u, CEIL_DIV(8 * ctx->device->shader_core_count, tiles)); + } + } + + // On RDNA, also split when n <= 64 only partly fills one tile column. + if (split_k == 1 && ctx->device->shader_core_count != 0 && (n >= pipeline->wg_denoms[1] || (rdna && n <= 64))) { // If k is 'large' and the SMs will fill less than halfway, use split_k. uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]); uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]); @@ -5878,25 +6205,30 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, split_k = ctx->device->shader_core_count / (m_tiles * n_tiles); } else if (m_tiles * n_tiles <= ctx->device->shader_core_count * 2 / 3) { split_k = 3; + } else if ((ctx->device->architecture == AMD_RDNA3 || ctx->device->architecture == AMD_RDNA4) && + m_tiles * n_tiles > ctx->device->shader_core_count && + m_tiles * n_tiles <= ctx->device->shader_core_count * 4 / 3) { + // two waves, the second one at most 1/3 full + split_k = 3; } // Cap the split at 8x. Unless k is huge this is a lot of overhead. split_k = std::min(split_k, 8u); + } + } - // ggml_vk_matmul will align the splits to be a multiple of 256. - // If this rounded up size would cause the last split to be empty, - // then reduce the split count. - while (true) { - if (split_k == 1) { - break; - } - uint32_t k_split = CEIL_DIV(k, split_k); - k_split = ROUNDUP_POW2(k_split, 256); - if (k_split * (split_k - 1) < k) { - break; - } - split_k--; - } + // ggml_vk_matmul will align the splits to be a multiple of 256. + // If this rounded up size would cause the last split to be empty, + // then reduce the split count. + while (true) { + if (split_k == 1) { + break; + } + uint32_t k_split = CEIL_DIV(k, split_k); + k_split = ROUNDUP_POW2(k_split, 256); + if (k_split * (split_k - 1) < k) { + break; } + split_k--; } return split_k; @@ -5908,18 +6240,34 @@ void ggml_vk_matmul( uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, - uint32_t padded_n) { + uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes) { VK_LOG_DEBUG("ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", padded_n: " << padded_n << ")"); if (split_k == 1) { - ggml_pipeline_request_descriptor_sets(ctx, pipeline, CEIL_DIV(batch, ctx->device->properties.limits.maxComputeWorkGroupCount[2])); - - uint32_t base_work_group_z = 0; - while (base_work_group_z < batch) { - uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]); - - const vk_mat_mat_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, padded_n }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d }, pc, { m, n, groups_z }); - base_work_group_z += groups_z; + // a small tile can still produce more workgroups than the device allows on the n axis: + // split the dispatch into n-chunks, shifting the B and D column windows + const uint64_t max_n = (uint64_t)ctx->device->properties.limits.maxComputeWorkGroupCount[1] * pipeline->wg_denoms[1]; + const uint64_t n_chunks = CEIL_DIV(n, max_n); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, n_chunks * CEIL_DIV(batch, ctx->device->properties.limits.maxComputeWorkGroupCount[2])); + + for (uint64_t n_base = 0; n_base < n; n_base += max_n) { + const uint32_t n_chunk = (uint32_t) std::min(n - n_base, max_n); + GGML_ASSERT(n_base * b_col_bytes < b.size && n_base * d_col_bytes < d.size); + GGML_ASSERT(b.size != VK_WHOLE_SIZE && d.size != VK_WHOLE_SIZE); + vk_subbuffer b_chunk = b; + vk_subbuffer d_chunk = d; + b_chunk.offset += n_base * b_col_bytes; + b_chunk.size -= n_base * b_col_bytes; + d_chunk.offset += n_base * d_col_bytes; + d_chunk.size -= n_base * d_col_bytes; + + uint32_t base_work_group_z = 0; + while (base_work_group_z < batch) { + uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + + const vk_mat_mat_push_constants pc = { m, n_chunk, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, uint32_t(padded_n - n_base) }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b_chunk, d_chunk }, pc, { m, n_chunk, groups_z }); + base_work_group_z += groups_z; + } } return; } @@ -5951,20 +6299,49 @@ void ggml_vk_matmul( ctx->prealloc_split_k_need_sync = true; } -static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { +static bool ggml_vk_src1_prec_allows_q8(const ggml_tensor * dst) { + switch (ggml_prec_get_src(dst, 1)) { + case GGML_PREC_UNDEFINED: + case GGML_PREC_Q8: + case GGML_PREC_Q4: + return true; + case GGML_PREC_F32: + case GGML_PREC_BF16: + case GGML_PREC_F16: + return false; + } + return false; +} + +static bool ggml_vk_prec_allows_f16acc(ggml_prec prec) { + switch (prec) { + case GGML_PREC_F32: + case GGML_PREC_BF16: + return false; + case GGML_PREC_UNDEFINED: + case GGML_PREC_F16: + case GGML_PREC_Q8: + case GGML_PREC_Q4: + return true; + } + return false; +} + +static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec acc_prec) { if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false; if (src1_type == GGML_TYPE_Q8_1) return false; + const bool prec_allows_f16 = ggml_vk_prec_allows_f16acc(acc_prec); if (src0_type == GGML_TYPE_F16) { - return prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support); + return prec_allows_f16 && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support); } // quant types if (ctx->device->coopmat2) { - return prec == GGML_PREC_DEFAULT; + return prec_allows_f16; } if (ctx->device->coopmat_support) { - return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT; + return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec_allows_f16; } - return ctx->device->fp16 && prec == GGML_PREC_DEFAULT; + return ctx->device->fp16 && prec_allows_f16; } static const std::vector<vk_matmul_pipeline_pair>* ggml_vk_get_mul_mat_mat_pipeline_map( @@ -6002,20 +6379,27 @@ static uint32_t ggml_vk_guess_matmul_pipeline_align_map(ggml_backend_vk_context return configs[idx].align; } +static uint64_t ggml_vk_mul_mat_id_max_tiles(uint64_t n_experts, uint64_t n_rows, uint32_t bn) { + // Each nonempty expert can add at most one partial tile to the combined grid. + return std::min(n_rows, CEIL_DIV(n_rows, bn) + std::min(n_experts, n_rows) - 1); +} + static void ggml_vk_matmul_id( ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& ids, const vk_subbuffer & expert_count_buf, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11, - bool hoist_row_ids) { + uint32_t row_ids_offset, uint32_t max_tiles) { VK_LOG_DEBUG("ggml_vk_matmul_id(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), ids: (" << ids.buffer->buffer << ", " << ids.offset << ", " << ids.size << "), expert_count: (" << expert_count_buf.buffer->buffer << ", " << expert_count_buf.offset << ", " << expert_count_buf.size << "), " << "m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", " << "batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", " << "n_as: " << n_as << ", nei0: " << nei0 << ", nei1: " << nei1 << ", nbi1: " << nbi1 << ", ne11: " << ne11 << ")"); const vk_mat_mat_id_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, - nei0, nei1, nbi1, ne11, n_as, uint32_t(hoist_row_ids) }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, { m, nei1, n_as }); + nei0, nei1, nbi1, ne11, n_as, row_ids_offset }; + const uint32_t tiles_y = std::min(max_tiles, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + const std::array<uint32_t, 3> elements = { m, tiles_y * pipeline->wg_denoms[1], uint32_t(CEIL_DIV(max_tiles, tiles_y)) }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, elements); } bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { @@ -6058,13 +6442,6 @@ vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_t } } - if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_F32) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f32_f32; - } else { - return ctx->device->pipeline_cpy_f32_f32; - } - } if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_F16) { if (contig) { return ctx->device->pipeline_contig_cpy_f32_f16; @@ -6072,13 +6449,6 @@ vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_t return ctx->device->pipeline_cpy_f32_f16; } } - if (src->type == GGML_TYPE_F16 && to == GGML_TYPE_F16) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f16_f16; - } else { - return ctx->device->pipeline_cpy_f16_f16; - } - } if (src->type == GGML_TYPE_F16 && to == GGML_TYPE_F32) { if (contig) { return ctx->device->pipeline_contig_cpy_f16_f32; @@ -6100,6 +6470,13 @@ vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_t return ctx->device->pipeline_cpy_bf16_f32; } } + if (src->type == GGML_TYPE_BF16 && to == GGML_TYPE_F16) { + if (contig) { + return ctx->device->pipeline_contig_cpy_bf16_f16; + } else { + return ctx->device->pipeline_cpy_bf16_f16; + } + } if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_I32) { if (contig) { return ctx->device->pipeline_contig_cpy_f32_i32; @@ -6154,15 +6531,15 @@ vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_t GGML_ASSERT(ggml_is_quantized(to) || ggml_type_size(src->type) == 2 || ggml_type_size(src->type) == 4); if ((ggml_type_size(src->type) % 4) == 0) { if (contig) { - return ctx->device->pipeline_contig_cpy_f32_f32; + return ctx->device->pipeline_contig_cpy_32; } else { - return ctx->device->pipeline_cpy_f32_f32; + return ctx->device->pipeline_cpy_32; } } else { if (contig) { - return ctx->device->pipeline_contig_cpy_f16_f16; + return ctx->device->pipeline_contig_cpy_16; } else { - return ctx->device->pipeline_cpy_f16_f16; + return ctx->device->pipeline_cpy_16; } } } @@ -6267,6 +6644,20 @@ static vk_pipeline ggml_vk_get_64b_indexing_pipeline(ggml_backend_vk_context * c return pipeline; } +// perf-heuristic for choosing mul_mat b type, based on shader availability +static ggml_type ggml_vk_mul_mat_b_type( + ggml_backend_vk_context * ctx, ggml_type src0_type, const ggml_tensor * src1, + ggml_type f16_type, ggml_prec prec, bool mul_mat_id, bool can_quantize_y, bool prefer_f16_b) { + auto available = [&](ggml_type b) { + return ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0_type, b, prec, mul_mat_id) != nullptr; + }; + if (can_quantize_y && available(GGML_TYPE_Q8_1)) return GGML_TYPE_Q8_1; + if (prefer_f16_b && available(f16_type)) return f16_type; + if (ggml_vk_dim01_contiguous(src1) && available(src1->type)) return src1->type; + if (src1->type == GGML_TYPE_BF16 && available(GGML_TYPE_F32)) return GGML_TYPE_F32; + return f16_type; +} + static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool disable_split_k) { VK_LOG_DEBUG("ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << ggml_type_name(src0->type) << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << ggml_type_name(src1->type) << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -6311,51 +6702,35 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub src1_uma = d_Qy != nullptr; } - // TODO: Clean up this logic to pick src1 type by capability - // Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf const bool x_non_contig = (ctx->device->coopmat2 && src0->type == GGML_TYPE_F32) || !ggml_vk_dim01_contiguous(src0); // If src0 is BF16, try to use a BF16 x BF16 multiply ggml_type f16_type = src0->type == GGML_TYPE_BF16 ? GGML_TYPE_BF16 : GGML_TYPE_F16; - // Prefer the int8 MMQ path (quantize src1 to q8_1) whenever a matching pipeline exists. - // The pipeline lookup returns nullptr for types without a q8_1 pipeline (e.g. RDNA4-skipped - // quants), in which case coopmat1 falls back to the f16 B-type quant matmul below. - bool quantize_y = (ctx->device->integer_dot_product || ctx->device->coopmat_int_support) && - src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0; + const bool is_coopmat = ctx->device->coopmat_support || ctx->device->coopmat2; + const bool can_quantize_y = (ctx->device->integer_dot_product || ctx->device->coopmat_int_support) && + src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst); + // coopmat shaders are fast enough that memory bandwidth reduction from f16 activations helps + const bool prefer_f16_b = is_coopmat && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32; - // Check for mmq first - const std::vector<vk_matmul_pipeline_pair>* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; - if (mmp_map == nullptr) { - quantize_y = false; - } + const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type, + ggml_prec_get_acc(dst), false, can_quantize_y, prefer_f16_b); - const bool y_non_contig = (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) || - // coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is - // used, but only when the int8 MMQ path above is not taken. - (ctx->device->coopmat_support && !ctx->device->coopmat2 && !quantize_y && - ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) || - (src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) || - !ggml_vk_dim01_contiguous(src1); + const bool quantize_y = y_kernel_type == GGML_TYPE_Q8_1; + const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32; + const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type); - const bool y_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig; - - if (mmp_map == nullptr) { - // Fall back to f16 dequant mul mat - mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); - } + const std::vector<vk_matmul_pipeline_pair>* mmp_map = + ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst)); const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; - const bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); + const bool qy_needs_dequant = y_needs_reformat; if (qx_needs_dequant) { - // Fall back to dequant + f16 mulmat - mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); + // dequant src0 to f16 + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst)); } - // Not implemented - GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - GGML_ASSERT(mmp_map != nullptr); const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false)); @@ -6391,8 +6766,8 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub } else { to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, src0->type); } - if (y_non_contig) { - to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, f16_type); + if (y_needs_reformat) { + to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, y_kernel_type); } else { to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } @@ -6490,7 +6865,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { vk_subbuffer{ d_Qx, qx_buf_offset, qx_sz }, vk_subbuffer{ d_X, 0, x_sz } }, pc, { (uint32_t)(x_ne), 1, 1}); ggml_vk_sync_buffers(ctx, subctx); } - if (y_non_contig) { + if (y_needs_reformat) { if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || ctx->prealloc_y_last_tensor_used != src1 || ctx->prealloc_y_last_k_padded) { @@ -6533,6 +6908,11 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub stride_batch_y = src1->nb[0] / ggml_type_size(src1->type); } + // column stride in bytes of the B buffer: raw src1 keeps its own stride, staging buffers are tightly packed + const uint64_t b_col_bytes = (y_needs_reformat || qy_needs_dequant || quantize_y) + ? (quantize_y ? (uint64_t)(ne10 / ggml_blck_size(GGML_TYPE_Q8_1)) * ggml_type_size(GGML_TYPE_Q8_1) : (uint64_t)ne10 * sizeof(ggml_fp16_t)) + : (uint64_t)src1->nb[1]; + // compute ggml_vk_matmul( ctx, subctx, pipeline, @@ -6540,13 +6920,14 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, d_sz * split_k }, ne01, ne11, ne10, ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d, - split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n + split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n, + b_col_bytes, (uint64_t)dst->nb[1] ); // NOLINT if (x_non_contig || qx_needs_dequant) { ctx->prealloc_x_need_sync = true; } - if (y_non_contig || quantize_y) { + if (y_needs_reformat || quantize_y) { ctx->prealloc_y_need_sync = true; } } @@ -6671,10 +7052,14 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& bool batch_n = ne11 > 1; const bool x_non_contig = !ggml_vk_dim01_contiguous(src0); - const bool y_non_contig = !ggml_vk_dim01_contiguous(src1); - const bool f16_f32_kernel = src1->type == GGML_TYPE_F32; - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type); + // BF16 src1 has no mul_mat_vec B-type; widen to F32 + const bool widen_bf16 = src1->type == GGML_TYPE_BF16; + const bool y_non_contig = !ggml_vk_dim01_contiguous(src1) || widen_bf16; + + const ggml_type y_kernel_type = widen_bf16 ? GGML_TYPE_F32 : src1->type; + const bool f16_f32_kernel = y_kernel_type == GGML_TYPE_F32; + bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type); vk_pipeline to_fp16_vk_0 = nullptr; vk_pipeline to_fp16_vk_1 = nullptr; @@ -6682,7 +7067,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, src0->type); } if (y_non_contig) { - to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, src1->type); + to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, y_kernel_type); } else { to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } @@ -6693,7 +7078,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& if (dmmv == nullptr) { // Fall back to f16 dequant mul mat - dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, src1->type, ne11, ne20, ne00); + dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, y_kernel_type, ne11, ne20, ne00); quantize_y = false; } @@ -6776,7 +7161,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, d_Qx, d_X); } if (y_non_contig) { - GGML_ASSERT(y_sz == ggml_type_size(src1->type) * y_ne); + GGML_ASSERT(y_sz == ggml_type_size(y_kernel_type) * y_ne); if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || ctx->prealloc_y_last_tensor_used != src1 || ctx->prealloc_y_last_k_padded) { @@ -7004,7 +7389,7 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con const uint32_t nb23 = (uint32_t)(dst->nb[3] / sizeof(float)); GGML_ASSERT(ne11 == 1); - GGML_ASSERT(src0->ne[3] == src1->ne[3]); // checked in supports_op + GGML_ASSERT(src0->ne[3] == src1->ne[3]); // checked in ggml_vk_mul_mat const uint32_t row_stride_x = nb01 / sizeof(ggml_fp16_t); const uint32_t channel_stride_x = nb02 / sizeof(ggml_fp16_t); @@ -7070,6 +7455,10 @@ static int ggml_vk_fwht_pipeline_idx(int64_t n) { case 128: return 1; case 256: return 2; case 512: return 3; + case 1024: return 4; + case 2048: return 5; + case 4096: return 6; + case 8192: return 7; default: return -1; } } @@ -7083,12 +7472,18 @@ bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor return false; } - const int idx = ggml_vk_fwht_pipeline_idx(src1->ne[0]); - if (idx < 0 || ctx->device->pipeline_fwht_f32[idx] == nullptr) { + if (dst->type != GGML_TYPE_F32) { return false; } - if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { + return false; + } + + const bool src_f16 = src1->type == GGML_TYPE_F16; + + const int idx = ggml_vk_fwht_pipeline_idx(src1->ne[0]); + if (idx < 0 || (src_f16 ? ctx->device->pipeline_fwht_f16[idx] : ctx->device->pipeline_fwht_f32[idx]) == nullptr) { return false; } @@ -7102,9 +7497,10 @@ bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) { const int idx = ggml_vk_fwht_pipeline_idx(src->ne[0]); - vk_pipeline pipeline = ctx->device->pipeline_fwht_f32[idx]; + vk_pipeline pipeline = src->type == GGML_TYPE_F16 ? ctx->device->pipeline_fwht_f16[idx] : ctx->device->pipeline_fwht_f32[idx]; - const uint32_t rows_per_workgroup = 4; + const uint32_t rows_per_workgroup = ctx->device->fwht_rows_per_wg[idx]; + GGML_ASSERT(rows_per_workgroup > 0); const uint32_t n_rows = (uint32_t)ggml_nrows(src); const uint32_t max_workgroups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; @@ -7262,7 +7658,7 @@ void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const st } } else if (ggml_vk_can_use_fwht(ctx, src1, dst)) { ggml_vk_fwht(ctx, subctx, src1, dst); - } else if (src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && dst->ne[1] == 1 && + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && dst->ne[1] == 1 && // detect 0213 permutation, and batch size of 1 src0->nb[0] <= src0->nb[2] && src0->nb[2] <= src0->nb[1] && @@ -7275,8 +7671,8 @@ void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const st src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { ggml_vk_mul_mat_vec_p021_f16_f32(ctx, subctx, cgraph, node_idx); - } else if (src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && dst->ne[1] == 1 && - !ggml_is_permuted(src0) && !ggml_is_permuted(src1) && + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && dst->ne[1] == 1 && + !ggml_is_permuted(src0) && !ggml_is_permuted(src1) && src0->ne[3] == src1->ne[3] && src0->ne[3] <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] && src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { @@ -7324,7 +7720,6 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint32_t nbi0 = ids->nb[0]; const uint32_t nbi1 = ids->nb[1]; - const uint32_t nbi2 = ids->nb[2]; const uint64_t ne20 = dst->ne[0]; const uint64_t ne21 = dst->ne[1]; @@ -7332,38 +7727,24 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& // const uint64_t ne23 = dst->ne[3]; const uint64_t n_as = ne02; - // n_as counts, n_as offsets, one total, then one packed row id per (expert, token). - // Hoisting requires 16-bit indices for the packing and a table that fits one binding. - const uint64_t hoisted_row_id_words = 2 * n_as + 1 + nei0 * nei1; - // 1024 matches MAX_EXPERTS in count_experts.comp and LLAMA_MAX_EXPERTS. It costs - // 3 * 1024 * 4 = 12 KiB of shared memory, within the 16 KiB Vulkan guarantees. - const bool hoist_row_ids = n_as <= 1024 && nei0 <= 0xffff && nei1 <= 0xffff && - hoisted_row_id_words * sizeof(uint32_t) <= - ctx->device->properties.limits.maxStorageBufferRange; ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context; ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context; - ggml_backend_vk_buffer_context * ids_buf_ctx = (ggml_backend_vk_buffer_context *)ids->buffer->context; vk_buffer d_Qx = nullptr; size_t qx_buf_offset = 0; vk_buffer d_Qy = nullptr; size_t qy_buf_offset = 0; - vk_buffer d_ids = nullptr; - size_t ids_buf_offset = 0; bool src0_uma = false; bool src1_uma = false; - bool ids_uma = false; if (ctx->device->uma) { ggml_vk_host_get(ctx->device, src0->data, d_Qx, qx_buf_offset); ggml_vk_host_get(ctx->device, src1->data, d_Qy, qy_buf_offset); - ggml_vk_host_get(ctx->device, ids->data, d_ids, ids_buf_offset); src0_uma = d_Qx != nullptr; src1_uma = d_Qy != nullptr; - ids_uma = d_ids != nullptr; } // Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf @@ -7387,52 +7768,49 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& #else const bool y_decode_vector_staging = false; #endif - const bool y_non_contig = y_decode_vector_staging || - (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) || - // Intel coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is used. - (ctx->device->coopmat_support && !ctx->device->coopmat2 && - ctx->device->vendor_id == VK_VENDOR_ID_INTEL && - ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) || - (src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) || - !ggml_vk_dim01_contiguous(src1); + const bool can_quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && + ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst); + // Intel coopmat1 matrix cores prefer f16 activations for quantized weights + const bool prefer_f16_b = ctx->device->coopmat_support && !ctx->device->coopmat2 && + ctx->device->vendor_id == VK_VENDOR_ID_INTEL && + ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32; - const bool y_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig; + const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type, + ggml_prec_get_acc(dst), true, can_quantize_y, prefer_f16_b); - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; + const bool quantize_y = y_kernel_type == GGML_TYPE_Q8_1; + const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32; - // Check for mmq first - const std::vector<vk_matmul_pipeline_pair>* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0], true) : nullptr; - - if (mmp_map == nullptr) { - // Fall back to f16 dequant mul mat - mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0], true); - quantize_y = false; - } + const std::vector<vk_matmul_pipeline_pair>* mmp_map = + ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst), true); const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; - bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); if (qx_needs_dequant) { - // Fall back to dequant + f16 mulmat - mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0], true); + // dequant src0 to f16 + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst), true); } + GGML_ASSERT(mmp_map != nullptr); + // Coopmat2 MUL_MAT_ID BK specialization constants in ggml_vk_load_shaders are at most 64. const uint32_t y_staged_row_stride = ctx->device->coopmat2 && !quantize_y ? ggml_vk_align_size(ne10, 64) : ne10; const bool y_needs_k_padding = ne10 != y_staged_row_stride; - const bool y_needs_reformat = y_non_contig || y_needs_k_padding; - qy_needs_dequant = qy_needs_dequant || y_needs_k_padding; - // Not implemented - GGML_ASSERT(y_needs_reformat || !qy_needs_dequant); // NOLINT - - GGML_ASSERT(mmp_map != nullptr); + const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type || + y_decode_vector_staging || y_needs_k_padding); + const bool qy_needs_dequant = y_needs_reformat; const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, nei1, true)); const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && nei1 > 8; vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, nei1, aligned, true); + const uint32_t bn = pipeline->wg_denoms[1]; + const uint64_t max_tiles = ggml_vk_mul_mat_id_max_tiles(n_as, nei0 * nei1, bn); + const uint64_t row_ids_offset = 1 + 2 * max_tiles; + const uint64_t row_map_words = row_ids_offset + max_tiles * bn; + if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); } @@ -7444,7 +7822,6 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint64_t qy_sz = ggml_type_size(src1->type) * ggml_nelements(src1) / ggml_blck_size(src1->type); const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne; const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (y_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); - const uint64_t ids_sz = nbi2; const uint64_t d_sz = sizeof(float) * d_ne; vk_pipeline to_fp16_vk_0 = nullptr; @@ -7486,8 +7863,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& } vk_pipeline count_experts = ctx->device->pipeline_count_experts; - const size_t expert_data_size = sizeof(uint32_t) * - (hoist_row_ids ? hoisted_row_id_words : n_as); + const size_t expert_data_size = sizeof(uint32_t) * row_map_words; { if ( @@ -7539,11 +7915,6 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& qy_buf_offset = vk_tensor_offset(src1) + src1->view_offs; GGML_ASSERT(d_Qy != nullptr); } - if (!ids_uma) { - d_ids = ids_buf_ctx->dev_buffer; - ids_buf_offset = vk_tensor_offset(ids) + ids->view_offs; - GGML_ASSERT(d_ids != nullptr); - } if (qx_needs_dequant) { d_X = ctx->prealloc_x; GGML_ASSERT(d_X->size >= x_sz); @@ -7569,24 +7940,29 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& ggml_vk_sync_buffers(ctx, subctx); } } + vk_subbuffer d_ids = ggml_vk_tensor_subbuffer(ctx, ids, true); + // Count how many times each expert is used vk_subbuffer expert_count_buf = { ctx->prealloc_split_k, 0, expert_data_size }; if (ctx->prealloc_split_k_need_sync) { ggml_vk_sync_buffers(ctx, subctx); } { + // this also zeroes the dst rows of the skipped slots (negative ids) vk_op_count_experts_push_constants pc = { (uint32_t)nei0, (uint32_t)nei1, (uint32_t)(nbi0 / ggml_type_size(ids->type)), (uint32_t)(nbi1 / ggml_type_size(ids->type)), (uint32_t)(get_misalign_bytes(ctx, ids) / ggml_type_size(ids->type)), (uint32_t)n_as, - uint32_t(hoist_row_ids), - 0, 0 }; + 0, 0, bn, (uint32_t)row_ids_offset, + (uint32_t)ne01, + (uint32_t)(dst->nb[1] / sizeof(float)), + (uint32_t)(dst->nb[2] / sizeof(float)) }; init_pushconst_fastdiv(pc); ggml_vk_dispatch_pipeline(ctx, subctx, count_experts, - { vk_subbuffer{ d_ids, ids_buf_offset, ids_sz }, expert_count_buf }, pc, - { hoist_row_ids ? 1u : (uint32_t)n_as, 1, 1}); + { d_ids, expert_count_buf, vk_subbuffer{ d_D, d_buf_offset, d_sz } }, pc, + { 1, 1, 1 }); } if (x_non_contig) { @@ -7659,10 +8035,10 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& ggml_vk_matmul_id( ctx, subctx, pipeline, { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range }, - { d_D, d_buf_offset, d_sz }, { d_ids, ids_buf_offset, ids_sz }, expert_count_buf, + { d_D, d_buf_offset, d_sz }, std::move(d_ids), expert_count_buf, ne01, ne21, ne10, ne10, stride_b_y, ne01, stride_batch_x, stride_batch_y, ne20*ne21, - n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, hoist_row_ids + n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, row_ids_offset, max_tiles ); // NOLINT if (x_non_contig || qx_needs_dequant) { @@ -7711,7 +8087,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte const bool y_non_contig = !ggml_vk_dim01_contiguous(src1); const bool f16_f32_kernel = src1->type == GGML_TYPE_F32; - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type); + bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type); vk_pipeline to_fp16_vk_0 = nullptr; vk_pipeline to_fp16_vk_1 = nullptr; @@ -7911,11 +8287,14 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte } } +static bool ggml_vk_use_mul_mat_vec_id(const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * ids = dst->src[2]; + return ids->ne[1] <= 8 && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)); +} + bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) { - ggml_tensor * dst = cgraph->nodes[node_idx]; - ggml_tensor * src0 = dst->src[0]; - ggml_tensor * src2 = dst->src[2]; - return (src2->ne[1] <= 8) && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)); + return ggml_vk_use_mul_mat_vec_id(cgraph->nodes[node_idx]); } void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { @@ -8087,7 +8466,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const uint32_t workgroups_y = (uint32_t)neq2; uint32_t workgroups_z = (uint32_t)neq3; - const bool f32acc = !ctx->device->fp16 || dst->op_params[3] == GGML_PREC_F32 || k->type == GGML_TYPE_BF16; + const bool f32acc = !ctx->device->fp16 || !ggml_vk_prec_allows_f16acc(ggml_prec_get_acc(dst)) || k->type == GGML_TYPE_BF16; // dequant K/V once into an f16 scratch, reordered KV layout so FA can read without a stride auto is_dense_kv_cache = [](const ggml_tensor * t) { @@ -8098,19 +8477,37 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const }; const bool k_quant = k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_BF16 && k->type != GGML_TYPE_F32; const bool v_quant = v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_BF16 && v->type != GGML_TYPE_F32; - const bool use_dequant_kv = k_quant && v_quant && neq1 >= 64 && + const bool kv_raw = (k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_BF16) && k->type == v->type; + // gated to AMD, except RDNA4, which regressed in benchmarks + const bool kv_raw_strided = ctx->device->vendor_id == VK_VENDOR_ID_AMD && + ctx->device->architecture != vk_device_architecture::AMD_RDNA4 && + neq1 >= 256 && + kv_raw && + (k->nb[1] != (uint64_t)HSK * ggml_type_size(k->type) || + v->nb[1] != (uint64_t)HSV * ggml_type_size(v->type)) && + (HSK % 8) == 0 && (HSV % 8) == 0; + const size_t kv_scratch_ts = kv_raw_strided ? ggml_type_size(k->type) : sizeof(ggml_fp16_t); + // K and V share one scratch buffer, so the sum must fit in one allocation + const uint64_t dequant_kv_sz = ((uint64_t)ggml_nelements(k) + (uint64_t)ggml_nelements(v)) * kv_scratch_ts; + const uint32_t cm2_min_queries = KV >= 8192 ? 512 : 1024; + const uint32_t cm2_min_reuse = KV >= 8192 ? 4096 : 8192; + // Query reuse must cover the conversion and synchronization cost. + const bool cm2_dequant_kv = KV >= 4096 && neq1 >= cm2_min_queries && (uint64_t) neq1 * qk_ratio >= cm2_min_reuse; + const bool use_dequant_kv = ((k_quant && v_quant) || kv_raw_strided) && neq1 >= 64 && is_dense_kv_cache(k) && is_dense_kv_cache(v) && - (uint64_t)ggml_nelements(k) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange && - (uint64_t)ggml_nelements(v) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange && - ctx->device->pipeline_dequant_transpose[k->type] != nullptr && - ctx->device->pipeline_dequant_transpose[v->type] != nullptr && - // coopmat2 path does not benefit from the f16 scratch - !ctx->device->coopmat2 && + (uint64_t)ggml_nelements(k) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange && + (uint64_t)ggml_nelements(v) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange && + dequant_kv_sz <= ctx->device->max_buffer_size && + dequant_kv_sz <= ctx->device->max_memory_allocation_size && + (kv_raw_strided || + (ctx->device->pipeline_dequant_transpose[k->type] != nullptr && + ctx->device->pipeline_dequant_transpose[v->type] != nullptr)) && // Intel Xe1 regresses, see PR 25494 (ctx->device->vendor_id != VK_VENDOR_ID_INTEL || - (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1)); - const ggml_type k_type_eff = use_dequant_kv ? GGML_TYPE_F16 : k->type; - const ggml_type v_type_eff = use_dequant_kv ? GGML_TYPE_F16 : v->type; + (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1)) && + (!ctx->device->coopmat2 || cm2_dequant_kv); + const ggml_type k_type_eff = (use_dequant_kv && !kv_raw_strided) ? GGML_TYPE_F16 : k->type; + const ggml_type v_type_eff = (use_dequant_kv && !kv_raw_strided) ? GGML_TYPE_F16 : v->type; // For scalar/coopmat1 FA, we can use the "large" size to accommodate qga. // For coopmat2 FA, we always use the small size (which is still pretty large for gqa). @@ -8148,15 +8545,20 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const // cm2 dense is fast, so it needs a larger reduction to win. // With quantized K/V, sparse only breaks even around 16x (measured on RDNA3/RDNA4). const int64_t min_ratio = tuning_params.path == FA_COOPMAT2 ? 4 : (kv_f16 ? 2 : 16); + // coopmat2 vector decode requires 8B strides. + auto sparse_gather_aligned = [](const ggml_tensor * t) { + return (t->type != GGML_TYPE_F16 && t->type != GGML_TYPE_BF16) || + (t->nb[1] | t->nb[2] | t->nb[3]) % (4 * sizeof(ggml_fp16_t)) == 0; + }; const bool use_sparse = !disable_sparse && n_kv_max > 0 && mask && max_bias == 0.0f && logit_softcap == 0.0f && - // the cm2 sparse gather only reads f16 - (kv_f16 || tuning_params.path != FA_COOPMAT2) && + (tuning_params.path != FA_COOPMAT2 || (sparse_gather_aligned(k) && sparse_gather_aligned(v))) && nem0 == KV && (int64_t)KV >= std::max<int64_t>(4096, min_ratio * (int64_t)n_kv_max) && (gqa_ratio > 1 || (tuning_params.path == FA_SCALAR && N == 1)); const uint32_t q_stride = (uint32_t)(nbq1 / ggml_type_size(q->type)); + const uint32_t q_stride_eff = gqa_ratio > 1 ? (uint32_t)(nbq2 / ggml_type_size(q->type)) : q_stride; uint32_t k_stride = (uint32_t)(nbk1 / ggml_type_size(k->type)); uint32_t v_stride = (uint32_t)(nbv1 / ggml_type_size(v->type)); @@ -8179,9 +8581,13 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const nbv3_eff = (uint32_t)((uint64_t)HSV * KV * nev2 * sizeof(ggml_fp16_t)); } const uint32_t alignment = tuning_params.block_cols; - bool aligned = (KV % alignment) == 0 && - // the "aligned" shader variant will forcibly align strides, for performance - (q_stride & 7) == 0 && (k_stride & 7) == 0 && (v_stride & 7) == 0; + // CM2 decoded K/V strides count whole blocks and do not need to be multiples of eight blocks. + const bool cm2_decode_k = tuning_params.path == FA_COOPMAT2 && + k_type_eff != GGML_TYPE_F16 && k_type_eff != GGML_TYPE_BF16; + const bool cm2_decode_v = tuning_params.path == FA_COOPMAT2 && + v_type_eff != GGML_TYPE_F16 && v_type_eff != GGML_TYPE_BF16; + bool aligned = (KV % alignment) == 0 && (q_stride_eff & 7) == 0 && + (cm2_decode_k || (k_stride & 7) == 0) && (cm2_decode_v || (v_stride & 7) == 0); // Need to use the coopmat2 variant that clamps loads when HSK/HSV aren't sufficiently aligned. if (((HSK | HSV) % 16) != 0 && tuning_params.path == FA_COOPMAT2) { @@ -8196,32 +8602,98 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_pipeline pipeline = nullptr; + auto tensor_aligned_with = [&](const ggml_tensor * t, uint32_t in) { + GGML_ASSERT(t != nullptr && in != 0); + return t->nb[1] % in == 0 && t->nb[2] % in == 0 && t->nb[3] % in == 0 && ggml_vk_tensor_buffer_offset(ctx, t) % in == 0; + }; + auto vk_dev = ctx->device.get(); bool xe_fa_opt = false; - bool fa_copy_qstate = false; - bool xe_fa_supported_platform = - (ctx->device.get()->architecture == INTEL_XE2 && ctx->device.get()->properties.deviceID != 0xFD80 && ctx->device.get()->properties.deviceID != 0xFD81) || - (ctx->device.get()->architecture == INTEL_XE1 && ctx->device.get()->coopmat_support && ctx->device.get()->uma); - bool xe_fa_supported_usage = neq0 % 32 == 0 && nev0 % 16 == 0 && q->nb[1] > q->nb[2] && k->nb[1] > k->nb[2] && v->nb[1] > v->nb[2] && mask != nullptr; - bool xe_fa_supported_dtype = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && (mask != nullptr && mask->type == GGML_TYPE_F16); + bool xe_fa_use_2phases = false; + bool xe_fa_supported_platform = vk_dev->xe_fa_support && vk_dev->properties.deviceID != 0xFD80 && vk_dev->properties.deviceID != 0xFD81; + bool xe_fa_supported_usage = neq0 % 32 == 0 && nev0 % 16 == 0 && mask != nullptr && !use_sparse && max_bias == 0.0f && logit_softcap == 0.0f && + tensor_aligned_with(k, 16u) && tensor_aligned_with(v, 16u); + bool xe_fa_supported_dtype = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && + mask != nullptr && mask->type == GGML_TYPE_F16; + xe_fa_opt = xe_fa_supported_platform && xe_fa_supported_usage && xe_fa_supported_dtype; std::pair<vk_pipeline, vk_pipeline> xe_fa_pipeline_dual_phases = { nullptr , nullptr }; vk_pipeline xe_fa_pipeline = nullptr; + size_t size_q_staging = 0; size_t size_p = 0; size_t size_group_max = 0; + size_t size_decode_temp = 0; + uint32_t pv_ratio = uint32_t(neq2 / nev2); + vk_xe_fa_tuning_params xe_fa_params {}; { std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); - auto &pipelines = ctx->device->pipeline_flash_attn_f32_f16; - auto it = pipelines.find(fa_pipeline_state); - if (it != pipelines.end()) { - pipeline = it->second; - } else { - pipelines[fa_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); + if (xe_fa_opt) { + if (neq1 > 8 || pv_ratio > 64) { + uint32_t head_stride_q = uint32_t(nbq2 / ggml_type_size(q->type)); + uint32_t batch_stride_q = uint32_t(nbq3 / ggml_type_size(q->type)); + vk_xe_fa_pipeline_state prefill_state = { (uint32_t)neq0, (uint32_t)nev0, qk_ratio, pv_ratio, 0, false, false, false, false }; + prefill_state.unaligned_seq_len = nek1 % 32 != 0; + prefill_state.unaligned_q = head_stride_q % 8 != 0 || q_stride % 8 != 0 || batch_stride_q % 8 != 0; + prefill_state.partial_row_mask = neq1 > nem1; + prefill_state.multi_head_mask = nem2 > 1; + xe_fa_params = get_xe_fa_tuning_params(vk_dev, prefill_state, true); + if (xe_fa_params.shmem_sz <= vk_dev->properties.limits.maxComputeSharedMemorySize) { + auto& xe_fa_pipelines = ctx->device->pipeline_xe_fa_prefill; + auto it = xe_fa_pipelines.find(prefill_state); + if (it != xe_fa_pipelines.end()) { + xe_fa_pipeline = it->second; + } else { + xe_fa_pipelines[prefill_state] = xe_fa_pipeline = std::make_shared<vk_pipeline_struct>(); + } + } else { + xe_fa_opt = false; + } + } else { + vk_xe_fa_pipeline_state decode_state = { (uint32_t)neq0, (uint32_t)nev0, qk_ratio, pv_ratio, (uint32_t)neq1, false, false, false, false }; + decode_state.unaligned_seq_len = nek1 % 256 != 0; + xe_fa_params = get_xe_fa_tuning_params(vk_dev, decode_state, false); + const uint32_t split_p_size = xe_fa_params.decode_params.p_chunk_ph1; + const size_t max_dim = std::max<size_t>(CEIL_DIV(nek1, split_p_size), 1); + const size_t p_dim = max_dim * split_p_size; + const size_t q_head_padding = 16; + size_q_staging = neq0 * neq1 * (neq2 * neq3 + q_head_padding) * sizeof(ggml_fp16_t); + size_p = neq1 * neq2 * p_dim * neq3 * sizeof(ggml_fp16_t); + size_group_max = neq1 * neq2 * max_dim * neq3 * sizeof(float); + size_decode_temp = size_q_staging + size_p + size_group_max; + uint32_t ph1_wg1 = CEIL_DIV(nek1, xe_fa_params.decode_params.p_chunk_ph1); + if (xe_fa_params.shmem_sz <= vk_dev->properties.limits.maxComputeSharedMemorySize && + ph1_wg1 <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && + size_decode_temp <= ctx->device->properties.limits.maxStorageBufferRange) { + auto& pipelines = ctx->device->pipeline_xe_fa_decode_dual_phases; + auto it = pipelines.find(decode_state); + if (it != pipelines.end()) { + xe_fa_pipeline_dual_phases = it->second; + } else { + pipelines[decode_state] = xe_fa_pipeline_dual_phases = + std::make_pair(std::make_shared<vk_pipeline_struct>(), std::make_shared<vk_pipeline_struct>()); + } + xe_fa_use_2phases = true; + } else { + xe_fa_opt = false; + } + } + } + + if (!xe_fa_opt) { + auto &pipelines = ctx->device->pipeline_flash_attn_f32_f16; + auto it = pipelines.find(fa_pipeline_state); + if (it != pipelines.end()) { + pipeline = it->second; + } else { + pipelines[fa_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); + } } } - assert(pipeline); - // Compile early to initialize wg_denoms. - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + if (!xe_fa_opt) { + assert(pipeline); + // Compile early to initialize wg_denoms. + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + } uint32_t split_kv = KV; uint32_t split_k = 1; @@ -8235,7 +8707,9 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const const uint32_t Br = fa_pipeline_state.Br; const uint32_t Bc = fa_pipeline_state.Bc; - GGML_ASSERT(Br == pipeline->wg_denoms[0]); + if (!xe_fa_opt) { + GGML_ASSERT(Br == pipeline->wg_denoms[0]); + } const uint32_t Tr = CEIL_DIV(N, Br); // Try to use split_k when KV is large enough to be worth the overhead. @@ -8265,45 +8739,24 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const // of "align", so recompute split_k based on that. split_kv = ROUNDUP_POW2(std::max(1u, KV / split_k), alignment); split_k = CEIL_DIV(KV, split_kv); - xe_fa_opt = xe_fa_supported_platform && xe_fa_supported_usage && xe_fa_supported_dtype; - if (xe_fa_opt) { - const uint32_t split_p_size = 32; - const size_t max_dim = (nek1 + split_p_size - 1) / split_p_size; - const size_t p_dim = max_dim * split_p_size; -#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); - auto& pipelines = ctx->device->pipeline_xe_fa_decode_dual_phases; - auto it = pipelines.find({ (uint32_t)neq0, (uint32_t)nev0, qk_ratio, (uint32_t)neq1 }); - if (it != pipelines.end()) { - xe_fa_pipeline_dual_phases = it->second; - } else { - pipelines[{(uint32_t)neq0, (uint32_t)nev0, qk_ratio, (uint32_t)neq1}] = xe_fa_pipeline_dual_phases = std::make_pair(std::make_shared<vk_pipeline_struct>(), std::make_shared<vk_pipeline_struct>()); + } + + if (xe_fa_opt) { + // Bypass mask_opt and split_k buffer grow when they are not in use + use_mask_opt = false; + split_k = 1; + if (xe_fa_use_2phases) { + if (ctx->prealloc_size_x < size_decode_temp) { + ctx->prealloc_size_x = size_decode_temp; + ggml_vk_preallocate_buffers(ctx, subctx); } -#endif - if (xe_fa_pipeline_dual_phases.first == nullptr || xe_fa_pipeline_dual_phases.second == nullptr) { - xe_fa_opt = false; - fa_copy_qstate = false; - } else { - size_p = neq1 * neq2 * p_dim * neq3 * sizeof(ggml_fp16_t); - size_group_max = neq1 * neq2 * max_dim * neq3 * sizeof(float); - size_t temp_size = ggml_nelements(q) * sizeof(ggml_fp16_t) + size_p + size_group_max; - fa_copy_qstate = true; - if (ctx->prealloc_size_x < temp_size) { - ctx->prealloc_size_x = temp_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } } } - if (xe_fa_opt == true) { - use_mask_opt = false; - } - // Reserve space for split_k temporaries. For each split x batch, we need to store the O matrix (D x ne1) // and the per-row m and L values (ne1 rows). We store all the matrices first, followed by the rows. // For matrices, the order is (inner to outer) [HSV, ne1, k, ne2, ne3]. @@ -8382,28 +8835,56 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const } if (use_dequant_kv) { - const uint64_t fp = sizeof(ggml_fp16_t); - const uint64_t k_f16_sz = (uint64_t)ggml_nelements(k) * fp; - const uint64_t v_f16_sz = (uint64_t)ggml_nelements(v) * fp; - if (ctx->prealloc_size_x < k_f16_sz + v_f16_sz) { - ctx->prealloc_size_x = k_f16_sz + v_f16_sz; + const uint64_t k_sz = (uint64_t)ggml_nelements(k) * kv_scratch_ts; + const uint64_t v_sz = (uint64_t)ggml_nelements(v) * kv_scratch_ts; + if (ctx->prealloc_size_x < k_sz + v_sz) { + ctx->prealloc_size_x = k_sz + v_sz; ggml_vk_preallocate_buffers(ctx, subctx); } - vk_pipeline tr_k = ctx->device->pipeline_dequant_transpose[k->type]; - vk_pipeline tr_v = ctx->device->pipeline_dequant_transpose[v->type]; - ggml_pipeline_request_descriptor_sets(ctx, tr_k, 1); - ggml_pipeline_request_descriptor_sets(ctx, tr_v, 1); - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - vk_subbuffer k_dst = vk_subbuffer{ ctx->prealloc_x, 0, k_f16_sz }; - vk_subbuffer v_dst = vk_subbuffer{ ctx->prealloc_x, k_f16_sz, v_f16_sz }; + vk_subbuffer k_dst = vk_subbuffer{ ctx->prealloc_x, 0, k_sz }; + vk_subbuffer v_dst = vk_subbuffer{ ctx->prealloc_x, k_sz, v_sz }; const uint32_t k_nel = (uint32_t)ggml_nelements(k); const uint32_t v_nel = (uint32_t)ggml_nelements(v); - { const std::vector<uint32_t> pc = { (uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, 0, k_nel }; - ggml_vk_dispatch_pipeline(ctx, subctx, tr_k, { k_buf, k_dst }, pc, { k_nel, 1, 1 }); } - { const std::vector<uint32_t> pc = { (uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, 0, v_nel }; - ggml_vk_dispatch_pipeline(ctx, subctx, tr_v, { v_buf, v_dst }, pc, { v_nel, 1, 1 }); } + if (kv_raw_strided) { + vk_pipeline cp_k = ggml_vk_get_cpy_pipeline(ctx, k, nullptr, k->type); + vk_pipeline cp_v = ggml_vk_get_cpy_pipeline(ctx, v, nullptr, v->type); + ggml_pipeline_request_descriptor_sets(ctx, cp_k, 1); + ggml_pipeline_request_descriptor_sets(ctx, cp_v, 1); + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + auto make_pc = [](uint32_t hs, uint32_t nh, uint32_t kv, uint32_t nel) { + vk_op_unary_push_constants pc{}; + pc.ne = nel; + pc.ne00 = hs; pc.ne01 = nh; pc.ne02 = kv; pc.ne03 = nel / (hs * nh * kv); + pc.nb00 = 1; pc.nb01 = hs; pc.nb02 = hs * nh; pc.nb03 = hs * nh * kv; + pc.ne10 = hs; pc.ne11 = nh; pc.ne12 = kv; pc.ne13 = pc.ne03; + pc.nb10 = 1; pc.nb11 = hs * kv; pc.nb12 = hs; pc.nb13 = hs * kv * nh; + init_pushconst_fastdiv(pc); + return pc; + }; + auto cpy_elems = [](uint32_t ne) -> std::array<uint32_t, 3> { + if (ne > 262144) { return { 512, 512, CEIL_DIV(ne, 262144) }; } + if (ne > 512) { return { 512, CEIL_DIV(ne, 512), 1 }; } + return { ne, 1, 1 }; + }; + ggml_vk_dispatch_pipeline(ctx, subctx, cp_k, { k_buf, k_dst }, + make_pc((uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, k_nel), cpy_elems(k_nel)); + ggml_vk_dispatch_pipeline(ctx, subctx, cp_v, { v_buf, v_dst }, + make_pc((uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, v_nel), cpy_elems(v_nel)); + } else { + vk_pipeline tr_k = ctx->device->pipeline_dequant_transpose[k->type]; + vk_pipeline tr_v = ctx->device->pipeline_dequant_transpose[v->type]; + ggml_pipeline_request_descriptor_sets(ctx, tr_k, 1); + ggml_pipeline_request_descriptor_sets(ctx, tr_v, 1); + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + { const std::vector<uint32_t> pc = { (uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, 0, k_nel }; + ggml_vk_dispatch_pipeline(ctx, subctx, tr_k, { k_buf, k_dst }, pc, { k_nel, 1, 1 }); } + { const std::vector<uint32_t> pc = { (uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, 0, v_nel }; + ggml_vk_dispatch_pipeline(ctx, subctx, tr_v, { v_buf, v_dst }, pc, { v_nel, 1, 1 }); } + } ggml_vk_sync_buffers(ctx, subctx); k_buf = k_dst; v_buf = v_dst; @@ -8462,67 +8943,59 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const mask_n_head_log2, m0, m1, gqa_ratio, split_kv, split_k }; - if (xe_fa_opt && split_k > 1) { - auto upper_power_of_2 = [&](uint32_t in) { - GGML_ASSERT(in != 0); - if (in <= 1) return 1u; - uint32_t ret = in - 1; - ret |= ret >> 1; - ret |= ret >> 2; - ret |= ret >> 4; - ret |= ret >> 8; - ret |= ret >> 16; - return ret + 1; - }; - auto to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, q->type); - const uint32_t out_dim_per_wg = qk_ratio > 16 ? 8 : 16; - size_t x_ne = ggml_nelements(q); - size_t temp_buf_offset = 0; + if (xe_fa_opt) { + uint32_t head_stride_q = uint32_t(nbq2 / ggml_type_size(q->type)); uint32_t head_stride_k = uint32_t(nbk2 / ggml_type_size(k->type)); uint32_t head_stride_v = uint32_t(nbv2 / ggml_type_size(v->type)); uint32_t batch_stride_q = uint32_t(nbq3 / ggml_type_size(q->type)); uint32_t batch_stride_k = uint32_t(nbk3 / ggml_type_size(k->type)); uint32_t batch_stride_v = uint32_t(nbv3 / ggml_type_size(v->type)); - uint32_t batch_stride_m = mask ? uint32_t(mask->nb[3] / ggml_type_size(mask->type)) : 0u; uint32_t batch_stride_o = uint32_t(nb3 / ggml_type_size(dst->type)); - vk_fa_xe_opt_push_constants pc_ph1 = { (uint32_t)nek1, (uint32_t)neq1, (uint32_t)neq2, (uint32_t)nek2, qk_ratio, 1, (sinks != nullptr) ? 1u : 0u, (uint32_t)k_stride, head_stride_k, - batch_stride_q, batch_stride_k, batch_stride_v, batch_stride_m, batch_stride_o, scale }; - vk_fa_xe_opt_push_constants pc_ph2 = pc_ph1; - pc_ph2.nbkv_tok = v_stride; - pc_ph2.nbkv_head = head_stride_v; - vk_subbuffer q_temp_buf = fa_copy_qstate ? ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset) : q_buf; - temp_buf_offset += fa_copy_qstate ? x_ne * sizeof(ggml_fp16_t) : 0; - vk_subbuffer p_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); - temp_buf_offset += size_p; - vk_subbuffer max_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); - temp_buf_offset += size_group_max; - uint32_t xe_native_sub_group_size = ctx->device.get()->architecture == INTEL_XE1 ? 8 : 16; - uint32_t aligned_gqa_ratio = upper_power_of_2(qk_ratio); - uint32_t out_per_wg_ph1 = std::min(256u / xe_native_sub_group_size, (uint32_t)neq1); - uint32_t out_per_wg_ph2 = std::min(std::max(16u / aligned_gqa_ratio, 1u), (uint32_t)neq1); - uint32_t ph1_wg = ((neq1 + out_per_wg_ph1 - 1) / out_per_wg_ph1) * nek2; - uint32_t ph2_wg = ((neq1 + out_per_wg_ph2 - 1) / out_per_wg_ph2) * ne0 / out_dim_per_wg; - if (fa_copy_qstate) { - const std::vector<uint32_t> pc_cpy_fp16 = - { (uint32_t)q->ne[0], (uint32_t)q->ne[1], (uint32_t)q->ne[2], (uint32_t)q->ne[3], (uint32_t)(x_ne) }; - ggml_vk_sync_buffers(ctx, subctx); + + vk_xe_fa_push_constants pc_xe_fa = { (uint32_t)neq1, (uint32_t)neq2, (uint32_t)neq3, + (uint32_t)nek1, (uint32_t)nek2, (uint32_t)nek3, (uint32_t)nev2, (uint32_t)nev3, + (uint32_t)nem1, (uint32_t)nem2, (uint32_t)nem3, + q_stride, head_stride_q, k_stride, head_stride_k, v_stride, head_stride_v, + batch_stride_q, batch_stride_k, batch_stride_v, batch_stride_o, + (sinks != nullptr) ? (1u << 16) : 0u, scale }; + + if (xe_fa_use_2phases) { + auto to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, q, nullptr, GGML_TYPE_F16); + size_t temp_buf_offset = 0; + vk_subbuffer q_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); + temp_buf_offset += size_q_staging; + vk_subbuffer p_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); + temp_buf_offset += size_p; + vk_subbuffer max_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); + temp_buf_offset += size_group_max; + uint32_t ph1_wg = CEIL_DIV(neq1, xe_fa_params.decode_params.out_per_wg_ph1) * nek2; + uint32_t ph2_wg = CEIL_DIV(neq1, xe_fa_params.decode_params.out_row_per_wg_ph2) * ne0 / xe_fa_params.decode_params.out_col_per_wg_ph2; + + pc_xe_fa.nbq1 = uint32_t(neq0); + pc_xe_fa.nbq2 = uint32_t(neq0 * neq1); + pc_xe_fa.nbq3 = uint32_t(neq0 * neq1 * neq2); + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { q_buf, q_temp_buf }, pc_cpy_fp16, { (uint32_t)(x_ne), 1, 1 }); - } + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, q, q_buf, q_temp_buf); - ggml_vk_sync_buffers(ctx, subctx); - ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.first, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.first, - { q_temp_buf, k_buf, mask_buf, p_temp_buf, max_temp_buf }, - pc_ph1, { (uint32_t)ph1_wg, (uint32_t)nek1, (uint32_t)neq3 }); + ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.first, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.first, + { q_temp_buf, k_buf, mask_buf, p_temp_buf, max_temp_buf }, + pc_xe_fa, { (uint32_t)ph1_wg, (uint32_t)nek1, (uint32_t)neq3 }); - ggml_vk_sync_buffers(ctx, subctx); - ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.second, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.second, - { p_temp_buf, v_buf, max_temp_buf, sinks_buf, dst_buf }, - pc_ph2, { (uint32_t)ph2_wg, (uint32_t)nev2, (uint32_t)neq3 }); + ggml_vk_sync_buffers(ctx, subctx); + ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.second, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.second, + { p_temp_buf, v_buf, max_temp_buf, sinks_buf, dst_buf }, + pc_xe_fa, { (uint32_t)ph2_wg, (uint32_t)nev2, (uint32_t)neq3 }); - ctx->prealloc_x_need_sync = true; + ctx->prealloc_x_need_sync = true; + } else { + ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline, + { q_buf, k_buf, v_buf, mask_buf, sinks_buf, dst_buf }, + pc_xe_fa, { (uint32_t)neq2, (uint32_t)neq1, (uint32_t)neq3 }); + } } else if (split_k > 1) { ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_flash_attn_split_k_reduce, 1); @@ -8601,6 +9074,56 @@ static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, u } } +static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) { + const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128]; + const uint32_t shader_core_count = ctx->device->shader_core_count; + // Slowdowns were observed on DGX Spark with drivers older than R615. + const bool driver_supported = ctx->device->driver_id != vk::DriverId::eNvidiaProprietary || + (ctx->device->properties.driverVersion >> 22) >= 615; + if (ctx->device->coopmat2 && shader_core_count > 0 && K % block.K == 0 && driver_supported) { + const uint64_t n_tiles = (uint64_t)(K / block.K) * CEIL_DIV((uint64_t)NPQ, block.NPQ); + // Use the largest cm2 padding and include the dummy Csh[8]. + const uint32_t pad = 8, csh_elems = 8; + const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t); + // Reuse each loaded input across 256 output channels instead of 128. + // Require full channel tiles and at least as many workgroups as GPU cores. + if (n_tiles >= shader_core_count && shmem <= ctx->device->properties.limits.maxComputeSharedMemorySize) { + return CONV_SHAPE_256x128; + } + } + return ggml_vk_conv_select_shape(ctx, K, NPQ); +} + +static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> & pipelines, int64_t ncols) { + if (pipelines[0] && ncols <= 128) { + return pipelines[0]; + } + if (pipelines[1] && ncols <= 256) { + return pipelines[1]; + } + return pipelines[2]; +} + +static bool ggml_vk_concat_is_transposed(const vk_device & device, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { + if (ggml_get_op_params_i32(dst, 0) != 0) { // dim 0 only + return false; + } + if (src0->ne[2] != 1 || src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1) { + return false; + } + const size_t ts = ggml_type_size(src0->type); + if (src0->nb[0] != ts || dst->nb[0] != ts) { // src0 and dst rows must be contiguous + return false; + } + if (src1->nb[0] <= src1->nb[1]) { // src1 must actually be transposed + return false; + } + if (CEIL_DIV(src1->ne[1], 32) > device->properties.limits.maxComputeWorkGroupCount[0]) { + return false; + } + return src0->ne[1] == src1->ne[1] && dst->ne[1] == src1->ne[1]; +} + static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_GET_ROWS: @@ -8693,6 +9216,18 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (!ggml_vk_concat_supported(src0, src1, dst)) { return nullptr; } + if (!ggml_is_quantized(src0->type) && ggml_vk_concat_is_transposed(ctx->device, src0, src1, dst)) { + switch (ggml_vk_concat_unit_size(src0->type)) { + case 1: + return ctx->device->pipeline_concat_transpose_i8; + case 2: + return ctx->device->pipeline_concat_transpose_i16; + case 4: + return ctx->device->pipeline_concat_transpose_i32; + default: + break; + } + } switch (ggml_vk_concat_unit_size(src0->type)) { case 1: return ctx->device->pipeline_concat_i8; @@ -8841,7 +9376,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (ctx->do_add_rms_partials) { return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32; } - return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32; + return ggml_vk_rms_norm_pipeline(ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32, src0->ne[0]); } return nullptr; case GGML_OP_RMS_NORM_BACK: @@ -8936,11 +9471,6 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const break; } return nullptr; - case GGML_OP_DIAG_MASK_INF: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_diag_mask_inf_f32; - } - return nullptr; case GGML_OP_SOFT_MAX: GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F32); @@ -9111,6 +9641,10 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return nullptr; case GGML_OP_POOL_2D: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + // Wide windows use neighboring subgroup lanes to read neighboring inputs. + if (ctx->device->pipeline_pool2d_subgroup_f32 && dst->op_params[1] >= int32_t(ctx->device->subgroup_size)) { + return ctx->device->pipeline_pool2d_subgroup_f32; + } return ctx->device->pipeline_pool2d_f32; } return nullptr; @@ -9266,7 +9800,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const const uint32_t IC = (uint32_t)ggml_get_op_params_i32(dst, 9); const uint32_t N = (uint32_t)ggml_get_op_params_i32(dst, 10); const uint32_t NPQ = N * dst->ne[2] * dst->ne[1] * dst->ne[0]; - const vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, OC, NPQ); + const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, OC, NPQ); const uint32_t KW = (uint32_t)src0->ne[0]; const uint32_t KH = (uint32_t)src0->ne[1]; @@ -9315,17 +9849,6 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return pipeline; } return nullptr; - case GGML_OP_ADD1: - if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_add1_f16_f16; - } - if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_add1_f16_f32; - } - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_add1_f32_f32; - } - return nullptr; case GGML_OP_ARANGE: if (dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_arange_f32; @@ -9445,6 +9968,8 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co elements = { (uint32_t)CEIL_DIV(ne00, 128), 1, 1 }; } else { elements = { (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne03 }; + elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); } break; @@ -9454,12 +9979,14 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co break; case GGML_OP_GROUP_NORM: { - const uint32_t num_groups = dst->op_params[0]; - elements = { num_groups * (uint32_t)src0->ne[3], 1, 1 }; + const uint32_t total_workgroups = dst->op_params[0] * (uint32_t)src0->ne[3]; + const uint32_t groups_x = std::min(total_workgroups, ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + const uint64_t groups_y_needed = CEIL_DIV(uint64_t(total_workgroups), groups_x); + const uint32_t groups_y = std::min<uint64_t>(groups_y_needed, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + const uint64_t groups_xy = uint64_t(groups_x) * groups_y; + elements = { groups_x, groups_y, uint32_t(CEIL_DIV(uint64_t(total_workgroups), groups_xy)) }; + GGML_ASSERT(elements[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]); } break; - case GGML_OP_DIAG_MASK_INF: - elements = { (uint32_t)ggml_nrows(src0), (uint32_t)ne00, 1 }; - break; case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: { @@ -9551,11 +10078,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co } break; case GGML_OP_POOL_2D: { - const uint32_t N = dst->ne[3]; - const uint32_t OC = dst->ne[2]; - const uint32_t OH = dst->ne[1]; - const uint32_t OW = dst->ne[0]; - elements = { N * OC * OH * OW, 1, 1}; + const uint32_t outputs = ggml_nelements(dst); + const uint32_t outputs_per_workgroup = pipeline == ctx->device->pipeline_pool2d_subgroup_f32 ? 128 / ctx->device->subgroup_size : 512; + const uint32_t workgroups = CEIL_DIV(outputs, outputs_per_workgroup); + const uint32_t groups_y = CEIL_DIV(workgroups, ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + const uint32_t groups_x = CEIL_DIV(workgroups, groups_y); + elements = { groups_x, groups_y, 1 }; } break; case GGML_OP_CONV_2D: case GGML_OP_CONV_TRANSPOSE_2D: @@ -9576,7 +10104,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co case GGML_OP_CONV_3D: if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) { const uint32_t NPQ = pc.N * pc.OD * pc.OH * pc.OW; - const vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, pc.OC, NPQ); + const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, pc.OC, NPQ); const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ); elements = { pc.OC, NPQ_blocks, 1 }; @@ -9592,7 +10120,6 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co case GGML_OP_SUB: case GGML_OP_DIV: case GGML_OP_MUL: - case GGML_OP_ADD1: case GGML_OP_OUT_PROD: case GGML_OP_ARANGE: case GGML_OP_FILL: @@ -9618,6 +10145,16 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co case GGML_OP_GLU: case GGML_OP_CONV_2D_DW: { + // dispatched per 32x32 tile, not per element + if (op == GGML_OP_CONCAT && + (pipeline == ctx->device->pipeline_concat_transpose_i8 || + pipeline == ctx->device->pipeline_concat_transpose_i16 || + pipeline == ctx->device->pipeline_concat_transpose_i32)) { + const uint32_t tiles_y = CEIL_DIV(dst->ne[0], 32); + const uint32_t groups_y = std::min(tiles_y, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements = { (uint32_t)src1->ne[1], groups_y * 32, CEIL_DIV(tiles_y, groups_y) }; + break; + } uint32_t ne = ggml_nelements(dst); if (op == GGML_OP_CPY && ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) { // Convert from number of logical elements to 2- or 4-byte units. @@ -10477,21 +11014,6 @@ void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQRT, vk_op_unary_push_constants_init(src0, dst)); } -void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ADD1, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} - void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { VK_LOG_DEBUG("ggml_vk_arange(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")"); @@ -10674,8 +11196,9 @@ void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const const uint32_t num_groups = int_op_params[0]; const float eps = float_op_params[1]; const uint32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups); + const uint32_t num_workgroups = num_groups * src0->ne[3]; - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_GROUP_NORM, { group_size, 0, eps, 0.0f, 0.0f, 0.0f }); + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_GROUP_NORM, { group_size, num_workgroups, eps, 0.0f, 0.0f, 0.0f }); } static uint32_t ggml_vk_rms_num_partials(ggml_backend_vk_context * ctx, const ggml_tensor *node) { @@ -10695,7 +11218,6 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor * const int n_dims = ((const int32_t *) dst->op_params)[1]; const int mode = ((const int32_t *) dst->op_params)[2]; const int n_offs = ((const int32_t *) dst->op_params)[15]; - // const int n_ctx = ((const int32_t *) dst->op_params)[3]; const int n_ctx_orig = ((const int32_t *) dst->op_params)[4]; const float freq_base = ((const float *) dst->op_params)[5]; const float freq_scale = ((const float *) dst->op_params)[6]; @@ -10711,7 +11233,7 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor * const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(dst), corr_dims); const float theta_scale = powf(freq_base, -2.0f/n_dims); @@ -10767,7 +11289,11 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s ggml_vk_tensor_subbuffer(ctx, src0, true), ggml_vk_tensor_subbuffer(ctx, set_rows, true), ggml_vk_tensor_subbuffer(ctx, indices), - }, pc, { (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3] }); + }, pc, { + (uint32_t)src0->ne[1], + std::min((uint32_t)src0->ne[2], ctx->device->properties.limits.maxComputeWorkGroupCount[1]), + std::min((uint32_t)src0->ne[3], ctx->device->properties.limits.maxComputeWorkGroupCount[2]), + }); ggml_vk_rms_norm_finish(ctx, src0); return; } @@ -10792,8 +11318,8 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s pipeline = do_post_multiply ? ctx->device->pipeline_rms_norm_mul_add_mul_partials_f32 : ctx->device->pipeline_rms_norm_mul_add_partials_f32; } else { - pipeline = do_post_multiply ? - ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32; + pipeline = ggml_vk_rms_norm_pipeline(do_post_multiply ? + ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32, src0->ne[0]); } ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); if (ctx->do_add_rms_partials) { @@ -10814,7 +11340,11 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s ggml_vk_tensor_subbuffer(ctx, dst, true), ggml_vk_tensor_subbuffer(ctx, residual), ggml_vk_tensor_subbuffer(ctx, post_scale), - }, pc, { (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3] }); + }, pc, { + (uint32_t)src0->ne[1], + std::min((uint32_t)src0->ne[2], ctx->device->properties.limits.maxComputeWorkGroupCount[1]), + std::min((uint32_t)src0->ne[3], ctx->device->properties.limits.maxComputeWorkGroupCount[2]), + }); } ggml_vk_rms_norm_finish(ctx, src0); return; @@ -10862,7 +11392,8 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s pc.bin = bin; pc.rope = ggml_vk_make_rope_constants(rope, rope->src[0], tensors[4] != nullptr, false, set_rows_stride); - vk_pipeline pipeline = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32; + const auto & pipelines = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32; + vk_pipeline pipeline = ggml_vk_rms_norm_pipeline(pipelines, src0->ne[0]); ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); @@ -10901,6 +11432,8 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s std::array<uint32_t, 3> elements; elements = { (uint32_t)rms->src[0]->ne[1], (uint32_t)rms->src[0]->ne[2], (uint32_t)rms->src[0]->ne[3] }; + elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); static_assert(max_tensors == 7); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, @@ -10923,7 +11456,7 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f }); + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)ggml_nrows(src0), op_params[0], 0.0f, 0.0f, 0.0f }); } void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { @@ -10995,11 +11528,6 @@ void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_t }); } -void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32<vk_op_diag_mask_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); -} - void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; @@ -11185,20 +11713,6 @@ void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_ const ggml_tensor * src1 = dst->src[1]; const ggml_tensor * src2 = dst->src[2]; const ggml_tensor * src3 = nullptr; - const int n_dims = ((int32_t *) dst->op_params)[1]; - const int mode = ((int32_t *) dst->op_params)[2]; - // const int n_ctx = ((int32_t *) dst->op_params)[3]; - const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; - const float freq_base = ((float *) dst->op_params)[5]; - const float beta_fast = ((float *) dst->op_params)[9]; - const float beta_slow = ((float *) dst->op_params)[10]; - int sections[4] {}; - if (mode & GGML_ROPE_TYPE_MROPE) { - memcpy(sections, (int32_t *) dst->op_params + 11, sizeof(int)*4); - } - - float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); uint32_t set_rows_stride = 0; // Fused rope + view + set_rows passes the set_rows destination stride in set_rows_stride @@ -11601,6 +12115,7 @@ void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subct sp.n_cols = nrows; sp.ne01 = 1; sp.ne02 = 1; + sp.ne03 = 1; sp.weight = 1.0f; init_pushconst_fastdiv(sp); sp.misalign_offsets = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); @@ -12203,11 +12718,12 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM && cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] && ggml_nrows(cgraph->nodes[next_node_idx]) == 1 && - ctx->device->add_rms_fusion) { + ctx->device->add_rms_fusion && !ctx->device->disable_fusion) { uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]); ctx->do_add_rms_partials_offset_calculation = true; if (ctx->prealloc_size_add_rms_partials_offset + size <= ctx->prealloc_size_add_rms_partials) { ctx->do_add_rms_partials = true; + ggml_vk_count_fusion(ctx->device, "ADD_RMS_NORM_PARTIALS"); } } } @@ -12383,10 +12899,6 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in case GGML_OP_UPSCALE: ggml_vk_upscale(ctx, compute_ctx, src0, node); - break; - case GGML_OP_ADD1: - ggml_vk_add1(ctx, compute_ctx, src0, src1, node); - break; case GGML_OP_ARANGE: ggml_vk_arange(ctx, compute_ctx, node); @@ -12537,10 +13049,6 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in default: return false; } - break; - case GGML_OP_DIAG_MASK_INF: - ggml_vk_diag_mask_inf(ctx, compute_ctx, src0, node); - break; case GGML_OP_SOFT_MAX: if (ctx->fused_topk_moe_mode != TOPK_MOE_COUNT) { @@ -12823,10 +13331,7 @@ void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { ggml_vk_command_pool_cleanup(ctx->device, ctx->transfer_cmd_pool); } - for (size_t i = 0; i < ctx->gc.semaphores.size(); i++) { - ctx->device->device.destroySemaphore({ ctx->gc.semaphores[i].s }); - } - ctx->gc.semaphores.clear(); + ctx->binary_semaphore_idx = 0; for (size_t i = 0; i < ctx->gc.tl_semaphores.size(); i++) { ctx->device->device.destroySemaphore({ ctx->gc.tl_semaphores[i].s }); @@ -12874,6 +13379,11 @@ void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { } ctx->gc.events.clear(); + for (auto & sem : ctx->gc.semaphores) { + ctx->device->device.destroySemaphore(sem.s); + } + ctx->gc.semaphores.clear(); + ctx->device->device.destroyFence(ctx->fence); ctx->device->device.destroyFence(ctx->almost_ready_fence); @@ -13025,6 +13535,54 @@ bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_ UNUSED(buffer); } +static vk_buffer & ggml_vk_tensor_dev_buffer(const ggml_tensor * tensor) { + ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; + return ((ggml_backend_vk_buffer_context *) buffer->context)->dev_buffer; +} + +void ggml_backend_vk_buffer_set_tensor_n(ggml_backend_buffer_t buffer, ggml_tensor * const * tensors, const void * const * data, + const size_t * offsets, const size_t * sizes, size_t n_tensors) { + VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_n(" << buffer << ", " << n_tensors << ")"); + std::vector<vk_buffer_region> regions(n_tensors); + for (size_t i = 0; i < n_tensors; i++) { + regions[i] = { ggml_vk_tensor_dev_buffer(tensors[i]), vk_tensor_offset(tensors[i]) + tensors[i]->view_offs + offsets[i], const_cast<void *>(data[i]), sizes[i] }; + } + ggml_vk_buffer_write_n(regions.data(), n_tensors); + + UNUSED(buffer); +} + +void ggml_backend_vk_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * tensors, void * const * data, + const size_t * offsets, const size_t * sizes, size_t n_tensors) { + VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_n(" << buffer << ", " << n_tensors << ")"); + std::vector<vk_buffer_region> regions(n_tensors); + for (size_t i = 0; i < n_tensors; i++) { + regions[i] = { ggml_vk_tensor_dev_buffer(tensors[i]), vk_tensor_offset(tensors[i]) + tensors[i]->view_offs + offsets[i], data[i], sizes[i] }; + } + ggml_vk_buffer_read_n(regions.data(), n_tensors); + + UNUSED(buffer); +} + +bool ggml_backend_vk_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * srcs, ggml_tensor * const * dsts, size_t n_tensors) { + VK_LOG_DEBUG("ggml_backend_vk_buffer_cpy_tensor_n(" << buffer << ", " << n_tensors << ")"); + for (size_t i = 0; i < n_tensors; i++) { + if (!ggml_backend_buffer_is_vk(srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer)) { + return false; + } + } + + std::vector<vk_buffer_copy_region> regions(n_tensors); + for (size_t i = 0; i < n_tensors; i++) { + regions[i] = { ggml_vk_tensor_dev_buffer(dsts[i]), vk_tensor_offset(dsts[i]) + dsts[i]->view_offs, + ggml_vk_tensor_dev_buffer(srcs[i]), vk_tensor_offset(srcs[i]) + srcs[i]->view_offs, ggml_nbytes(srcs[i]) }; + } + ggml_vk_buffer_copy_n(regions.data(), n_tensors); + return true; + + UNUSED(buffer); +} + void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; @@ -13087,18 +13645,16 @@ static const char * ggml_backend_vk_host_buffer_type_name(ggml_backend_buffer_ty static void ggml_backend_vk_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { VK_LOG_MEMORY("ggml_backend_vk_host_buffer_free_buffer()"); - ggml_vk_host_free(vk_instance.devices[0], buffer->context); + ggml_vk_host_free(buffer->context); } static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { VK_LOG_MEMORY("ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")"); size += 32; // Behave like the CPU buffer type - void * ptr = nullptr; - try { - ptr = ggml_vk_host_malloc(vk_instance.devices[0], size); - } catch (vk::SystemError& e) { - GGML_LOG_WARN("ggml_vulkan: Failed to allocate pinned memory (%s)\n", e.what()); + void * ptr = ggml_vk_host_malloc(size); + if (ptr == nullptr) { + GGML_LOG_WARN("ggml_vulkan: Failed to allocate pinned memory\n"); // fallback to cpu buffer return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); } @@ -14276,7 +14832,9 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg flops_cap = 2'000'000'000ULL * ctx->device->shader_core_count; } } - uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u); + // small graphs gain nothing from splitting, each submit costs CPU time + uint64_t flops_per_submit = ctx->last_total_flops < 1'000'000'000ULL ? 0 : + std::clamp(ctx->last_total_flops / 40u, std::min<uint64_t>(100'000'000ULL, flops_cap), flops_cap); auto const submit_after = [&](int start, int end) { if (ctx->device->serialize_submissions) { @@ -14613,6 +15171,16 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg } if (enqueued) { + if (fusion_string && ctx->device->fusion_stats_enabled.load(std::memory_order_relaxed)) { + std::string label = fusion_string; + if (cgraph->nodes[i]->op == GGML_OP_ADD) { + label += "_" + std::to_string(ctx->num_additional_fused_ops + 1); + } + if (ctx->fused_topk_moe_scale) { + label += "_SCALE"; + } + ggml_vk_count_fusion(ctx->device, label.c_str()); + } ++submitted_nodes; #ifndef GGML_VULKAN_CHECK_RESULTS @@ -15210,7 +15778,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total *total += heap.size; if (membudget_supported && i < budgetprops.heapUsage.size()) { - *free += budgetprops.heapBudget[i] - budgetprops.heapUsage[i]; + // Driver may report heapBudget > heap.size so we add a safeguard here + // See https://github.com/ggml-org/llama.cpp/issues/29277 + const vk::DeviceSize budget = std::min(budgetprops.heapBudget[i], heap.size); + *free += budget > budgetprops.heapUsage[i] ? budget - budgetprops.heapUsage[i] : 0; } else { *free += heap.size; } @@ -15341,13 +15912,14 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm } return true; }; - // reject any tensors larger than the max buffer size + // non-compute ops are never executed here; their size is bounded by the buffer allocation limit + const bool non_compute = ggml_op_is_empty(op->op); for (int i = 0; i < GGML_MAX_SRC; i++) { - if (op->src[i] && !tensor_size_supported(ggml_nbytes(op->src[i]))) { + if (op->src[i] && !non_compute && !tensor_size_supported(ggml_nbytes(op->src[i]))) { return false; } } - if (!tensor_size_supported(ggml_nbytes(op))) { + if (!non_compute && !tensor_size_supported(ggml_nbytes(op))) { return false; } @@ -15403,13 +15975,20 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm { ggml_type src0_type = op->src[0]->type; if (op->op == GGML_OP_MUL_MAT_ID) { + if (!ggml_vk_use_mul_mat_vec_id(op)) { + const ggml_tensor * ids = op->src[2]; + // Shaders store row IDs as uint16_t slot and token indices. + if (ids->ne[0] > int64_t(UINT16_MAX) + 1 || ids->ne[1] > int64_t(UINT16_MAX) + 1) { + return false; + } + } if (!device->mul_mat_id_s[src0_type] && !device->mul_mat_id_m[src0_type] && !device->mul_mat_id_l[src0_type]) { // If there's not enough shared memory for row_ids and the result tile, fallback to CPU return false; } - if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { - return false; - } + } + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; } switch (src0_type) { case GGML_TYPE_F32: @@ -15444,16 +16023,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm default: return false; } - struct ggml_tensor * a; - struct ggml_tensor * b; - if (op->op == GGML_OP_MUL_MAT) { - a = op->src[0]; - b = op->src[1]; - } else { - a = op->src[2]; - b = op->src[1]; - } - if (a->ne[3] != b->ne[3]) { + // MUL_MAT broadcasts src0 over dim 3 like dim 2, MUL_MAT_ID needs the ids to match src1 + if (op->op == GGML_OP_MUL_MAT_ID && op->src[2]->ne[3] != op->src[1]->ne[3]) { return false; } if (!(ggml_vk_dim01_contiguous(op->src[0]) || op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) || @@ -15465,8 +16036,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm // So don't support this combination for now. return false; } - if (op->src[1]->type == GGML_TYPE_BF16 && op->src[0]->type != GGML_TYPE_BF16) { - // BF16 in src1 is only served by the BF16 x BF16 pipelines + if (op->src[1]->type == GGML_TYPE_BF16 && op->src[0]->type != GGML_TYPE_BF16 && op->op != GGML_OP_MUL_MAT) { + // BF16 in src1 (widened to F32/F16) is only wired up for MUL_MAT return false; } @@ -15474,6 +16045,10 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm } case GGML_OP_FLASH_ATTN_EXT: { + // K/V row lists (ggml_flash_attn_ext_rows) are not supported + if (op->src[5] != nullptr) { + return false; + } bool coopmat2 = device->coopmat2; uint32_t HSK = op->src[1]->ne[0]; uint32_t HSV = op->src[2]->ne[0]; @@ -15740,10 +16315,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_CONCAT: { return ggml_vk_concat_supported(op->src[0], op->src[1], op); } - case GGML_OP_ADD1: - return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32) - || (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32) - || (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16); case GGML_OP_ARANGE: return op->type == GGML_TYPE_F32; case GGML_OP_FILL: @@ -15754,8 +16325,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_PAD_REFLECT_1D: case GGML_OP_ROLL: return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_DIAG_MASK_INF: - return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_SOFT_MAX: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32 && (!op->src[1] || (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16)); @@ -15763,6 +16332,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32; case GGML_OP_SUM: + return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguously_allocated(op->src[0]); case GGML_OP_SUM_ROWS: case GGML_OP_MEAN: return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); @@ -16075,7 +16645,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm } } -static ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { +ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")"); GGML_UNUSED(max_tensor_size); @@ -16157,11 +16727,42 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, return devices[device]; } +static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { + UNUSED(reg); +#if !defined(_WIN32) + if (strcmp(name, "ggml_backend_comm_init") == 0) { + return (void *) ggml_backend_vk_comm_init; + } + if (strcmp(name, "ggml_backend_comm_free") == 0) { + return (void *) ggml_backend_vk_comm_free; + } + if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { + return (void *) ggml_backend_vk_comm_allreduce_tensor; + } +#endif + if (strcmp(name, "ggml_backend_fusion_get") == 0) { + return (void *) ggml_backend_vk_fusion_get; + } + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *) ggml_backend_vk_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *) ggml_backend_vk_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *) ggml_backend_vk_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *) ggml_backend_vk_fusion_set_enabled; + } + return nullptr; +} + static const struct ggml_backend_reg_i ggml_backend_vk_reg_i = { /* .get_name = */ ggml_backend_vk_reg_get_name, /* .get_device_count = */ ggml_backend_vk_reg_get_device_count, /* .get_device = */ ggml_backend_vk_reg_get_device, - /* .get_proc_address = */ NULL, + /* .get_proc_address = */ ggml_backend_vk_reg_get_proc_address, }; ggml_backend_reg_t ggml_backend_vk_reg() { @@ -16303,7 +16904,10 @@ uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) { return 18; case 0xE20B: // B580 case 0xE211: // Pro B60 + case 0xE222: // Pro B65 return 20; + case 0xE223: // Pro B70 + return 32; case 0xB080: // PTL Xe3 LPG 2x6 (12 subslices) return 12; default: @@ -16370,6 +16974,11 @@ void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo> } vk_device_struct::~vk_device_struct() { + // the build failed before the device was created, nothing to destroy + if (device == VK_NULL_HANDLE) { + return; + } + VK_LOG_DEBUG("destroy device " << name); device.destroyFence(fence); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp b/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp index 495249d5f..bf6206704 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp @@ -27,14 +27,22 @@ void main() { const uint i1 = gl_WorkGroupID.x; const uint i2 = gl_WorkGroupID.y; - const uint i11 = data_c[i1 + i2 * p.s21]; + const int i11 = data_c[i1 + i2 * p.s21]; const uint s1 = p.ne0; const uint s2 = p.ne0 * p.ne1; const uint d0 = i1 * s1 + i2 * s2; const uint a0 = i1 * p.s01 + i2 * p.s02; - const uint b0 = i11 * p.s11; + + if (i11 == -1) { // skipped slot, add nothing + for (uint i0 = gl_LocalInvocationID.x; i0 < p.ne0; i0 += BLOCK_SIZE) { + data_d[d0 + i0] = data_a[a0 + i0]; + } + return; + } + + const uint b0 = uint(i11) * p.s11; for (uint i0 = gl_LocalInvocationID.x; i0 < p.ne0; i0 += BLOCK_SIZE) { data_d[d0 + i0] = data_a[a0 + i0] + data_b[b0 + i0]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp b/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp new file mode 100644 index 000000000..e598ad818 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp @@ -0,0 +1,39 @@ +#version 450 + +#include "types.glsl" +#include "generic_binary_head.glsl" + +#define TILE_DIM 32 +layout(local_size_x = TILE_DIM, local_size_y = 8, local_size_z = 1) in; + +// 33 columns (not 32) pads away shared-memory bank conflicts. +shared A_TYPE tile[TILE_DIM][TILE_DIM + 1]; + +void main() { + const uint tid_x = gl_LocalInvocationID.x; + const uint tid_y = gl_LocalInvocationID.y; + + // workgroup x walks rows (src1's contiguous dim1) so consecutive workgroups stream src1, y walks dst columns + const uint row0 = gl_WorkGroupID.x * TILE_DIM; + const uint col0 = (gl_WorkGroupID.z * gl_NumWorkGroups.y + gl_WorkGroupID.y) * TILE_DIM; + + const uint i1 = row0 + tid_x; + [[unroll]] for (uint j = 0; j < TILE_DIM; j += gl_WorkGroupSize.y) { + const uint i0 = col0 + tid_y + j; + if (i0 >= p.ne00 && i0 - p.ne00 < p.ne10 && i1 < p.ne11) { + tile[tid_y + j][tid_x] = A_TYPE(data_b[get_boffset() + (i0 - p.ne00)*p.nb10 + i1*p.nb11]); + } + } + + barrier(); + + const uint d0 = col0 + tid_x; + [[unroll]] for (uint j = 0; j < TILE_DIM; j += gl_WorkGroupSize.y) { + const uint d1 = row0 + tid_y + j; + if (d0 < p.ne20 && d1 < p.ne21) { + data_d[get_doffset() + d1*p.nb21 + d0*p.nb20] = d0 < p.ne00 + ? D_TYPE(data_a[get_aoffset() + d1*p.nb01 + d0*p.nb00]) + : D_TYPE(tile[tid_x][tid_y + j]); + } + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp index 06a50181c..c7c0ac341 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp @@ -5,6 +5,7 @@ #ifdef USE_SUBGROUPS #extension GL_KHR_shader_subgroup_basic : enable #extension GL_KHR_shader_subgroup_arithmetic : enable +#extension GL_KHR_shader_subgroup_vote : enable #endif #include "types.glsl" @@ -18,40 +19,93 @@ layout (push_constant) uniform parameter uint32_t nb01; uint32_t a_offset; uint32_t n_experts; - uint32_t hoist_row_ids; uint32_t ne00mp; uint32_t ne00L; + uint32_t row_tile_size; + uint32_t row_ids_offset; + uint32_t ne0_dst; + uint32_t stride_dst; + uint32_t batch_stride_dst; } p; #define BLOCK_SIZE 256 +#define EXPERTS_PER_CHUNK 1024 layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {uint data_a[];}; layout (binding = 1) writeonly buffer D {uint data_d[];}; +layout (binding = 2) writeonly buffer DST {float data_dst[];}; + +shared uint vals[EXPERTS_PER_CHUNK]; +shared uint offsets[EXPERTS_PER_CHUNK]; +shared uint cursors[EXPERTS_PER_CHUNK]; +shared uint skipped_rows; +shared uint skip_count; + +// data_d[0] is the tile count. Each tile has (expert, valid rows). +// Packed (token << 16) | slot row ids start at p.row_ids_offset, with each expert on a tile boundary. +void write_tiles(uint expert, uint count, uint tile_begin, uint n_tiles) { + for (uint t = 0; t < n_tiles; ++t) { + const uint descriptor = 1 + 2 * (tile_begin + t); + data_d[descriptor] = expert; + data_d[descriptor + 1] = min(p.row_tile_size, count - t * p.row_tile_size); + } +} + +// The matmul does not write skipped rows, so collect their offsets and zero them cooperatively. +void zero_skipped_rows() { + const uint num_elements = p.ne00 * p.ne01; + const uint tid = gl_LocalInvocationID.x; + + // Reuse offsets[] as a queue, so each batch must fit in EXPERTS_PER_CHUNK. + for (uint base = 0; base < num_elements; base += EXPERTS_PER_CHUNK) { + const uint end = min(base + EXPERTS_PER_CHUNK, num_elements); + if (tid == 0) { + skip_count = 0; + } + barrier(); + + for (uint idx = base + tid; idx < end; idx += BLOCK_SIZE) { + const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L); + const uint i00 = idx - i01 * p.ne00; + + if (int(data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]) == -1) { + const uint slot = atomicAdd(skip_count, 1); + offsets[slot] = i00 * p.stride_dst + i01 * p.batch_stride_dst; + } + } + barrier(); + + const uint n_skipped = skip_count; + for (uint r = 0; r < n_skipped; ++r) { + const uint d_off = offsets[r]; + for (uint i = tid; i < p.ne0_dst; i += BLOCK_SIZE) { + data_dst[d_off + i] = 0.0f; + } + } + barrier(); + } +} -// Upper bound on n_experts for the hoisted row-id path. Must match the limit in -// ggml_vk_mul_mat_id_q_f16 (hoist_row_ids). The non-hoisted reduction below only -// needs BLOCK_SIZE entries. -#define MAX_EXPERTS 1024 - -shared uint vals[MAX_EXPERTS]; -shared uint offsets[MAX_EXPERTS]; -shared uint cursors[MAX_EXPERTS]; - -// data_d layout when p.hoist_row_ids is set: -// [0, n_experts) per-expert row count -// [n_experts, 2*n_experts) per-expert start offset into the row id region -// [2*n_experts] total row count -// [2*n_experts + 1, ) row ids grouped by expert, packed as (i01 << 16) | (i00 & 0xffff) -// Otherwise only data_d[expert_id] is written, holding that expert's row count. void main() { - const uint expert_id = gl_WorkGroupID.x; const uint num_elements = p.ne00 * p.ne01; const uint tid = gl_LocalInvocationID.x; + const uint tile_shift = findLSB(p.row_tile_size); + uint total_tiles = 0; + uint local_skipped = 0; + if (tid == 0) { + skipped_rows = 0; + } + barrier(); +#ifdef USE_SUBGROUPS + // Use the subgroup that contains invocation 0. + const bool prefix_subgroup = subgroupAny(tid == 0); +#endif - if (p.hoist_row_ids != 0) { - for (uint e = tid; e < p.n_experts; e += BLOCK_SIZE) { + for (uint expert_base = 0; expert_base < p.n_experts;) { + const uint n_experts = min(EXPERTS_PER_CHUNK, p.n_experts - expert_base); + for (uint e = tid; e < n_experts; e += BLOCK_SIZE) { vals[e] = 0; } barrier(); @@ -59,46 +113,43 @@ void main() { for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) { const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L); const uint i00 = idx - i01 * p.ne00; - const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]; - if (expert < p.n_experts) { + const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00] - expert_base; + if (expert < n_experts) { atomicAdd(vals[expert], 1); } } barrier(); #ifdef USE_SUBGROUPS - if (gl_SubgroupID == 0) { - // pad the trip count so the subgroup ops stay in uniform control flow - const uint n_experts_padded = (p.n_experts + gl_SubgroupSize - 1) & ~(gl_SubgroupSize - 1); - uint base = 0; - for (uint expert = gl_SubgroupInvocationID; expert < n_experts_padded; expert += gl_SubgroupSize) { - const bool in_range = expert < p.n_experts; - const uint count = in_range ? vals[expert] : 0; - const uint offset = base + subgroupExclusiveAdd(count); + if (prefix_subgroup) { + const uint n_experts_padded = (n_experts + gl_SubgroupSize - 1) & ~(gl_SubgroupSize - 1); + uint base = subgroupAdd(tid == 0 ? total_tiles : 0u); + for (uint e = gl_SubgroupInvocationID; e < n_experts_padded; e += gl_SubgroupSize) { + const bool in_range = e < n_experts; + const uint count = in_range ? vals[e] : 0; + const uint n_tiles = (count + p.row_tile_size - 1) >> tile_shift; + const uint tile_begin = base + subgroupExclusiveAdd(n_tiles); if (in_range) { - data_d[expert] = count; - data_d[p.n_experts + expert] = offset; - offsets[expert] = offset; - cursors[expert] = 0; + offsets[e] = tile_begin * p.row_tile_size; + cursors[e] = 0; + write_tiles(expert_base + e, count, tile_begin, n_tiles); } - base += subgroupAdd(count); + base += subgroupAdd(n_tiles); } - if (subgroupElect()) { - data_d[2 * p.n_experts] = base; + if (tid == 0) { + total_tiles = base; } } #else if (tid == 0) { - uint offset = 0; - for (uint expert = 0; expert < p.n_experts; ++expert) { - const uint count = vals[expert]; - data_d[expert] = count; - data_d[p.n_experts + expert] = offset; - offsets[expert] = offset; - cursors[expert] = 0; - offset += count; + for (uint e = 0; e < n_experts; ++e) { + const uint count = vals[e]; + const uint n_tiles = (count + p.row_tile_size - 1) >> tile_shift; + offsets[e] = total_tiles * p.row_tile_size; + cursors[e] = 0; + write_tiles(expert_base + e, count, total_tiles, n_tiles); + total_tiles += n_tiles; } - data_d[2 * p.n_experts] = offset; } #endif barrier(); @@ -106,35 +157,31 @@ void main() { for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) { const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L); const uint i00 = idx - i01 * p.ne00; - const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]; - if (expert < p.n_experts) { + const uint id = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]; + // Count skipped slots only in the first expert chunk. + if (expert_base == 0 && int(id) == -1) { + local_skipped++; + } + const uint expert = id - expert_base; + if (expert < n_experts) { const uint row = atomicAdd(cursors[expert], 1); - const uint packed_row_id = (i01 << 16) | (i00 & 0xffffu); - data_d[2 * p.n_experts + 1 + offsets[expert] + row] = packed_row_id; + data_d[p.row_ids_offset + offsets[expert] + row] = (i01 << 16) | i00; } } - return; + // The next chunk clears vals. Scatter uses only offsets and cursors. + expert_base += n_experts; } - - uint count = 0; - for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) { - const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L); - const uint i00 = idx - i01 * p.ne00; - const uint a = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]; - - count += uint(a == expert_id); + if (local_skipped != 0) { + atomicAdd(skipped_rows, local_skipped); } - - vals[tid] = count; + // All scatter reads must finish before reusing offsets[] as a queue. barrier(); - [[unroll]] for (uint s = BLOCK_SIZE / 2; s > 0; s >>= 1) { - if (tid < s) { - vals[tid] += vals[tid + s]; - } - barrier(); - } if (tid == 0) { - data_d[expert_id] = vals[0]; + data_d[0] = total_tiles; + } + + if (skipped_rows != 0) { + zero_skipped_rows(); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp b/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp index 75e3c3b0e..08c785ad3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp @@ -25,6 +25,8 @@ void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; const uint tid = gl_LocalInvocationID.x; + if (row >= p.ne01 * p.ne02 * p.ne03) { return; } + const uint i03 = fastdiv(row, p.ne0_12mp, p.ne0_12L); const uint i03_offset = i03 * p.ne01*p.ne02; const uint i02 = fastdiv(row - i03_offset, p.ne0_1mp, p.ne0_1L); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index cc6e242a9..ecea26672 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -10,7 +10,7 @@ #include "types.glsl" layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufF32 { - vec4 block; + readonly vec4 block; }; float16_t dequantFuncF32(const in decodeBufF32 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -22,7 +22,7 @@ float16_t dequantFuncF32(const in decodeBufF32 bl, const in uint blockCoords[2], } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ1_0 { - block_q1_0 block; + readonly block_q1_0 block; }; float16_t dequantFuncQ1_0(const in decodeBufQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -47,7 +47,7 @@ f16vec4 dequantFuncQ1_0_v(const in decodeBufQ1_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ2_0 { - block_q2_0 block; + readonly block_q2_0 block; }; float16_t dequantFuncQ2_0(const in decodeBufQ2_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -67,7 +67,7 @@ f16vec4 dequantFuncQ2_0_v(const in decodeBufQ2_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ4_0 { - block_q4_0_packed16 block; + readonly block_q4_0_packed16 block; }; float16_t dequantFuncQ4_0(const in decodeBufQ4_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -98,11 +98,11 @@ f16vec4 dequantFuncQ4_0_v(const in decodeBufQ4_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ4_1 { - block_q4_1 block; + readonly block_q4_1 block; }; layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ4_1_packed32 { - block_q4_1_packed32 block; + readonly block_q4_1_packed32 block; }; float16_t dequantFuncQ4_1(const in decodeBufQ4_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -133,11 +133,11 @@ f16vec4 dequantFuncQ4_1_v(const in decodeBufQ4_1 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ5_0 { - block_q5_0 block; + readonly block_q5_0 block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ5_0_packed16 { - block_q5_0_packed16 block; + readonly block_q5_0_packed16 block; }; float16_t dequantFuncQ5_0(const in decodeBufQ5_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -177,11 +177,11 @@ f16vec4 dequantFuncQ5_0_v(const in decodeBufQ5_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufQ5_1 { - block_q5_1 block; + readonly block_q5_1 block; }; layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufQ5_1_packed32 { - block_q5_1_packed32 block; + readonly block_q5_1_packed32 block; }; float16_t dequantFuncQ5_1(const in decodeBufQ5_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -221,7 +221,7 @@ f16vec4 dequantFuncQ5_1_v(const in decodeBufQ5_1 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ8_0 { - block_q8_0_packed16 block; + readonly block_q8_0_packed16 block; }; float16_t dequantFuncQ8_0(const in decodeBufQ8_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -248,7 +248,7 @@ f16vec4 dequantFuncQ8_0_v(const in decodeBufQ8_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ1_0 { - block_tq1_0 block; + readonly block_tq1_0 block; }; float16_t dequantFuncTQ1_0(const in decodeBufTQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -275,11 +275,11 @@ f16vec4 dequantFuncTQ1_0_v(const in decodeBufTQ1_0 bl, const in uint blockCoords } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0 { - block_tq2_0 block; + readonly block_tq2_0 block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0_packed16 { - block_tq2_0_packed16 block; + readonly block_tq2_0_packed16 block; }; float16_t dequantFuncTQ2_0(const in decodeBufTQ2_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -313,15 +313,15 @@ f16vec4 dequantFuncTQ2_0_v(const in decodeBufTQ2_0 bl, const in uint blockCoords } layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ2_K { - block_q2_K block; + readonly block_q2_K block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ2_K_packed16 { - block_q2_K_packed16 block; + readonly block_q2_K_packed16 block; }; layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ2_K_packed32 { - block_q2_K_packed32 block; + readonly block_q2_K_packed32 block; }; float16_t dequantFuncQ2_K(const in decodeBufQ2_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -365,11 +365,11 @@ f16vec4 dequantFuncQ2_K_v(const in decodeBufQ2_K bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ3_K { - block_q3_K block; + readonly block_q3_K block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ3_K_packed16 { - block_q3_K_packed16 block; + readonly block_q3_K_packed16 block; }; float16_t dequantFuncQ3_K(const in decodeBufQ3_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -442,19 +442,19 @@ f16vec4 dequantFuncQ3_K_v(const in decodeBufQ3_K bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K { - block_q4_K block; + readonly block_q4_K block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K_packed16 { - block_q4_K_packed16 block; + readonly block_q4_K_packed16 block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K_packed32 { - block_q4_K_packed32 block; + readonly block_q4_K_packed32 block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K_packed128 { - block_q4_K_packed128 block; + readonly block_q4_K_packed128 block; }; #if defined(IS_MUL_MM2) @@ -650,19 +650,19 @@ f16vec4 dequantFuncQ4_K_v(const in decodeBufQ4_K bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K { - block_q5_K block; + readonly block_q5_K block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K_packed16 { - block_q5_K_packed16 block; + readonly block_q5_K_packed16 block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K_packed128 { - block_q5_K_packed128 block; + readonly block_q5_K_packed128 block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K_packed32 { - block_q5_K_packed32 block; + readonly block_q5_K_packed32 block; }; float16_t dequantFuncQ5_K(const in decodeBufQ5_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -771,11 +771,11 @@ f16vec4 dequantFuncQ5_K_v(const in decodeBufQ5_K bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ6_K { - block_q6_K block; + readonly block_q6_K block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ6_K_packed16 { - block_q6_K_packed16 block; + readonly block_q6_K_packed16 block; }; float16_t dequantFuncQ6_K(const in decodeBufQ6_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -833,7 +833,7 @@ f16vec4 dequantFuncQ6_K_v(const in decodeBufQ6_K bl, const in uint blockCoords[2 #if defined(DATA_A_IQ1_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_S { - block_iq1_s block; + readonly block_iq1_s block; }; float16_t dequantFuncIQ1_S(const in decodeBufIQ1_S bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -880,11 +880,11 @@ f16vec4 dequantFuncIQ1_S_v(const in decodeBufIQ1_S bl, const in uint blockCoords #if defined(DATA_A_IQ1_M) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_M { - block_iq1_m block; + readonly block_iq1_m block; }; layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufIQ1_M_packed64 { - block_iq1_m_packed64 block; + readonly block_iq1_m_packed64 block; }; float16_t dequantFuncIQ1_M(const in decodeBufIQ1_M bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -939,11 +939,11 @@ f16vec4 dequantFuncIQ1_M_v(const in decodeBufIQ1_M bl, const in uint blockCoords #if defined(DATA_A_IQ2_XXS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XXS { - block_iq2_xxs block; + readonly block_iq2_xxs block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XXS_packed16 { - block_iq2_xxs_packed16 block; + readonly block_iq2_xxs_packed16 block; }; float16_t dequantFuncIQ2_XXS(const in decodeBufIQ2_XXS bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1001,7 +1001,7 @@ f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCo #if defined(DATA_A_IQ2_XS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XS { - block_iq2_xs block; + readonly block_iq2_xs block; }; float16_t dequantFuncIQ2_XS(const in decodeBufIQ2_XS bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1054,7 +1054,7 @@ f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoor #if defined(DATA_A_IQ2_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_S { - block_iq2_s block; + readonly block_iq2_s block; }; float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1108,11 +1108,11 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords #if defined(DATA_A_IQ3_XXS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_XXS { - block_iq3_xxs block; + readonly block_iq3_xxs block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_XXS_packed16 { - block_iq3_xxs_packed16 block; + readonly block_iq3_xxs_packed16 block; }; float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1167,7 +1167,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo #if defined(DATA_A_IQ3_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_S { - block_iq3_s block; + readonly block_iq3_s block; }; float16_t dequantFuncIQ3_S(const in decodeBufIQ3_S bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1217,11 +1217,11 @@ f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords #if defined(DATA_A_IQ4_XS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_XS { - block_iq4_xs block; + readonly block_iq4_xs block; }; layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufIQ4_XS_packed32 { - block_iq4_xs_packed32 block; + readonly block_iq4_xs_packed32 block; }; float16_t dequantFuncIQ4_XS(const in decodeBufIQ4_XS bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1267,11 +1267,11 @@ f16vec4 dequantFuncIQ4_XS_v(const in decodeBufIQ4_XS bl, const in uint blockCoor #if defined(DATA_A_IQ4_NL) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL { - block_iq4_nl block; + readonly block_iq4_nl block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL_packed16 { - block_iq4_nl_packed16 block; + readonly block_iq4_nl_packed16 block; }; float16_t dequantFuncIQ4_NL(const in decodeBufIQ4_NL bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1308,7 +1308,7 @@ f16vec4 dequantFuncIQ4_NL_v(const in decodeBufIQ4_NL bl, const in uint blockCoor #if defined(DATA_A_MXFP4) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufMXFP4 { - block_mxfp4 block; + readonly block_mxfp4 block; }; float16_t dequantFuncMXFP4(const in decodeBufMXFP4 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1362,11 +1362,11 @@ f16vec4 dequantFuncMXFP4_v(const in decodeBufMXFP4 bl, const in uint blockCoords #if defined(DATA_A_NVFP4) layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufNVFP4 { - block_nvfp4 block; + readonly block_nvfp4 block; }; layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufNVFP4_packed32 { - block_nvfp4_packed32 block; + readonly block_nvfp4_packed32 block; }; float16_t dequantFuncNVFP4(const in decodeBufNVFP4 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp b/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp deleted file mode 100644 index 9cef8a8ec..000000000 --- a/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +++ /dev/null @@ -1,34 +0,0 @@ -#version 450 - -#extension GL_EXT_shader_16bit_storage : require -#extension GL_EXT_control_flow_attributes : enable - -layout (push_constant) uniform parameter -{ - uint ncols; - uint rows_per_channel; - uint n_past; -} p; - -#include "types.glsl" - -layout(local_size_x = 1, local_size_y = 512, local_size_z = 1) in; - -layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; -layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; - -void main() { - const uint col = gl_GlobalInvocationID.y; - const uint row = gl_GlobalInvocationID.x; - - if (col >= p.ncols) { - return; - } - - const uint i = row*p.ncols + col; - if (col > p.n_past + row % p.rows_per_channel) { - data_d[i] = D_TYPE(uintBitsToFloat(0xFF800000)); - } else { - data_d[i] = D_TYPE(data_a[i]); - } -} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl index c474bfe09..d7a9804ba 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl @@ -13,6 +13,14 @@ ACC_TYPE dot_product(f16vec2 a, f16vec2 b, ACC_TYPE acc) { return ACC_TYPE(v_dot2_f32_f16(a, b, float(acc))); } +ACC_TYPEV2 dot_product_v2(f16vec4 a0, f16vec4 a1, f16vec4 b, ACC_TYPEV2 acc) { + return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y)); +} + +ACC_TYPEV2 dot_product_v2(f16vec2 a0, f16vec2 a1, f16vec2 b, ACC_TYPEV2 acc) { + return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y)); +} + #else ACC_TYPE dot_product(FLOAT_TYPEV4 a, FLOAT_TYPEV4 b, ACC_TYPE acc) { @@ -24,4 +32,27 @@ ACC_TYPE dot_product(FLOAT_TYPEV2 a, FLOAT_TYPEV2 b, ACC_TYPE acc) { return fma(ACC_TYPE(a.x), ACC_TYPE(b.x), fma(ACC_TYPE(a.y), ACC_TYPE(b.y), acc)); } +#if defined(ACC_F16) +// two rows at once, so f16 accumulation can use packed fma +ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV4 a0, FLOAT_TYPEV4 a1, FLOAT_TYPEV4 b, ACC_TYPEV2 acc) { + acc = fma(ACC_TYPEV2(a0.w, a1.w), ACC_TYPEV2(b.w), acc); + acc = fma(ACC_TYPEV2(a0.z, a1.z), ACC_TYPEV2(b.z), acc); + acc = fma(ACC_TYPEV2(a0.y, a1.y), ACC_TYPEV2(b.y), acc); + return fma(ACC_TYPEV2(a0.x, a1.x), ACC_TYPEV2(b.x), acc); +} + +ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV2 a0, FLOAT_TYPEV2 a1, FLOAT_TYPEV2 b, ACC_TYPEV2 acc) { + acc = fma(ACC_TYPEV2(a0.y, a1.y), ACC_TYPEV2(b.y), acc); + return fma(ACC_TYPEV2(a0.x, a1.x), ACC_TYPEV2(b.x), acc); +} +#else +ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV4 a0, FLOAT_TYPEV4 a1, FLOAT_TYPEV4 b, ACC_TYPEV2 acc) { + return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y)); +} + +ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV2 a0, FLOAT_TYPEV2 a1, FLOAT_TYPEV2 b, ACC_TYPEV2 acc) { + return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y)); +} +#endif + #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp index c6ed63dd4..c7253784c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp @@ -18,7 +18,8 @@ #ifdef GL_NV_cooperative_matrix_decode_vector #extension GL_NV_cooperative_matrix_decode_vector : enable #endif -#extension GL_EXT_buffer_reference : enable +#extension GL_EXT_buffer_reference2 : enable +#extension GL_EXT_shader_explicit_arithmetic_types_int64 : enable #extension GL_KHR_shader_subgroup_ballot : enable #extension GL_KHR_shader_subgroup_vote : enable #extension GL_EXT_null_initializer : enable @@ -35,6 +36,10 @@ #define FA_GATHER_BS 1u #endif +layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufFA_Byte { + uint8_t raw; +}; + // buffer_reference stride = sizeof(struct) = FaBlockBytesK/V. layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufFA_K { uint8_t raw[FaBlockBytesK]; @@ -113,48 +118,71 @@ layout (binding = 1) readonly buffer K {uint8_t data_k[];}; layout (binding = 2) readonly buffer V {uint8_t data_v[];}; layout (binding = 3) readonly buffer M {uint8_t data_m[];}; -// f16 aliases for the sparse gather callbacks. -layout (binding = 1) readonly buffer KF16 {float16_t data_kf16[];}; -layout (binding = 2) readonly buffer VF16 {float16_t data_vf16[];}; +// Native 16-bit aliases for the sparse gather callbacks. +layout (binding = 1) readonly buffer K16 {FLOAT_TYPE data_k16[];}; +layout (binding = 2) readonly buffer V16 {FLOAT_TYPE data_v16[];}; layout (binding = 3) readonly buffer MF16 {float16_t data_mf16[];}; #ifdef GL_NV_cooperative_matrix_decode_vector -layout (binding = 1) readonly buffer KF16V4 {f16vec4 data_kf16v4[];}; -layout (binding = 2) readonly buffer VF16V4 {f16vec4 data_vf16v4[];}; +layout (binding = 1) readonly buffer K16V4 {FLOAT_TYPEV4 data_k16v4[];}; +layout (binding = 2) readonly buffer V16V4 {FLOAT_TYPEV4 data_v16v4[];}; #endif -// K/V/mask f16-element offsets for the current head/batch, set in main(). +// K/V/mask offsets in 16-bit elements for the current head/batch, set in main(). uint32_t g_k_off_elem, g_v_off_elem, g_m_off_elem; -#if !defined(BFLOAT16) -// blockCoords are in block units: KV slot = blockCoords[0], -// head dim = blockCoords[1]*FA_GATHER_BS + coordInBlock[1]. -float16_t faGatherK(const decodeBufFA_K unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { - if (blockCoords[0] >= p.split_kv) { return float16_t(0); } +FLOAT_TYPE faGatherK(const decodeBufFA_K bl_in, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return FLOAT_TYPE(0.0); } const int r = data_sparse[sparse_base + blockCoords[0]]; - return r < 0 ? float16_t(0) : data_kf16[g_k_off_elem + uint(r) * k_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]]; + if (r < 0) { return FLOAT_TYPE(0.0); } +#if !defined(BFLOAT16) + if (USE_DECODE_K) { + decodeBufFA_K block = decodeBufFA_K(decodeBufFA_Byte(bl_in) + uint64_t(uint(r) - blockCoords[0]) * k_stride * FaBlockBytesK); + return faDecodeK(block, blockCoords, coordInBlock); + } +#endif + return data_k16[g_k_off_elem + uint(r) * k_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]]; } -float16_t faGatherV(const decodeBufFA_V unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { - if (blockCoords[0] >= p.split_kv) { return float16_t(0); } +FLOAT_TYPE faGatherV(const decodeBufFA_V bl_in, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return FLOAT_TYPE(0.0); } const int r = data_sparse[sparse_base + blockCoords[0]]; - return r < 0 ? float16_t(0) : data_vf16[g_v_off_elem + uint(r) * v_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]]; + if (r < 0) { return FLOAT_TYPE(0.0); } +#if !defined(BFLOAT16) + if (USE_DECODE_V) { + decodeBufFA_V block = decodeBufFA_V(decodeBufFA_Byte(bl_in) + uint64_t(uint(r) - blockCoords[0]) * v_stride * FaBlockBytesV); + return faDecodeV(block, blockCoords, coordInBlock); + } +#endif + return data_v16[g_v_off_elem + uint(r) * v_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]]; } #ifdef GL_NV_cooperative_matrix_decode_vector -f16vec4 faGatherKVector(const decodeBufFA_K unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { - if (blockCoords[0] >= p.split_kv) { return f16vec4(0); } +FLOAT_TYPEV4 faGatherKVector(const decodeBufFA_K bl_in, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return FLOAT_TYPEV4(0.0); } const int r = data_sparse[sparse_base + blockCoords[0]]; - if (r < 0) { return f16vec4(0); } + if (r < 0) { return FLOAT_TYPEV4(0.0); } +#if !defined(BFLOAT16) + if (USE_DECODE_K) { + decodeBufFA_K block = decodeBufFA_K(decodeBufFA_Byte(bl_in) + uint64_t(uint(r) - blockCoords[0]) * k_stride * FaBlockBytesK); + return faDecodeKVector(block, blockCoords, coordInBlock); + } +#endif const uint32_t o = g_k_off_elem + uint(r) * k_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]; - return data_kf16v4[o / 4]; + return data_k16v4[o / 4]; } -f16vec4 faGatherVVector(const decodeBufFA_V unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { - if (blockCoords[0] >= p.split_kv) { return f16vec4(0); } +FLOAT_TYPEV4 faGatherVVector(const decodeBufFA_V bl_in, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return FLOAT_TYPEV4(0.0); } const int r = data_sparse[sparse_base + blockCoords[0]]; - if (r < 0) { return f16vec4(0); } + if (r < 0) { return FLOAT_TYPEV4(0.0); } +#if !defined(BFLOAT16) + if (USE_DECODE_V) { + decodeBufFA_V block = decodeBufFA_V(decodeBufFA_Byte(bl_in) + uint64_t(uint(r) - blockCoords[0]) * v_stride * FaBlockBytesV); + return faDecodeVVector(block, blockCoords, coordInBlock); + } +#endif const uint32_t o = g_v_off_elem + uint(r) * v_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]; - return data_vf16v4[o / 4]; + return data_v16v4[o / 4]; } #define FAGATHERK , faGatherK, faGatherKVector @@ -163,7 +191,6 @@ f16vec4 faGatherVVector(const decodeBufFA_V unused, const uint32_t blockCoords[2 #define FAGATHERK , faGatherK #define FAGATHERV , faGatherV #endif -#endif // Add gathered mask to S (slope==1 since sparse requires max_bias==0). col = slot in block jblk. ACC_TYPE faAddSparseMask(const uint32_t row, const uint32_t col, const ACC_TYPE elem, const uint32_t jblk) { @@ -252,8 +279,8 @@ void main() { tensorViewNV<2, false, 1, 0> tensorViewTranspose = createTensorViewNV(2, false, 1, 0); - const uint bs_k = USE_SPARSE ? FA_GATHER_BS : fa_block_elems(FaTypeK); - const uint bs_v = USE_SPARSE ? FA_GATHER_BS : fa_block_elems(FaTypeV); + const uint bs_k = USE_SPARSE ? max(FA_GATHER_BS, BLOCK_SIZE_K) : BLOCK_SIZE_K; + const uint bs_v = USE_SPARSE ? max(FA_GATHER_BS, BLOCK_SIZE_V) : BLOCK_SIZE_V; tensorLayoutK = setTensorLayoutBlockSizeNV(tensorLayoutK, 1, bs_k); tensorLayoutV = setTensorLayoutBlockSizeNV(tensorLayoutV, 1, bs_v); @@ -384,18 +411,15 @@ void main() { uint32_t k_offset = ik2*p.nb12 + ik3*p.nb13; // F16: bs_k==1 (direct load). F32: bs_k==4 (vec4 / dequantFuncF32). Quantized types: bs_k==32. -#if defined(BFLOAT16) - coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose); -#else - const bool k_use_decode = (bs_k > 1u); if (USE_SPARSE) { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose FAGATHERK); - } else if (k_use_decode) { +#if !defined(BFLOAT16) + } else if (USE_DECODE_K) { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose FADECODEK); +#endif } else { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose); } -#endif S = coopMatMulAdd(Qf16, K_T, S); if (LOGIT_SOFTCAP) { @@ -458,18 +482,15 @@ void main() { coopmat<FLOAT_TYPE, gl_ScopeWorkgroup, Bc, HSV_pad, gl_MatrixUseB> V; uint32_t v_offset = iv2*p.nb22 + iv3*p.nb23; -#if defined(BFLOAT16) - coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad)); -#else - const bool v_use_decode = (bs_v > 1u); if (USE_SPARSE) { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad) FAGATHERV); - } else if (v_use_decode) { +#if !defined(BFLOAT16) + } else if (USE_DECODE_V) { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad) FADECODEV); +#endif } else { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad)); } -#endif L = eM*L + rowsum; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp index b5f95aaa0..53beab149 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp @@ -3,261 +3,327 @@ #extension GL_EXT_control_flow_attributes : enable #extension GL_EXT_shader_16bit_storage : require #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require -#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #extension GL_KHR_memory_scope_semantics : enable #extension GL_KHR_shader_subgroup_basic : enable -#extension GL_KHR_shader_subgroup_ballot : enable #extension GL_KHR_shader_subgroup_arithmetic : enable #extension GL_KHR_cooperative_matrix : enable #extension GL_EXT_shared_memory_block : enable -layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +layout (local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -layout (binding = 0) readonly buffer Q {float16_t qState[];}; -layout (binding = 1) readonly buffer K_VEC4 {f16vec4 kStateVec4[];}; -layout (binding = 2) buffer MASK_F16 {float16_t mState_f16[];}; -layout (binding = 3) buffer P_FP16 {float16_t matP_f16[];}; -layout (binding = 4) buffer OUT_MAX {float out_max_f32[];}; +layout (binding = 0) readonly buffer Q {float16_t data_q[];}; +layout (binding = 1) readonly buffer KV4 {f16vec4 data_kv4[];}; +layout (binding = 2) readonly buffer M {float16_t data_m[];}; +layout (binding = 3) buffer P {float16_t data_p[];}; +layout (binding = 4) buffer MAX {float data_max[];}; layout (push_constant) uniform parameter { - uint kvSeqLen; - uint activationLength; - uint qHead; - uint kvHead; - uint qkRatio; - uint qkSubGroups; - uint flag; - uint kvStride1; - uint kvStride2; - uint batchStrideQ; - uint batchStrideK; - uint batchStrideV; - uint batchStrideM; - uint batchStrideO; - float softMaxScale; + uint neq1; + uint neq2; + uint neq3; + uint nek1; + uint nek2; + uint nek3; + uint nev2; + uint nev3; + uint nem1; + uint nem2; + uint nem3; + uint nbq1; + uint nbq2; + uint nbk1; + uint nbk2; + uint nbv1; + uint nbv2; + uint nbq3; + uint nbk3; + uint nbv3; + uint nb3; + uint flag; + float scale; } p; -layout (constant_id = 0) const uint GROUPSIZE = 128; -layout (constant_id = 1) const uint GQA_RATIO = 8; -layout (constant_id = 2) const uint HEAD_DIM = 128; -layout (constant_id = 3) const uint WARPSIZE = 16; -layout (constant_id = 4) const uint MATP_REDUCE = 32; -layout (constant_id = 5) const uint N_TOK = 1; -layout (constant_id = 6) const uint COOP_MAT_P_PER_LOOP = 4; - -#define MAX_HEADS 8 - -#define TN WARPSIZE -#define TM 8 -#define TK 16 -#define SUBGROUP_COUNT (GROUPSIZE / WARPSIZE) -#define MATP_PER_LOOP (COOP_MAT_P_PER_LOOP * TM) -#define P_LOOP_COUNT (MATP_REDUCE / MATP_PER_LOOP) - -#define COOP_MAT_Q_PER_TOKEN ((GQA_RATIO + TN - 1) / TN) -#define COOP_MAT_P_M COOP_MAT_Q_PER_TOKEN -#define COOP_MAT_P_N (MATP_REDUCE / TM) -#define SLM_PV_SIZE (MATP_REDUCE * COOP_MAT_P_M * TN) -#define SLM_MASK_SIZE (N_TOK * MATP_REDUCE) -#define SLM_POOL_SIZE_K (MATP_PER_LOOP * HEAD_DIM) -#define K_LOAD_PER_LOOP (GROUPSIZE * 4) -#define HEAD_DIM_VEC4 (HEAD_DIM / 4) -#define SLM_CHUNK_SIZE (TK / 4) -#define K_LOAD_LOOPS ((SLM_POOL_SIZE_K + K_LOAD_PER_LOOP - 1) / K_LOAD_PER_LOOP) -#define O_COUNT ((GQA_RATIO + SUBGROUP_COUNT - 1) / SUBGROUP_COUNT) - -shared slm_pool_block { - float slm_pool_pv[SLM_PV_SIZE + SLM_MASK_SIZE]; -} slm_pool_f32; - -shared slm_pool_alias_block { - float16_t slm_pool_k[SLM_POOL_SIZE_K]; -} slm_pool_f16; +layout (constant_id = 0) const uint WorkGroupSize = 128; +layout (constant_id = 1) const uint gqa_ratio = 8; +layout (constant_id = 2) const uint HSK = 128; +layout (constant_id = 3) const uint SubGroupSize = 16; +layout (constant_id = 4) const uint p_reduce = 32; +layout (constant_id = 5) const uint Br = 1; +layout (constant_id = 6) const uint pmat_count = 4; +layout (constant_id = 7) const uint TM = 8; +layout (constant_id = 8) const uint TN = 16; +layout (constant_id = 9) const uint shmem_size = 49152; + +const uint TK = 16; +const uint sg_count = WorkGroupSize / SubGroupSize; +const uint p_per_loop = pmat_count * TM; +const uint p_loops = p_reduce / p_per_loop; + +const uint pmat_row = (gqa_ratio + TN - 1) / TN; +const uint pmat_col = p_reduce / TM; + +const uint k_per_loop = WorkGroupSize * 4; +const uint hsk_v4 = HSK / 4; +const uint slm_col_chunk = TK / 4; +const uint k_loops = (p_per_loop * HSK + k_per_loop - 1) / k_per_loop; +const uint warp_o_row = (gqa_ratio + sg_count - 1) / sg_count; + +#define SLM_PV_SIZE (p_reduce * pmat_row * TN) +#define SLM_MASK_SIZE (Br * p_reduce) + +const bool k_in_shmem = k_loops * k_per_loop <= shmem_size / 2; +const uint ksh_size = k_in_shmem ? (k_loops * k_per_loop) : (sg_count * TM * TK); + +shared sh_block { + float F32sh[SLM_PV_SIZE + SLM_MASK_SIZE]; +} F32Poolsh; + +shared shf16_alias { + float16_t F16sh[ksh_size]; +} F16Poolsh; + +shared shf16v4_alias { + f16vec4 F16V4sh[ksh_size / 4]; +} F16V4Poolsh; void main() { - const uint lane = gl_SubgroupInvocationID; - const uint kHeadIdx = gl_WorkGroupID.x % p.kvHead; - const uint outGroupIdx = gl_WorkGroupID.x / p.kvHead; - const uint v = gl_WorkGroupID.y; - const uint d = gl_WorkGroupID.z; - const uint localLinearId = gl_SubgroupID; - const uint wgLane = localLinearId * WARPSIZE + lane; - const uint qDim = p.qHead * HEAD_DIM; - const uint kvDim = p.kvStride1; - const uint maskDim = p.kvSeqLen; - const uint maxDim = (p.kvSeqLen + MATP_REDUCE - 1) / MATP_REDUCE; - const uint pDim = maxDim * MATP_REDUCE; - const uint tokFlatIdx = localLinearId + outGroupIdx * N_TOK; - uint offsetBaseQ = min(tokFlatIdx, p.activationLength - 1) * qDim; - offsetBaseQ = offsetBaseQ + d * p.batchStrideQ + kHeadIdx * HEAD_DIM * GQA_RATIO; - const uint offsetBaseK = (d * p.batchStrideK + (v * MATP_REDUCE) * kvDim + kHeadIdx * p.kvStride2) / 4; - uint offsetOut = d * p.qHead * p.activationLength * pDim + v * MATP_REDUCE + kHeadIdx * GQA_RATIO * pDim + (localLinearId * O_COUNT + outGroupIdx * N_TOK * p.qHead) * pDim + lane; - uint offsetMax = d * p.qHead * p.activationLength * maxDim + v + kHeadIdx * GQA_RATIO * maxDim + (localLinearId * O_COUNT + outGroupIdx * N_TOK * p.qHead) * maxDim; - const uint offsetSlmLoadPv = (localLinearId * O_COUNT * MATP_REDUCE + lane); - const uint offsetBaseM = v * MATP_REDUCE + lane; - const float fp32Min = uintBitsToFloat(0xFEFFFFFF); - - const uint loopCount = HEAD_DIM / TK; - float maskFp32[MATP_REDUCE / WARPSIZE]; - - if (tokFlatIdx < p.activationLength) { - [[unroll]] for (uint mk = 0; mk < MATP_REDUCE / WARPSIZE; mk++) { - const uint maskOffset = mk * WARPSIZE + offsetBaseM; - if (maskOffset < maskDim) { - maskFp32[mk] = float(mState_f16[d * p.batchStrideM + tokFlatIdx * maskDim + maskOffset]); - } else { - maskFp32[mk] = fp32Min; - } + const uint lane = gl_SubgroupInvocationID; + const uint head_k_idx = gl_WorkGroupID.x % p.nek2; + const uint out_idx = gl_WorkGroupID.x / p.nek2; + const uint p_idx = gl_WorkGroupID.y; + const uint batch_k_ratio = p.neq3 / p.nek3; + const uint batch_idx = gl_WorkGroupID.z; + const uint batch_k_idx = batch_idx / batch_k_ratio; + const uint batch_mask_idx = batch_idx % p.nem3; + const uint batch_mask_stride = p.nek1 * p.nem1 * p.nem2; + const uint sg = gl_SubgroupID; + const uint wg_lane = sg * SubGroupSize + lane; + const uint k_dim = p.nbk1; + const uint max_dim = (p.nek1 + p_reduce - 1) / p_reduce; + const uint p_dim = max_dim * p_reduce; + const uint out_row_idx = sg + out_idx * Br; + const bool single_head_mask = p.nem2 == 1; + uint q_offset = min(out_row_idx, p.neq1 - 1) * p.nbq1; + q_offset = q_offset + batch_idx * p.nbq3 + head_k_idx * p.nbq2 * gqa_ratio; + const uint k_base = (batch_k_idx * p.nbk3 + (p_idx * p_reduce) * k_dim + head_k_idx * p.nbk2) / 4; + uint o_offset = batch_idx * p.neq2 * p.neq1 * p_dim + p_idx * p_reduce + head_k_idx * gqa_ratio * p_dim + + (sg * warp_o_row + out_idx * Br * p.neq2) * p_dim + lane; + uint max_offset = batch_idx * p.neq2 * p.neq1 * max_dim + p_idx + head_k_idx * gqa_ratio * max_dim + + (sg * warp_o_row + out_idx * Br * p.neq2) * max_dim; + const uint pvsh_load_offset = sg * warp_o_row * p_reduce + lane; + const uint mask_base = p_idx * p_reduce + lane; + const uint k_seq_boundary = ((p.nek1 - 1) * k_dim + batch_k_idx * p.nbk3 + head_k_idx * p.nbk2 + HSK) / 4 - 1; + const uint k_head_base = (batch_k_idx * p.nbk3 + head_k_idx * p.nbk2) / 4; + const float NEG_FLT_MAX_OVER_2 = uintBitsToFloat(0xFEFFFFFF); + + float Maskf[p_reduce / SubGroupSize]; + + if (out_row_idx < p.neq1 && single_head_mask) { + if (out_row_idx < p.nem1) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint mask_offset = idx * SubGroupSize + mask_base; + if (mask_offset < p.nek1) { + Maskf[idx] = float(data_m[batch_mask_idx * batch_mask_stride + out_row_idx * p.nek1 + mask_offset]); + } else { + Maskf[idx] = NEG_FLT_MAX_OVER_2; + } + } + } else { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint mask_offset = idx * SubGroupSize + mask_base; + Maskf[idx] = (mask_offset < p.nek1) ? 0.0f : NEG_FLT_MAX_OVER_2; + } + } } - } - coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> matP[COOP_MAT_P_M][COOP_MAT_P_N]; + coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> PMat[pmat_row][pmat_col]; - [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { - [[unroll]] for (uint np = 0; np < COOP_MAT_P_N; np++) { - matP[mp][np] = coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f); + [[unroll]] for (uint r = 0; r < pmat_row; ++r) { + [[unroll]] for (uint c = 0; c < pmat_col; ++c) { + PMat[r][c] = coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f); + } } - } - - [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { - const uint flatOffset = kLoad * GROUPSIZE + wgLane; - const uint kRowIdx = flatOffset / HEAD_DIM_VEC4; - const uint kColIdx = flatOffset % HEAD_DIM_VEC4; - const uint slmChunkCol = kColIdx % SLM_CHUNK_SIZE; - const uint slmChunkRow = kColIdx / SLM_CHUNK_SIZE; - const uint offsetK = offsetBaseK + kRowIdx * kvDim / 4 + kColIdx; - const uint offsetSlmK = kRowIdx * TK + slmChunkRow * TK * MATP_PER_LOOP + slmChunkCol * 4; - slm_pool_f16.slm_pool_k[offsetSlmK + 0] = kStateVec4[offsetK].x; - slm_pool_f16.slm_pool_k[offsetSlmK + 1] = kStateVec4[offsetK].y; - slm_pool_f16.slm_pool_k[offsetSlmK + 2] = kStateVec4[offsetK].z; - slm_pool_f16.slm_pool_k[offsetSlmK + 3] = kStateVec4[offsetK].w; - } - - [[unroll]] for (uint pLoop = 0; pLoop < P_LOOP_COUNT; pLoop++) { - f16vec4 kTemp[K_LOAD_LOOPS]; - - if (pLoop + 1 < P_LOOP_COUNT) { - [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { - const uint flatOffset = kLoad * GROUPSIZE + wgLane; - const uint kRowIdx = flatOffset / HEAD_DIM_VEC4 + (pLoop + 1) * MATP_PER_LOOP; - const uint kColIdx = flatOffset % HEAD_DIM_VEC4; - const uint offsetK = offsetBaseK + kRowIdx * kvDim / 4 + kColIdx; - kTemp[kLoad] = kStateVec4[offsetK]; - } + + if (k_in_shmem) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + const uint flat_offset = idx * WorkGroupSize + wg_lane; + const uint row_idx = flat_offset / hsk_v4; + const uint col_idx = flat_offset % hsk_v4; + const uint sh_chunk_c = col_idx % slm_col_chunk; + const uint sh_chunk_r = col_idx / slm_col_chunk; + uint k_offset = k_base + row_idx * k_dim / 4 + col_idx; + k_offset = min(k_offset, k_seq_boundary); + const uint ksh_offset = (row_idx * TK + sh_chunk_r * TK * p_per_loop) / 4 + sh_chunk_c; + F16V4Poolsh.F16V4sh[ksh_offset] = data_kv4[k_offset]; + } } - barrier(); - if (localLinearId < N_TOK) { - [[unroll]] for (uint loop = 0; loop < loopCount; loop++) { - coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> matQ[COOP_MAT_P_M]; - coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> matK[COOP_MAT_P_PER_LOOP]; - - [[unroll]] for (uint mq = 0; mq < COOP_MAT_P_M; mq++) { - coopMatLoad( - matQ[mq], - qState, - offsetBaseQ + mq * TN * HEAD_DIM + loop * TK, - HEAD_DIM, - gl_CooperativeMatrixLayoutColumnMajor); + [[unroll]] for (uint pl = 0; pl < p_loops; ++pl) { + f16vec4 kv4_temp[k_loops]; + + if (k_in_shmem) { + if (pl + 1 < p_loops) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + const uint flat_offset = idx * WorkGroupSize + wg_lane; + const uint row_idx = flat_offset / hsk_v4 + (pl + 1) * p_per_loop; + const uint col_idx = flat_offset % hsk_v4; + uint k_offset = k_base + row_idx * k_dim / 4 + col_idx; + k_offset = min(k_offset, k_seq_boundary); + kv4_temp[idx] = data_kv4[k_offset]; + } + } } - [[unroll]] for (uint np = 0; np < COOP_MAT_P_PER_LOOP; np++) { - coopMatLoad( - matK[np], - slm_pool_f16.slm_pool_k, - loop * TK * MATP_PER_LOOP + np * TM * TK, - TK, - gl_CooperativeMatrixLayoutRowMajor); + barrier(); + if (sg < Br) { + [[unroll]] for (uint loop = 0; loop < HSK / TK; ++loop) { + coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> QMat[pmat_row]; + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> KMat[pmat_count]; + + [[unroll]] for (uint r = 0; r < pmat_row; ++r) { + const uint coord = q_offset + r * TN * p.nbq2 + loop * TK; + coopMatLoad(QMat[r], data_q, coord, p.nbq2, gl_CooperativeMatrixLayoutColumnMajor); + } + + [[unroll]] for (uint c = 0; c < pmat_count; ++c) { + if (k_in_shmem) { + const uint coord = loop * TK * p_per_loop + c * TM * TK; + coopMatLoad(KMat[c], F16Poolsh.F16sh, coord, TK, gl_CooperativeMatrixLayoutRowMajor); + } else { + const uint k_base_row = (c * TM + pl * pmat_count * TM + p_idx * p_reduce); + const uint kv4_base_offset = k_head_base + k_base_row * p.nbk1 / 4 + loop * TK / 4; + [[unroll]] for (uint idx = 0; idx < TM * TK / 4 / SubGroupSize; ++idx) { + const uint flat_offset = idx * SubGroupSize + lane; + const uint row_idx = flat_offset / (TK / 4); + const uint col_idx = flat_offset % (TK / 4); + const uint k_temp_offset = kv4_base_offset + row_idx * p.nbk1 / 4 + col_idx; + const uint ksh_temp_offset = sg * TM * TK / 4 + idx * SubGroupSize + lane; + f16vec4 kv4_temp = (k_base_row + row_idx) < p.nek1 ? data_kv4[k_temp_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[ksh_temp_offset] = kv4_temp; + } + + subgroupBarrier(); + coopMatLoad(KMat[c], F16Poolsh.F16sh, sg * TM * TK, TK, gl_CooperativeMatrixLayoutRowMajor); + subgroupBarrier(); + } + } + + [[unroll]] for (uint r = 0; r < pmat_row; ++r) { + [[unroll]] for (uint c = 0; c < pmat_count; ++c) { + PMat[r][pl * pmat_count + c] = coopMatMulAdd(KMat[c], QMat[r], PMat[r][pl * pmat_count + c]); + } + } + } } - [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { - [[unroll]] for (uint np = 0; np < COOP_MAT_P_PER_LOOP; np++) { - matP[mp][pLoop * COOP_MAT_P_PER_LOOP + np] = coopMatMulAdd(matK[np], matQ[mp], matP[mp][pLoop * COOP_MAT_P_PER_LOOP + np]); - } + barrier(); + + if (k_in_shmem) { + if (pl + 1 < p_loops) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + const uint flat_offset = idx * WorkGroupSize + wg_lane; + const uint row_idx = flat_offset / hsk_v4; + const uint col_idx = flat_offset % hsk_v4; + const uint sh_chunk_c = col_idx % slm_col_chunk; + const uint sh_chunk_r = col_idx / slm_col_chunk; + const uint ksh_offset = (row_idx * TK + sh_chunk_r * TK * p_per_loop) / 4 + sh_chunk_c; + F16V4Poolsh.F16V4sh[ksh_offset] = kv4_temp[idx]; + } + } } - } } barrier(); - if (pLoop + 1 < P_LOOP_COUNT) { - [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { - const uint flatOffset = kLoad * GROUPSIZE + wgLane; - const uint kRowIdx = flatOffset / HEAD_DIM_VEC4; - const uint kColIdx = flatOffset % HEAD_DIM_VEC4; - const uint slmChunkCol = kColIdx % SLM_CHUNK_SIZE; - const uint slmChunkRow = kColIdx / SLM_CHUNK_SIZE; - const uint offsetSlmK = kRowIdx * TK + slmChunkRow * TK * MATP_PER_LOOP + slmChunkCol * 4; - slm_pool_f16.slm_pool_k[offsetSlmK + 0] = kTemp[kLoad].x; - slm_pool_f16.slm_pool_k[offsetSlmK + 1] = kTemp[kLoad].y; - slm_pool_f16.slm_pool_k[offsetSlmK + 2] = kTemp[kLoad].z; - slm_pool_f16.slm_pool_k[offsetSlmK + 3] = kTemp[kLoad].w; - } - } - } - - barrier(); - - if (tokFlatIdx < p.activationLength) { - [[unroll]] for (uint mk = 0; mk < MATP_REDUCE / WARPSIZE; mk++) { - slm_pool_f32.slm_pool_pv[SLM_PV_SIZE + localLinearId * MATP_REDUCE + mk * WARPSIZE + lane] = maskFp32[mk]; - } - } - - [[unroll]] for (uint oLoop = 0; oLoop < N_TOK; oLoop++) { - if (oLoop + outGroupIdx * N_TOK < p.activationLength) { - if (localLinearId == oLoop) { - [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { - [[unroll]] for (uint np = 0; np < COOP_MAT_P_N; np++) { - coopMatStore(matP[mp][np], slm_pool_f32.slm_pool_pv, mp * MATP_REDUCE * TN + np * TM, MATP_REDUCE, gl_CooperativeMatrixLayoutColumnMajor); - } - } - } - - barrier(); - - [[unroll]] for (uint maskIdx = 0; maskIdx < MATP_REDUCE / WARPSIZE; maskIdx++) { - maskFp32[maskIdx] = slm_pool_f32.slm_pool_pv[SLM_PV_SIZE + oLoop * MATP_REDUCE + maskIdx * WARPSIZE + lane]; - } - - float fp32O[O_COUNT][MATP_REDUCE / WARPSIZE]; - float maxOut[O_COUNT]; - - [[unroll]] for (uint oc = 0; oc < O_COUNT; oc++) { - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - fp32O[oc][os] = slm_pool_f32.slm_pool_pv[offsetSlmLoadPv + os * WARPSIZE + oc * MATP_REDUCE] * p.softMaxScale; + if (sg < Br && out_row_idx < p.neq1 && single_head_mask) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + F32Poolsh.F32sh[SLM_PV_SIZE + sg * p_reduce + idx * SubGroupSize + lane] = Maskf[idx]; } + } - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - fp32O[oc][os] = fp32O[oc][os] + maskFp32[os]; - } - - float maxTemp = fp32Min; - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - maxTemp = max(maxTemp, fp32O[oc][os]); - } - maxOut[oc] = subgroupMax(maxTemp); - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - fp32O[oc][os] = exp(fp32O[oc][os] - maxOut[oc]); - } - } - - [[unroll]] for (uint oc = 0; oc < O_COUNT; oc++) { - if (localLinearId * O_COUNT + oc < GQA_RATIO) { - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - matP_f16[offsetOut + oc * pDim + os * WARPSIZE] = float16_t(fp32O[oc][os]); - } - - if (lane == 0) { - out_max_f32[offsetMax + oc * maxDim] = maxOut[oc]; - } + [[unroll]] for (uint out_r = 0; out_r < Br; ++out_r) { + if (out_r + out_idx * Br < p.neq1) { + const uint mask_global_base = batch_mask_idx * batch_mask_stride + (out_r + out_idx * Br) * p.nek1; + if (sg == out_r) { + [[unroll]] for (uint r = 0; r < pmat_row; ++r) { + [[unroll]] for (uint c = 0; c < pmat_col; ++c) { + const uint coord = r * p_reduce * TN + c * TM; + coopMatStore(PMat[r][c], F32Poolsh.F32sh, coord, p_reduce, gl_CooperativeMatrixLayoutColumnMajor); + } + } + } + + barrier(); + + if (single_head_mask) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + Maskf[idx] = F32Poolsh.F32sh[SLM_PV_SIZE + out_r * p_reduce + idx * SubGroupSize + lane]; + } + } + + float Of[warp_o_row][p_reduce / SubGroupSize]; + float Mf[warp_o_row]; + + [[unroll]] for (uint r = 0; r < warp_o_row; ++r) { + const uint head_q_idx = head_k_idx * gqa_ratio + sg * warp_o_row + r; + const uint mask_head_offset = (head_q_idx % p.nem2) * p.nek1 * p.nem1; + + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + Of[r][idx] = F32Poolsh.F32sh[pvsh_load_offset + idx * SubGroupSize + r * p_reduce] * p.scale; + } + + if (!single_head_mask) { + if (out_r + out_idx * Br < p.nem1) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint mask_offset = idx * SubGroupSize + mask_base; + if (mask_offset < p.nek1) { + Maskf[idx] = float(data_m[mask_global_base + mask_head_offset + mask_offset]); + } else { + Maskf[idx] = NEG_FLT_MAX_OVER_2; + } + } + } else { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint mask_offset = idx * SubGroupSize + mask_base; + Maskf[idx] = (mask_offset < p.nek1) ? 0.0f : NEG_FLT_MAX_OVER_2; + } + } + } + + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + Of[r][idx] = Of[r][idx] + Maskf[idx]; + } + + float Mtempf = NEG_FLT_MAX_OVER_2; + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint p_col_offset = idx * SubGroupSize + mask_base; + if (p_col_offset < p.nek1) { + Mtempf = max(Mtempf, Of[r][idx]); + } + } + Mf[r] = subgroupMax(Mtempf); + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint p_col_offset = idx * SubGroupSize + mask_base; + Of[r][idx] = (p_col_offset < p.nek1) ? exp(Of[r][idx] - Mf[r]) : 0.0f; + } + } + + [[unroll]] for (uint r = 0; r < warp_o_row; ++r) { + if (sg * warp_o_row + r < gqa_ratio) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + data_p[o_offset + r * p_dim + idx * SubGroupSize] = float16_t(Of[r][idx]); + } + + if (lane == 0) { + data_max[max_offset + r * max_dim] = Mf[r]; + } + } + } + + o_offset = o_offset + p.neq2 * p_dim; + max_offset = max_offset + p.neq2 * max_dim; + barrier(); } - } - - offsetOut = offsetOut + p.qHead * pDim; - offsetMax = offsetMax + p.qHead * maxDim; - barrier(); } - } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp index 60a1c2ce7..42a5101d8 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp @@ -3,406 +3,470 @@ #extension GL_EXT_control_flow_attributes : enable #extension GL_EXT_shader_16bit_storage : require #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require -#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #extension GL_KHR_memory_scope_semantics : enable #extension GL_KHR_shader_subgroup_basic : enable -#extension GL_KHR_shader_subgroup_ballot : enable #extension GL_KHR_shader_subgroup_arithmetic : enable #extension GL_KHR_cooperative_matrix : enable #extension GL_EXT_shared_memory_block : enable -layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +layout (local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -layout (binding = 0) readonly buffer P {f16vec4 pStateVec4[];}; -layout (binding = 1) readonly buffer V {float16_t vState[];}; -layout (binding = 1) readonly buffer V_VEC4 {f16vec4 vStateVec4[];}; -layout (binding = 2) buffer MAX_FP32 {float max_f32[];}; -layout (binding = 3) buffer SINK_FP32 {float sink_f32[];}; -layout (binding = 4) buffer OUT_FP32 {float out_f32[];}; -layout (binding = 4) buffer OUT_VEC4 {vec4 out_f32_vec4[];}; -layout (binding = 4) buffer OUT_F16 {float16_t out_f16[];}; +layout (binding = 0) readonly buffer PV4 {f16vec4 data_pv4[];}; +layout (binding = 1) readonly buffer V {float16_t data_v[];}; +layout (binding = 1) readonly buffer VV4 {f16vec4 data_vv4[];}; +layout (binding = 2) readonly buffer MAX {float data_max[];}; +layout (binding = 3) readonly buffer S {float data_s[];}; +layout (binding = 4) buffer OV4 {vec4 data_ov4[];}; layout (push_constant) uniform parameter { - uint kvSeqLen; - uint activationLength; - uint qHead; - uint kvHead; - uint qkRatio; - uint qkSubGroups; - uint flag; - uint kvStride1; - uint kvStride2; - uint batchStrideQ; - uint batchStrideK; - uint batchStrideV; - uint batchStrideM; - uint batchStrideO; - float softMaxScale; + uint neq1; + uint neq2; + uint neq3; + uint nek1; + uint nek2; + uint nek3; + uint nev2; + uint nev3; + uint nem1; + uint nem2; + uint nem3; + uint nbq1; + uint nbq2; + uint nbk1; + uint nbk2; + uint nbv1; + uint nbv2; + uint nbq3; + uint nbk3; + uint nbv3; + uint nb3; + uint flag; + float scale; } p; -layout (constant_id = 0) const uint GROUPSIZE = 256; -layout (constant_id = 1) const uint GQA_RATIO = 8; -layout (constant_id = 2) const uint HEAD_DIM = 128; -layout (constant_id = 3) const uint N_TOKS_PER_GROUP = 1; -layout (constant_id = 4) const uint WARPSIZE = 16; -layout (constant_id = 5) const uint MATP_PER_LOOP = 64; -layout (constant_id = 6) const uint MATP_REDUCE = 32; -layout (constant_id = 7) const uint WARP_V_DIM = 16; - -#define TN WARPSIZE -#define TM 8 -#define TK 16 -#define MAT_O_N (WARP_V_DIM / TM) -#define MAT_P_M (GQA_RATIO * N_TOKS_PER_GROUP) -#define ALIGNED_P_M ((MAT_P_M + WARPSIZE - 1) / WARPSIZE) -#define V_HEAD_GROUPS (HEAD_DIM / WARP_V_DIM) - -#define SUBGROUP_COUNT (GROUPSIZE / WARPSIZE) -#define SPLIT_P_GROUPS (MATP_PER_LOOP / TK) - -#define SLM_POOL_SIZE_O (SUBGROUP_COUNT * ALIGNED_P_M * TN * MAT_O_N * TM) - -#define P_LOAD_PER_LOOP (GROUPSIZE * 4) -#define P_LOAD_LOOPS ((MAT_P_M * MATP_PER_LOOP + P_LOAD_PER_LOOP - 1) / P_LOAD_PER_LOOP) -#define SLM_POOL_SIZE_P (P_LOAD_LOOPS * P_LOAD_PER_LOOP) -#define SIZE_LOCAL_MAX (MAT_P_M * MATP_PER_LOOP / MATP_REDUCE) -#define MAX_LOAD_LOOPS ((SIZE_LOCAL_MAX + GROUPSIZE - 1) / GROUPSIZE) -#define SLM_POOL_SIZE_LOCAL_MAX (MAX_LOAD_LOOPS * GROUPSIZE) -#define MAX_REDUCE_COUNT ((MAT_P_M + SUBGROUP_COUNT - 1) / SUBGROUP_COUNT) -#define GLOBAL_MAX_SIZE (MAX_REDUCE_COUNT * SUBGROUP_COUNT) - -#define SLM_POOL_SIZE_SOFTMAX_SUM (SUBGROUP_COUNT * P_LOAD_LOOPS) - -#define SLM_OFFSET_P (GLOBAL_MAX_SIZE * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_LOCAL_MAX * 2 * 2) - -#define SLM_OFFSET_GLOBAL_MAX 0 -#define SLM_OFFSET_SOFTMAX_SUM (SLM_OFFSET_GLOBAL_MAX + GLOBAL_MAX_SIZE) -#define SLM_OFFSET_O (SLM_OFFSET_SOFTMAX_SUM + SLM_POOL_SIZE_SOFTMAX_SUM) -#define SLM_OFFSET_LOCAL_MAX (GLOBAL_MAX_SIZE + SLM_POOL_SIZE_SOFTMAX_SUM) - -#define P_REDUCE_VEC4 (MATP_PER_LOOP / 4) -#define MAX_PER_LOOP (MATP_PER_LOOP / MATP_REDUCE) -#define SLM_MAX_STRIDE (MATP_REDUCE / 4) -#define SUB_GROUPS_PER_LINE (MATP_PER_LOOP / WARPSIZE / 4) - -shared slm_pool_block { - float slm_pool_o[GLOBAL_MAX_SIZE + SLM_POOL_SIZE_SOFTMAX_SUM + SLM_POOL_SIZE_O]; -} slm_pool_f32; - -shared slm_pool_alias_block { - float16_t slm_pool_pv[GLOBAL_MAX_SIZE * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_LOCAL_MAX * 2 * 2 + SLM_POOL_SIZE_P * 2]; -} slm_pool_alias_f16; +layout (constant_id = 0) const uint WorkGroupSize = 256; +layout (constant_id = 1) const uint gqa_ratio = 8; +layout (constant_id = 2) const uint HSV = 128; +layout (constant_id = 3) const uint Wr = 1; +layout (constant_id = 4) const uint SubGroupSize = 16; +layout (constant_id = 5) const uint p_per_loop = 64; +layout (constant_id = 6) const uint p_reduce = 32; +layout (constant_id = 7) const uint Bc = 16; +layout (constant_id = 8) const uint TM = 8; +layout (constant_id = 9) const uint TN = 16; +layout (constant_id = 10) const uint Flags = 0; + +#define UNALIGNED_SEQ_LEN_BIT (1 << 0) +#define SINK_ENABLE_BIT (1 << 16) + +const uint TK = 16; +const uint o_row = gqa_ratio * Wr; +const uint omat_row = (o_row + TN - 1) / TN; +const uint o_scalar_row = (o_row + SubGroupSize - 1) / SubGroupSize; + +const uint omat_col = Bc / TM; +const uint v_head_wgs = HSV / Bc; + +const uint sg_count = WorkGroupSize / SubGroupSize; +const uint split_p_sg = p_per_loop / TK; +const uint p_loop_col = WorkGroupSize * 4; +const uint p_loops = (o_row * p_per_loop + p_loop_col - 1) / p_loop_col; + +const uint size_max = o_row * p_per_loop / p_reduce; +const uint max_loops = (size_max + WorkGroupSize - 1) / WorkGroupSize; +const uint global_max_loops = (o_row + sg_count - 1) / sg_count; + +#define SLM_POOL_SIZE_GLOBAL_MAX ((global_max_loops * sg_count + SubGroupSize - 1) / SubGroupSize * SubGroupSize) +#define SLM_POOL_SIZE_O (sg_count * omat_row * TN * omat_col * TM) +#define SLM_POOL_SIZE_MAX (max_loops * WorkGroupSize) +#define SLM_POOL_SIZE_SOFTMAX_SUM ((sg_count * p_loops + SubGroupSize - 1) / SubGroupSize * SubGroupSize) +#define SLM_POOL_SIZE_OOB_STAGING (sg_count * TM * TK / 4) + +const uint psh_size = (p_loops * p_loop_col < omat_row * TN * p_per_loop) ? (omat_row * TN * p_per_loop) : (p_loops * p_loop_col); +const uint psh_base = SLM_POOL_SIZE_GLOBAL_MAX * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_MAX * 2 * 2; + +const uint global_maxsh_offset = 0; +const uint lfsh_offset = global_maxsh_offset + SLM_POOL_SIZE_GLOBAL_MAX; +const uint osh_offset = lfsh_offset + SLM_POOL_SIZE_SOFTMAX_SUM; +const uint maxsh_offset = SLM_POOL_SIZE_GLOBAL_MAX + SLM_POOL_SIZE_SOFTMAX_SUM; + +const uint p_v4 = p_per_loop / 4; +const uint max_per_loop = p_per_loop / p_reduce; +const uint maxsh_stride = p_reduce / 4; +const uint sg_per_line = p_per_loop / SubGroupSize / 4; +const bool seq_boundary_check = (Flags & UNALIGNED_SEQ_LEN_BIT) != 0; + +shared sh_block { + float F32sh[SLM_POOL_SIZE_GLOBAL_MAX + SLM_POOL_SIZE_SOFTMAX_SUM + SLM_POOL_SIZE_O]; +} F32Poolsh; + +shared shf16_alias { + float16_t F16sh[SLM_POOL_SIZE_GLOBAL_MAX * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_MAX * 2 * 2 + psh_size * 2]; +} F16Poolsh; + +shared shf16v4_alias { + f16vec4 F16V4sh[(SLM_POOL_SIZE_GLOBAL_MAX + SLM_POOL_SIZE_SOFTMAX_SUM + SLM_POOL_SIZE_MAX * 2) / 2 + SLM_POOL_SIZE_OOB_STAGING]; +} F16V4Poolsh; void main() { - const uint lane = gl_SubgroupInvocationID; - const uint v = gl_WorkGroupID.y; - const uint d = gl_WorkGroupID.z; - const uint vWarpIdx = gl_WorkGroupID.x % V_HEAD_GROUPS; - const uint outTokIdx = gl_WorkGroupID.x / V_HEAD_GROUPS; - const uint localLinearId = gl_SubgroupID; - const uint wgLane = localLinearId * WARPSIZE + lane; - const uint splitIdx = localLinearId; - const uint maxDim = (p.kvSeqLen + MATP_REDUCE - 1) / MATP_REDUCE; - const uint pDim = maxDim * MATP_REDUCE; - const uint kvDim = p.kvStride1; - const uint oDim = p.qHead * HEAD_DIM; - const uint offsetBaseP = (d * p.activationLength * p.qHead + v * GQA_RATIO + outTokIdx * N_TOKS_PER_GROUP * p.qHead) * pDim / 4; - const uint offsetBaseMax = (d * p.activationLength * p.qHead + v * GQA_RATIO + outTokIdx * N_TOKS_PER_GROUP * p.qHead) * maxDim; - const uint offsetBaseV = (d * p.batchStrideV + v * p.kvStride2 + vWarpIdx * WARP_V_DIM + splitIdx * TK * kvDim); - const uint offsetSlmP = (SLM_OFFSET_P + wgLane * 4); - const float fp32Min = uintBitsToFloat(0xFEFFFFFF); - const float fp32Max = uintBitsToFloat(0x7EFFFFFF); - uint offsetV = offsetBaseV; - - coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> sums[ALIGNED_P_M][MAT_O_N]; - f16vec4 pStateTemp[P_LOAD_LOOPS]; - - float fp32CompensationP[P_LOAD_LOOPS]; - - uint loadRowBase[P_LOAD_LOOPS]; - uint loadColBase[P_LOAD_LOOPS]; - float fp32SoftMaxSum[P_LOAD_LOOPS]; - float fp32GlobalMaxP[P_LOAD_LOOPS]; - uint maxRowBase[MAX_LOAD_LOOPS]; - uint maxColBase[MAX_LOAD_LOOPS]; - uint outOffsets[ALIGNED_P_M]; - bool outputMask[ALIGNED_P_M]; - float fp32SinkCoeff[ALIGNED_P_M]; - - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - const uint flatOffset = pm * WARPSIZE + lane; - const uint inGroupTokIdx = flatOffset / GQA_RATIO; - const uint inGroupHeadIdx = flatOffset % GQA_RATIO; - outputMask[pm] = (N_TOKS_PER_GROUP * outTokIdx + inGroupTokIdx < p.activationLength) && (inGroupHeadIdx < GQA_RATIO) && (inGroupTokIdx < N_TOKS_PER_GROUP); - outOffsets[pm] = (inGroupTokIdx * oDim + inGroupHeadIdx * HEAD_DIM) / 4; - if ((0x1 & p.flag) != 0) { - fp32SinkCoeff[pm] = sink_f32[inGroupHeadIdx + v * GQA_RATIO]; - } - } - - [[unroll]] for (uint maxCount = 0; maxCount < MAX_REDUCE_COUNT; maxCount++) { - const uint flatIdx = maxCount * SUBGROUP_COUNT + localLinearId; - const uint rowIdx = flatIdx % GQA_RATIO; - const uint tokIdx = flatIdx / GQA_RATIO; - - if (tokIdx < N_TOKS_PER_GROUP) { - float fp32MaxReduce = fp32Min; - const uint maxOffset = offsetBaseMax + (tokIdx * p.qHead + rowIdx) * maxDim; - [[unroll]] for (uint maxReduce = 0; maxReduce < (maxDim + WARPSIZE - 1) / WARPSIZE; maxReduce++) { - if (maxReduce * WARPSIZE + lane < maxDim) { - fp32MaxReduce = max(fp32MaxReduce, max_f32[maxOffset + maxReduce * WARPSIZE + lane]); + const uint lane = gl_SubgroupInvocationID; + const uint head_v_idx = gl_WorkGroupID.y; + const uint batch_v_ratio = p.neq3 / p.nev3; + const uint batch_idx = gl_WorkGroupID.z; + const uint batch_v_idx = batch_idx / batch_v_ratio; + const uint v_idx = gl_WorkGroupID.x % v_head_wgs; + const uint out_row_idx = gl_WorkGroupID.x / v_head_wgs; + const uint sg = gl_SubgroupID; + const uint wg_lane = sg * SubGroupSize + lane; + const uint max_dim = (p.nek1 + p_reduce - 1) / p_reduce; + const uint p_dim = max_dim * p_reduce; + const uint v_dim = p.nbv1; + const uint o_dim = p.neq2 * HSV; + const uint p_base = (batch_idx * p.neq1 * p.neq2 + head_v_idx * gqa_ratio + out_row_idx * Wr * p.neq2) * p_dim / 4; + const uint max_base = (batch_idx * p.neq1 * p.neq2 + head_v_idx * gqa_ratio + out_row_idx * Wr * p.neq2) * max_dim; + const uint v_base = batch_v_idx * p.nbv3 + head_v_idx * p.nbv2 + v_idx * Bc + sg * TK * v_dim; + const uint o_base = (batch_idx * p.nb3 + v_idx * Bc + head_v_idx * gqa_ratio * HSV + out_row_idx * o_dim * Wr) / 4; + const uint psh_offset = psh_base + wg_lane * 4; + const float NEG_FLT_MAX_OVER_2 = uintBitsToFloat(0xFEFFFFFF); + const float POS_FLT_MAX_OVER_2 = uintBitsToFloat(0x7EFFFFFF); + uint v_offset = v_base; + + coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> OMat[omat_row][omat_col]; + f16vec4 pv4_temp[p_loops]; + + float eMf[p_loops]; + + uint p_row_base[p_loops]; + uint p_col_base[p_loops]; + float Lf[p_loops]; + float global_max[p_loops]; + uint max_row_base[max_loops]; + uint max_col_base[max_loops]; + uint o_offset[o_scalar_row]; + bool o_mask[o_scalar_row]; + float sink[o_scalar_row]; + + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + const uint flat_offset = r * SubGroupSize + lane; + const uint row_idx = flat_offset / gqa_ratio; + const uint col_idx = flat_offset % gqa_ratio; + o_mask[r] = (Wr * out_row_idx + row_idx < p.neq1) && (col_idx < gqa_ratio) && (row_idx < Wr); + o_offset[r] = (row_idx * o_dim + col_idx * HSV) / 4; + if ((p.flag & SINK_ENABLE_BIT) != 0) { + sink[r] = data_s[col_idx + head_v_idx * gqa_ratio]; } - } - fp32MaxReduce = subgroupMax(fp32MaxReduce); - if (lane == 0) { - slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + maxCount * SUBGROUP_COUNT + localLinearId] = fp32MaxReduce; - } - } else { - if (lane == 0) { - slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + maxCount * SUBGROUP_COUNT + localLinearId] = fp32Max; - } - } - } - - barrier(); - - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint flatOffset = (pLoad * GROUPSIZE + wgLane) / P_REDUCE_VEC4; - const uint rowIdxFlat = flatOffset % GQA_RATIO; - const uint tokenIdxFlat = min(flatOffset / GQA_RATIO, N_TOKS_PER_GROUP - 1); - loadColBase[pLoad] = (pLoad * GROUPSIZE + wgLane) % P_REDUCE_VEC4; - loadRowBase[pLoad] = (tokenIdxFlat * p.qHead + rowIdxFlat); - fp32SoftMaxSum[pLoad] = 0.0f; - fp32GlobalMaxP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + flatOffset]; - } - - [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { - const uint flatOffset = (maxLoad * GROUPSIZE + wgLane) / MAX_PER_LOOP; - const uint rowIdxFlat = flatOffset % GQA_RATIO; - const uint tokenIdxFlat = min(flatOffset / GQA_RATIO, N_TOKS_PER_GROUP - 1); - maxColBase[maxLoad] = (maxLoad * GROUPSIZE + wgLane) % MAX_PER_LOOP; - maxRowBase[maxLoad] = (tokenIdxFlat * p.qHead + rowIdxFlat); - } - - [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { - const uint flatMaxOffset = maxRowBase[maxLoad] * maxDim + maxColBase[maxLoad]; - slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + maxLoad * GROUPSIZE + wgLane] = max_f32[offsetBaseMax + flatMaxOffset]; - maxColBase[maxLoad] = maxColBase[maxLoad] + MATP_PER_LOOP / MATP_REDUCE; - } - - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint flatOffset = loadRowBase[pLoad] * pDim / 4 + loadColBase[pLoad]; - pStateTemp[pLoad] = pStateVec4[offsetBaseP + flatOffset]; - } - - [[unroll]] for (uint n = 0; n < ALIGNED_P_M; n++) { - [[unroll]] for (uint i = 0; i < MAT_O_N; i++) { - sums[n][i] = coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f); - } - } - - barrier(); - - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint maxOffset = (pLoad * GROUPSIZE + wgLane) / SLM_MAX_STRIDE; - if (loadColBase[pLoad] < pDim / 4) { - fp32CompensationP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + maxOffset]; - float pTemp[4] = float[4](pStateTemp[pLoad].x, pStateTemp[pLoad].y, pStateTemp[pLoad].z, pStateTemp[pLoad].w); - float compTemp = exp(fp32CompensationP[pLoad] - fp32GlobalMaxP[pLoad]); - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - pTemp[kk] = pTemp[kk] * compTemp; - fp32SoftMaxSum[pLoad] = fp32SoftMaxSum[pLoad] + pTemp[kk]; - slm_pool_alias_f16.slm_pool_pv[offsetSlmP + pLoad * GROUPSIZE * 4 + kk] = float16_t(pTemp[kk]); - } - } else { - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - slm_pool_alias_f16.slm_pool_pv[offsetSlmP + pLoad * GROUPSIZE * 4 + kk] = float16_t(0.0f); - } } - loadColBase[pLoad] = loadColBase[pLoad] + P_REDUCE_VEC4; - } + if (p.nek1 == 0) { + if (sg == 0) { + [[unroll]] for (uint c = 0; c < Bc / 4; ++c) { + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + if (o_mask[r]) { + data_ov4[o_base + o_offset[r] + c] = vec4(0.0f, 0.0f, 0.0f, 0.0f); + } + } + } + } - const uint loopCount = (p.kvSeqLen + MATP_PER_LOOP - 1) / MATP_PER_LOOP; + return; + } - for (uint loop = 0; loop < loopCount; loop++) { - const uint slmPingPongLoad = (loop & 0x1); - const uint slmPingPongStore = ((loop + 1) & 0x1); + [[unroll]] for (uint idx = 0; idx < global_max_loops; ++idx) { + const uint flat_offset = idx * sg_count + sg; + const uint row_idx = flat_offset % gqa_ratio; + const uint tok_idx = flat_offset / gqa_ratio; + + if (tok_idx < Wr) { + float Maxf = NEG_FLT_MAX_OVER_2; + const uint max_offset = max_base + (tok_idx * p.neq2 + row_idx) * max_dim; + [[unroll]] for (uint max_reduce = 0; max_reduce < (max_dim + SubGroupSize - 1) / SubGroupSize; ++max_reduce) { + if (max_reduce * SubGroupSize + lane < max_dim) { + Maxf = max(Maxf, data_max[max_offset + max_reduce * SubGroupSize + lane]); + } + } + Maxf = subgroupMax(Maxf); + if (lane == 0) { + F32Poolsh.F32sh[global_maxsh_offset + idx * sg_count + sg] = Maxf; + } + } else { + if (lane == 0) { + F32Poolsh.F32sh[global_maxsh_offset + idx * sg_count + sg] = POS_FLT_MAX_OVER_2; + } + } + } - if (loop + 1 < loopCount) { - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint flatOffset = loadRowBase[pLoad] * pDim / 4 + loadColBase[pLoad]; - pStateTemp[pLoad] = pStateVec4[offsetBaseP + flatOffset]; - } + barrier(); - [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { - const uint flatMaxOffset = maxRowBase[maxLoad] * maxDim + maxColBase[maxLoad]; - slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + slmPingPongStore * SLM_POOL_SIZE_LOCAL_MAX + maxLoad * GROUPSIZE + wgLane] = max_f32[offsetBaseMax + flatMaxOffset]; - maxColBase[maxLoad] = maxColBase[maxLoad] + MATP_PER_LOOP / MATP_REDUCE; - } + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint flat_offset = (idx * WorkGroupSize + wg_lane) / p_v4; + const uint row_idx = flat_offset % gqa_ratio; + const uint tok_idx = min(flat_offset / gqa_ratio, Wr - 1); + p_col_base[idx] = (idx * WorkGroupSize + wg_lane) % p_v4; + p_row_base[idx] = (tok_idx * p.neq2 + row_idx); + Lf[idx] = 0.0f; + global_max[idx] = F32Poolsh.F32sh[global_maxsh_offset + flat_offset]; } - barrier(); + [[unroll]] for (uint idx = 0; idx < max_loops; ++idx) { + const uint flat_offset = (idx * WorkGroupSize + wg_lane) / max_per_loop; + const uint row_idx = flat_offset % gqa_ratio; + const uint tok_idx = min(flat_offset / gqa_ratio, Wr - 1); + max_col_base[idx] = (idx * WorkGroupSize + wg_lane) % max_per_loop; + max_row_base[idx] = (tok_idx * p.neq2 + row_idx); + } - { - const uint coopMatOffsetP = SLM_OFFSET_P + slmPingPongLoad * SLM_POOL_SIZE_P + splitIdx * TK; - coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> matV[MAT_O_N]; - [[unroll]] for (uint cc = 0; cc < MAT_O_N; cc++) { - coopMatLoad( - matV[cc], - vState, - offsetV + TM * cc, - kvDim, - gl_CooperativeMatrixLayoutColumnMajor); - } - [[unroll]] for (uint mo = 0; mo < ALIGNED_P_M; mo++) { - coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> matP; - coopMatLoad( - matP, - slm_pool_alias_f16.slm_pool_pv, - coopMatOffsetP + mo * TN * MATP_PER_LOOP, - MATP_PER_LOOP, - gl_CooperativeMatrixLayoutColumnMajor); - - [[unroll]] for (uint no = 0; no < MAT_O_N; no++) { - sums[mo][no] = coopMatMulAdd(matV[no], matP, sums[mo][no]); + [[unroll]] for (uint idx = 0; idx < max_loops; ++idx) { + const uint flat_offset = max_row_base[idx] * max_dim + max_col_base[idx]; + if (seq_boundary_check) { + F32Poolsh.F32sh[maxsh_offset + idx * WorkGroupSize + wg_lane] = + (max_col_base[idx] < max_dim) ? data_max[max_base + flat_offset] : NEG_FLT_MAX_OVER_2; + } else { + F32Poolsh.F32sh[maxsh_offset + idx * WorkGroupSize + wg_lane] = data_max[max_base + flat_offset]; } - } + max_col_base[idx] = max_col_base[idx] + p_per_loop / p_reduce; } - offsetV += MATP_PER_LOOP * kvDim; - if (loop * MATP_PER_LOOP + splitIdx * TK >= p.kvSeqLen) { - offsetV = 0; - } - if (loop + 1 < loopCount) { - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint maxOffset = (pLoad * GROUPSIZE + wgLane) / SLM_MAX_STRIDE; - if (loadColBase[pLoad] < pDim / 4) { - fp32CompensationP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + slmPingPongStore * SLM_POOL_SIZE_LOCAL_MAX + maxOffset]; - float pTemp[4] = float[4](pStateTemp[pLoad].x, pStateTemp[pLoad].y, pStateTemp[pLoad].z, pStateTemp[pLoad].w); - float compTemp = exp(fp32CompensationP[pLoad] - fp32GlobalMaxP[pLoad]); - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - pTemp[kk] = pTemp[kk] * compTemp; - fp32SoftMaxSum[pLoad] = fp32SoftMaxSum[pLoad] + pTemp[kk]; - slm_pool_alias_f16.slm_pool_pv[offsetSlmP + slmPingPongStore * SLM_POOL_SIZE_P + pLoad * GROUPSIZE * 4 + kk] = float16_t(pTemp[kk]); - } + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint flat_offset = p_row_base[idx] * p_dim / 4 + p_col_base[idx]; + if (seq_boundary_check) { + pv4_temp[idx] = (p_col_base[idx] < p_dim / 4) ? data_pv4[p_base + flat_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); } else { - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - slm_pool_alias_f16.slm_pool_pv[offsetSlmP + slmPingPongStore * SLM_POOL_SIZE_P + pLoad * GROUPSIZE * 4 + kk] = float16_t(0.0f); - } + pv4_temp[idx] = data_pv4[p_base + flat_offset]; } - loadColBase[pLoad] = loadColBase[pLoad] + P_REDUCE_VEC4; - } } - } - - barrier(); - - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - fp32SoftMaxSum[pLoad] = subgroupAdd(fp32SoftMaxSum[pLoad]); - } - - [[unroll]] for (uint mo = 0; mo < ALIGNED_P_M; mo++) { - [[unroll]] for (uint no = 0; no < MAT_O_N; no++) { - coopMatStore( - sums[mo][no], - slm_pool_f32.slm_pool_o, - SLM_OFFSET_O + mo * TN * WARP_V_DIM + TM * no + localLinearId * ALIGNED_P_M * TN * WARP_V_DIM, - WARP_V_DIM, - gl_CooperativeMatrixLayoutColumnMajor); + + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + OMat[r][c] = coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f); + } } - } - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pLoad * SUBGROUP_COUNT + localLinearId] = fp32SoftMaxSum[pLoad]; - } + barrier(); - barrier(); + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint max_offset = (idx * WorkGroupSize + wg_lane) / maxsh_stride; + eMf[idx] = F32Poolsh.F32sh[maxsh_offset + max_offset]; + float p_temp[4] = float[4](pv4_temp[idx].x, pv4_temp[idx].y, pv4_temp[idx].z, pv4_temp[idx].w); + float eMf_temp = exp(eMf[idx] - global_max[idx]); + [[unroll]] for (uint i = 0; i < 4; ++i) { + p_temp[i] = p_temp[i] * eMf_temp; + Lf[idx] = Lf[idx] + p_temp[i]; + F16Poolsh.F16sh[psh_offset + idx * WorkGroupSize * 4 + i] = float16_t(p_temp[i]); + } + p_col_base[idx] = p_col_base[idx] + p_v4; + } - if (localLinearId == 1) { - const uint sumBase = SLM_OFFSET_SOFTMAX_SUM + lane * SUB_GROUPS_PER_LINE; - float sumTemp[ALIGNED_P_M][SUB_GROUPS_PER_LINE]; - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - [[unroll]] for (uint reduce = 0; reduce < SUB_GROUPS_PER_LINE; reduce++) { - sumTemp[pm][reduce] = slm_pool_f32.slm_pool_o[sumBase + pm * WARPSIZE * SUB_GROUPS_PER_LINE + reduce]; - } + const uint loop_count = seq_boundary_check ? ((p.nek1 + p_per_loop - 1) / p_per_loop - 1) : (p.nek1 / p_per_loop); + uint sh_pingpong_load = 0; + uint sh_pingpong_store = 0; + + for (uint loop = 0; loop < loop_count; ++loop) { + sh_pingpong_load = loop & 0x1; + sh_pingpong_store = (loop + 1) & 0x1; + + if (seq_boundary_check || loop + 1 < loop_count) { + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint flat_offset = p_row_base[idx] * p_dim / 4 + p_col_base[idx]; + if (seq_boundary_check) { + pv4_temp[idx] = (p_col_base[idx] < p_dim / 4) ? data_pv4[p_base + flat_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + } else { + pv4_temp[idx] = data_pv4[p_base + flat_offset]; + } + } + + [[unroll]] for (uint idx = 0; idx < max_loops; ++idx) { + const uint flat_offset = max_row_base[idx] * max_dim + max_col_base[idx]; + if (seq_boundary_check) { + F32Poolsh.F32sh[maxsh_offset + sh_pingpong_store * SLM_POOL_SIZE_MAX + idx * WorkGroupSize + wg_lane] = + (max_col_base[idx] < max_dim) ? data_max[max_base + flat_offset] : NEG_FLT_MAX_OVER_2; + } else { + F32Poolsh.F32sh[maxsh_offset + sh_pingpong_store * SLM_POOL_SIZE_MAX + idx * WorkGroupSize + wg_lane] = data_max[max_base + flat_offset]; + } + max_col_base[idx] = max_col_base[idx] + p_per_loop / p_reduce; + } + } + + barrier(); + + { + const uint coord_p = psh_base + sh_pingpong_load * psh_size + sg * TK; + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> VMat[omat_col]; + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + coopMatLoad(VMat[c], data_v, v_offset + TM * c, v_dim, gl_CooperativeMatrixLayoutColumnMajor); + } + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> PMat; + coopMatLoad(PMat, F16Poolsh.F16sh, coord_p + r * TN * p_per_loop, p_per_loop, gl_CooperativeMatrixLayoutColumnMajor); + + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + OMat[r][c] = coopMatMulAdd(VMat[c], PMat, OMat[r][c]); + } + } + } + + v_offset += p_per_loop * v_dim; + + if (seq_boundary_check || loop + 1 < loop_count) { + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint max_offset = (idx * WorkGroupSize + wg_lane) / maxsh_stride; + eMf[idx] = F32Poolsh.F32sh[maxsh_offset + sh_pingpong_store * SLM_POOL_SIZE_MAX + max_offset]; + float p_temp[4] = float[4](pv4_temp[idx].x, pv4_temp[idx].y, pv4_temp[idx].z, pv4_temp[idx].w); + float eMf_temp = exp(eMf[idx] - global_max[idx]); + [[unroll]] for (uint i = 0; i < 4; ++i) { + p_temp[i] = p_temp[i] * eMf_temp; + Lf[idx] = Lf[idx] + p_temp[i]; + F16Poolsh.F16sh[psh_offset + sh_pingpong_store * psh_size + idx * WorkGroupSize * 4 + i] = float16_t(p_temp[i]); + } + p_col_base[idx] = p_col_base[idx] + p_v4; + } + } } - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - [[unroll]] for (uint reduce = 1; reduce < SUB_GROUPS_PER_LINE; reduce++) { - sumTemp[pm][0] = sumTemp[pm][0] + sumTemp[pm][reduce]; - } + barrier(); + + // last loop for oob handling + if (seq_boundary_check) { + const uint coord_p = psh_base + sh_pingpong_store * psh_size + sg * TK; + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> VMat[omat_col]; + coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> PMat[omat_row]; + + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + coopMatLoad(PMat[r], F16Poolsh.F16sh, coord_p + r * TN * p_per_loop, p_per_loop, gl_CooperativeMatrixLayoutColumnMajor); + } + + barrier(); + const uint v_global_coord_base = sg * TK + loop_count * p_per_loop; + const uint v_global_offset = (v_base + loop_count * p_per_loop * p.nbv1) / 4; + const uint vv4sh_base = psh_base / 4 + sg * TM * TK / 4; + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + [[unroll]] for (uint idx = 0; idx < TM * TK / SubGroupSize / 4; ++idx) { + const uint vv4_idx = idx * SubGroupSize + lane; + const uint row_idx = vv4_idx / (TM / 4); + const uint col_idx = vv4_idx % (TM / 4); + const uint vv4_offset = v_global_offset + row_idx * p.nbv1 / 4 + c * TM / 4 + col_idx; + + f16vec4 vv4_temp = (v_global_coord_base + row_idx < p.nek1) ? data_vv4[vv4_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[vv4sh_base + vv4_idx] = vv4_temp; + } + + subgroupBarrier(); + coopMatLoad(VMat[c], F16V4Poolsh.F16V4sh, vv4sh_base, TM / 4, gl_CooperativeMatrixLayoutColumnMajor); + subgroupBarrier(); + } + + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + OMat[r][c] = coopMatMulAdd(VMat[c], PMat[r], OMat[r][c]); + } + } } - if ((0x1 & p.flag) != 0) { - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - float fp32GlobalMax = slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + pm * WARPSIZE + lane]; - float sinkCompensation = fp32GlobalMax - fp32SinkCoeff[pm]; - sinkCompensation = exp(sinkCompensation); - float softmaxSumTemp = sumTemp[pm][0] * sinkCompensation; - sumTemp[pm][0] = sumTemp[pm][0] + 1.0f / sinkCompensation; - sumTemp[pm][0] = 1.0f / sumTemp[pm][0]; - sinkCompensation = sinkCompensation / (1.0f + softmaxSumTemp); - sumTemp[pm][0] = fp32GlobalMax < fp32SinkCoeff[pm] ? sinkCompensation : sumTemp[pm][0]; - slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane] = sumTemp[pm][0]; - } - } else { - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane] = 1.0f / sumTemp[pm][0]; - } + barrier(); + + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + Lf[idx] = subgroupAdd(Lf[idx]); } - } - - [[unroll]] for (uint reduce = 2; reduce < SPLIT_P_GROUPS; reduce = reduce << 1 ) { - const uint stride = (reduce >> 1) * ALIGNED_P_M * TN * MAT_O_N * TM; - if ((localLinearId % reduce) == 0) { - const uint reduceBase = localLinearId * ALIGNED_P_M * TN * MAT_O_N * TM + SLM_OFFSET_O; - float sumTemp0[4]; - float sumTemp1[4]; - const uint reduceVec4Count = ALIGNED_P_M * TN * MAT_O_N * TM / 4 / WARPSIZE; - [[unroll]] for (uint totalLoads = 0; totalLoads < reduceVec4Count; totalLoads++) { - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - sumTemp0[kk] = slm_pool_f32.slm_pool_o[reduceBase + totalLoads * 4 * WARPSIZE + 4 * lane + kk]; - sumTemp1[kk] = slm_pool_f32.slm_pool_o[reduceBase + stride + totalLoads * 4 * WARPSIZE + 4 * lane + kk]; - } - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - sumTemp0[kk] = sumTemp0[kk] + sumTemp1[kk]; + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + const uint coord = osh_offset + r * TN * Bc + TM * c + sg * omat_row * TN * Bc; + coopMatStore(OMat[r][c], F32Poolsh.F32sh, coord, Bc, gl_CooperativeMatrixLayoutColumnMajor); } + } - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - slm_pool_f32.slm_pool_o[reduceBase + totalLoads * 4 * WARPSIZE + 4 * lane + kk] = sumTemp0[kk]; + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + if (lane == 0) { + F32Poolsh.F32sh[lfsh_offset + idx * sg_count + sg] = Lf[idx]; } - } } + barrier(); - } - if (localLinearId == 0) { - const uint slmBase0 = SLM_OFFSET_O + lane * WARP_V_DIM; - const uint slmBase1 = slmBase0 + SPLIT_P_GROUPS / 2 * ALIGNED_P_M * TN * MAT_O_N * TM; + if (sg == 1) { + const uint lf_base = lfsh_offset + lane * sg_per_line; + float Lfrcp_temp[o_scalar_row][sg_per_line]; + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + [[unroll]] for (uint reduce = 0; reduce < sg_per_line; ++reduce) { + Lfrcp_temp[r][reduce] = F32Poolsh.F32sh[lf_base + r * SubGroupSize * sg_per_line + reduce]; + } + } + + [[unroll]] for (uint pm = 0; pm < o_scalar_row; ++pm) { + [[unroll]] for (uint reduce = 1; reduce < sg_per_line; ++reduce) { + Lfrcp_temp[pm][0] = Lfrcp_temp[pm][0] + Lfrcp_temp[pm][reduce]; + } + } - const uint offsetOutBase = (d * p.batchStrideO + vWarpIdx * WARP_V_DIM + v * GQA_RATIO * HEAD_DIM + outTokIdx * oDim * N_TOKS_PER_GROUP) / 4; - float fp32SoftMaxMul[ALIGNED_P_M]; - float fp32Output[ALIGNED_P_M][4]; - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - fp32SoftMaxMul[pm] = slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane]; + subgroupBarrier(); + + if ((p.flag & SINK_ENABLE_BIT) != 0) { + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + float Mf = F32Poolsh.F32sh[global_maxsh_offset + r * SubGroupSize + lane]; + float sink_emf = Mf - sink[r]; + sink_emf = exp(sink_emf); + float Lf_sink = Lfrcp_temp[r][0] * sink_emf; + Lfrcp_temp[r][0] = Lfrcp_temp[r][0] + 1.0f / sink_emf; + Lfrcp_temp[r][0] = 1.0f / Lfrcp_temp[r][0]; + sink_emf = sink_emf / (1.0f + Lf_sink); + Lfrcp_temp[r][0] = (Mf < sink[r]) ? sink_emf : Lfrcp_temp[r][0]; + F32Poolsh.F32sh[lfsh_offset + r * SubGroupSize + lane] = Lfrcp_temp[r][0]; + } + } else { + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + F32Poolsh.F32sh[lfsh_offset + r * SubGroupSize + lane] = (Lfrcp_temp[r][0] == 0.0f) ? 0.0f : (1.0f / Lfrcp_temp[r][0]); + } + } } - [[unroll]] for (uint vg = 0; vg < WARP_V_DIM / 4; vg++) { - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - [[unroll]] for (uint vc = 0; vc < 4; vc++) { - fp32Output[pm][vc] = slm_pool_f32.slm_pool_o[slmBase0 + pm * WARPSIZE * WARP_V_DIM + vg * 4 + vc] * fp32SoftMaxMul[pm]; - fp32Output[pm][vc] = fp32Output[pm][vc] + slm_pool_f32.slm_pool_o[slmBase1 + pm * WARPSIZE * WARP_V_DIM + vg * 4 + vc] * fp32SoftMaxMul[pm]; + [[unroll]] for (uint reduce = 2; reduce < split_p_sg; reduce = reduce << 1) { + const uint stride = (reduce >> 1) * omat_row * TN * omat_col * TM; + if ((sg % reduce) == 0) { + const uint reduce_base = sg * omat_row * TN * omat_col * TM + osh_offset; + float sum_temp0[4]; + float sum_temp1[4]; + const uint reduce_count = omat_row * TN * omat_col * TM / 4 / SubGroupSize; + [[unroll]] for (uint idx = 0; idx < reduce_count; ++idx) { + [[unroll]] for (uint i = 0; i < 4; ++i) { + sum_temp0[i] = F32Poolsh.F32sh[reduce_base + idx * 4 * SubGroupSize + 4 * lane + i]; + sum_temp1[i] = F32Poolsh.F32sh[reduce_base + stride + idx * 4 * SubGroupSize + 4 * lane + i]; + } + + [[unroll]] for (uint i = 0; i < 4; ++i) { + sum_temp0[i] = sum_temp0[i] + sum_temp1[i]; + } + + [[unroll]] for (uint i = 0; i < 4; ++i) { + F32Poolsh.F32sh[reduce_base + idx * 4 * SubGroupSize + 4 * lane + i] = sum_temp0[i]; + } + } + } + barrier(); + } + + if (sg == 0) { + const uint sh_base0 = osh_offset + lane * Bc; + const uint sh_base1 = sh_base0 + split_p_sg / 2 * omat_row * TN * omat_col * TM; + + float Lfrcp[o_scalar_row]; + float Of[o_scalar_row][4]; + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + Lfrcp[r] = F32Poolsh.F32sh[lfsh_offset + r * SubGroupSize + lane]; } - } - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - if (outputMask[pm] == true) { - out_f32_vec4[offsetOutBase + outOffsets[pm] + vg] = vec4(fp32Output[pm][0], fp32Output[pm][1], fp32Output[pm][2], fp32Output[pm][3]); + [[unroll]] for (uint c = 0; c < Bc / 4; ++c) { + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + [[unroll]] for (uint vc = 0; vc < 4; ++vc) { + Of[r][vc] = F32Poolsh.F32sh[sh_base0 + r * SubGroupSize * Bc + c * 4 + vc] * Lfrcp[r]; + Of[r][vc] = Of[r][vc] + F32Poolsh.F32sh[sh_base1 + r * SubGroupSize * Bc + c * 4 + vc] * Lfrcp[r]; + } + } + + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + if (o_mask[r]) { + data_ov4[o_base + o_offset[r] + c] = vec4(Of[r][0], Of[r][1], Of[r][2], Of[r][3]); + } + } } - } } - } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_prefill.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_prefill.comp new file mode 100644 index 000000000..407654ca5 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_prefill.comp @@ -0,0 +1,817 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_KHR_cooperative_matrix : enable +#extension GL_KHR_memory_scope_semantics : enable +#extension GL_KHR_shader_subgroup_basic : enable +#extension GL_EXT_shared_memory_block : enable +#extension GL_KHR_shader_subgroup_shuffle : enable +#extension GL_KHR_shader_subgroup_shuffle_relative : enable + +layout (local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer Q {float data_q[];}; +layout (binding = 0) readonly buffer QM2x4 {mat2x4 data_qm2x4[];}; +layout (binding = 1) readonly buffer K {float16_t data_k[];}; +layout (binding = 1) readonly buffer KV4 {f16vec4 data_kv4[];}; +layout (binding = 1) readonly buffer KM2x4 {f16mat2x4 data_km2x4[];}; +layout (binding = 2) readonly buffer V {float16_t data_v[];}; +layout (binding = 2) readonly buffer VV4 {f16vec4 data_vv4[];}; +layout (binding = 3) readonly buffer M {float16_t data_m[];}; +layout (binding = 3) readonly buffer MV2 {f16vec2 data_mv2[];}; +layout (binding = 3) readonly buffer MV4 {f16vec4 data_mv4[];}; +layout (binding = 4) readonly buffer S {float data_s[];}; +layout (binding = 5) buffer OV4 {vec4 data_ov4[];}; + +layout (push_constant) uniform parameter +{ + uint neq1; + uint neq2; + uint neq3; + uint nek1; + uint nek2; + uint nek3; + uint nev2; + uint nev3; + uint nem1; + uint nem2; + uint nem3; + uint nbq1; + uint nbq2; + uint nbk1; + uint nbk2; + uint nbv1; + uint nbv2; + uint nbq3; + uint nbk3; + uint nbv3; + uint nb3; + uint flag; + float scale; +} p; + +layout (constant_id = 0) const uint WorkGroupSize = 256; +layout (constant_id = 1) const uint Wr = 128; +layout (constant_id = 2) const uint Bc = 64; +layout (constant_id = 3) const uint HSK = 128; +layout (constant_id = 4) const uint HSV = 128; +layout (constant_id = 5) const uint qk_ratio = 8; +layout (constant_id = 6) const uint step_p = 32; +layout (constant_id = 7) const uint TM = 8; +layout (constant_id = 8) const uint TN = 16; +layout (constant_id = 9) const uint q_reg = 96; +layout (constant_id = 10) const uint shmem_size = 64; +layout (constant_id = 11) const uint acc_count = 1; +layout (constant_id = 12) const uint Flags = 0; +layout (constant_id = 13) const uint SubGroupSize = 16; +layout (constant_id = 14) const uint gqa_fold = 1; + +#define UNALIGNED_SEQ_LEN_BIT (1 << 0) +#define UNALIGNED_Q_STRIDE_BIT (1 << 1) +#define PARTIAL_ROW_MASK_BIT (1 << 2) +#define INTEL_GPU_BIT (1 << 3) +#define SINK_ENABLE_BIT (1 << 16) + +const bool aligned_q_stride = (Flags & UNALIGNED_Q_STRIDE_BIT) == 0; +const bool seq_boundary_check = (Flags & UNALIGNED_SEQ_LEN_BIT) != 0; +const bool partial_row_mask = (Flags & PARTIAL_ROW_MASK_BIT) != 0; +const uint TK = 16; +const uint sg_count = WorkGroupSize / SubGroupSize; +const uint matp_count = step_p / TM; +const uint mato_count = Bc / TM; +const uint qk_sg = Wr / TN; +const uint folded_row_wg = Wr / gqa_fold; +const uint loadv_sg = (qk_sg == sg_count) ? sg_count : (sg_count - qk_sg); +const uint k_col = (HSK % 64 == 0 && HSK <= 512 && WorkGroupSize <= 256) ? 8 : 4; +const uint k_loop_col = WorkGroupSize * k_col; +const uint k_loops = (step_p * HSK + k_loop_col - 1) / k_loop_col; +const bool k_boundary_check = (step_p * HSK) % k_loop_col != 0; +const uint v_vert_groups = step_p / TK; +const uint v_hori_groups = loadv_sg / v_vert_groups; +const uint v_loops = (HSV / TM + v_hori_groups - 1) / v_hori_groups; +const bool v_boundary_check = HSV % (v_hori_groups * TM) != 0; +const uint m_vert_groups = folded_row_wg / SubGroupSize; +const uint m_hori_groups = sg_count / m_vert_groups; +const uint m_loops = (step_p + m_hori_groups - 1) / m_hori_groups; + +const uint matq_count = q_reg / TK; +const uint q_shmem = HSK - q_reg; +const uint matq_shmem_count = q_shmem / TK; +const uint max_q_reg_staging = shmem_size / 2 / Wr; + +const bool intel_gpu = (Flags & INTEL_GPU_BIT) != 0; +const bool k_in_shmem = HSK <= 576; +const bool v_in_shmem = HSK <= 128 && HSV <= 128; +const bool mask_in_shmem = folded_row_wg <= 64; + +const uint oobsh_staging_size = sg_count * TM * TK; +const uint qsh_size = Wr * q_shmem; +const uint ksh_size = k_in_shmem ? (k_loops * k_loop_col) : oobsh_staging_size; +const uint msh_size = mask_in_shmem ? (folded_row_wg * step_p) : 0; +const uint psh_size = Wr * step_p; +const uint vsh_size = v_in_shmem ? (step_p * HSV) : 0; +const uint emfsh_size = (qk_sg == sg_count) ? 0 : Wr; +const uint pvsh_size = intel_gpu ? 0 : (acc_count * TM * TN * sg_count); + +const uint qsh_base = 0; +const uint emfsh_base = (qsh_base + qsh_size) / 2; +const uint vsh_base = (emfsh_base + emfsh_size) * 2; +const uint psh_base = (vsh_base + vsh_size) / 2; +const uint ksh_base = (psh_base + psh_size) * 2; +const uint pvsh_base = ksh_base / 2; +const uint msh_base = ksh_base + ksh_size; +const uint vsh_oob_staging_base = ksh_base / 4; + +const uint sg_mat_reduce = SubGroupSize / TN; +const uint softmax_loops = step_p / sg_mat_reduce; +const uint acc_loops = acc_count * TM / sg_mat_reduce; +const uint CM1_ELEMS = (TM * TN) / SubGroupSize; +const uint tk_sg_reduce = (TK + SubGroupSize - 1) / SubGroupSize; +const uint o_sg_cols = (sg_count + qk_sg - 1) / qk_sg; +const bool v_row_overshoot_check = o_sg_cols * Bc > HSV; + +shared sh_block { + float F32sh[shmem_size / 4]; +} F32Poolsh; + +shared shf16_alias { + float16_t F16sh[shmem_size / 2]; +} F16Poolsh; + +shared shf16v2_alias { + f16vec2 F16V2sh[shmem_size / 4]; +} F16V2Poolsh; + +shared shf16v4_alias { + f16vec4 F16V4sh[shmem_size / 8]; +} F16V4Poolsh; + +shared shf16m2x4_alias { + f16mat2x4 M2x4sh[shmem_size / 16]; +} F16M2x4Poolsh; + +void main() { + const uint lane = gl_SubgroupInvocationID; + const uint lane_c = lane % TN; + const uint lane_r = lane / TN; + const uint pv_ratio = p.neq2 / p.nev2; + const uint head_q_idx = gl_WorkGroupID.x; + const uint head_k_idx = head_q_idx * gqa_fold / qk_ratio; + const uint head_v_idx = head_q_idx * gqa_fold / pv_ratio; + const uint batch_k_ratio = p.neq3 / p.nek3; + const uint batch_v_ratio = p.neq3 / p.nev3; + const uint head_mask_idx = head_q_idx % p.nem2; + const uint out_r = gl_WorkGroupID.y; + const uint batch_idx = gl_WorkGroupID.z; + const uint batch_k_idx = batch_idx / batch_k_ratio; + const uint batch_v_idx = batch_idx / batch_v_ratio; + const uint batch_mask_idx = batch_idx % p.nem3; + const uint sg = gl_SubgroupID; + const uint batch_mask_stride = p.nek1 * p.nem1 * p.nem2; + uint sg_v_r = sg / v_vert_groups; + const uint sg_v_c = sg % v_vert_groups; + if (qk_sg < sg_count && sg >= qk_sg) { + sg_v_r -= qk_sg / v_vert_groups; + } + const uint sg_m_c = sg % m_hori_groups; + const uint sg_m_r = sg / m_hori_groups; + const uint sg_o_c = sg / qk_sg; + const uint sg_o_r = sg % qk_sg; + const uint sg_o_r_h = sg_o_r % gqa_fold; + const uint sg_o_r_row = sg_o_r / gqa_fold; + const uint o_dim = p.neq2 * HSV; + uint q_offset = out_r * folded_row_wg + sg_o_r_row * TN + lane_c; + q_offset = min(q_offset, p.neq1 - 1); + if (aligned_q_stride) { + q_offset = (q_offset * p.nbq1 + batch_idx * p.nbq3 + (head_q_idx * gqa_fold + sg_o_r_h) * p.nbq2) / 8 + lane_r; + } else { + q_offset = (q_offset * p.nbq1 + batch_idx * p.nbq3 + (head_q_idx * gqa_fold + sg_o_r_h) * p.nbq2) + lane_r; + } + uint k_base; + if (k_in_shmem) { + k_base = (batch_k_idx * p.nbk3 + head_k_idx * p.nbk2) / k_col; + } else { + k_base = batch_k_idx * p.nbk3 + head_k_idx * p.nbk2; + } + uint k_offset[k_loops]; + const uint v_base = batch_v_idx * p.nbv3 + head_v_idx * p.nbv2; + uint v_offset; + if (v_in_shmem) { + v_offset = v_base + sg_v_r * TM + sg_v_c * p.nbv1 * TK; + } else { + v_offset = v_base + sg_o_c * Bc; + } + uint m_offset; + bool mask_in_boundary; + if (mask_in_shmem) { + m_offset = out_r * folded_row_wg + sg_m_r * SubGroupSize + lane; + mask_in_boundary = m_offset < p.nem1; + m_offset = min(m_offset, p.nem1 - 1); + m_offset = m_offset * p.nek1 + batch_mask_idx * batch_mask_stride + head_mask_idx * p.nek1 * p.nem1; + if (seq_boundary_check) { + m_offset += sg_m_c; + } else if (m_loops % 4 == 0) { + m_offset = m_offset / 4 + sg_m_c; + } else if (m_loops % 2 == 0) { + m_offset = m_offset / 2 + sg_m_c; + } else { + m_offset += sg_m_c; + } + } else { + m_offset = out_r * folded_row_wg + sg_o_r_row * TN + lane_c; + mask_in_boundary = m_offset < p.nem1; + m_offset = min(m_offset, p.nem1 - 1); + m_offset = m_offset * p.nek1 + batch_mask_idx * batch_mask_stride + lane_r * softmax_loops + head_mask_idx * p.nek1 * p.nem1; + if (!seq_boundary_check) { + m_offset /= 4; + } + } + const bool mask_row_valid = !partial_row_mask || mask_in_boundary; + const uint o_offset = (batch_idx * p.nb3 + out_r * folded_row_wg * o_dim + (head_q_idx * gqa_fold + sg_o_r_h) * HSV + + sg_o_r_row * TN * o_dim + sg_o_c * Bc + lane_c * o_dim) / 4 + lane_r; + const uint vsh_offset = sg_v_c * TK * HSV + sg_v_r * TM * TK + vsh_base; + const uint msh_store_offset = (sg_m_r * SubGroupSize + lane) * step_p + msh_base; + const uint msh_load_offset = sg_o_r_row * TN * step_p + lane_c * step_p + lane_r * softmax_loops + msh_base; + const uint psh_load_offset = psh_base + sg * TN * step_p + lane_c * step_p + lane_r * softmax_loops; + const uint psh_store_offset = (psh_base + sg * TN * step_p + lane_c * step_p) * 2 + lane_r * softmax_loops; + const float NEG_FLT_MAX_OVER_2 = uintBitsToFloat(0xFEFFFFFF); + const float16_t NEG_F16_MAX_OVER_2 = float16_t(NEG_FLT_MAX_OVER_2); + const float sink = ((p.flag & SINK_ENABLE_BIT) != 0) ? data_s[head_q_idx * gqa_fold + sg_o_r_h] : 0.0f; + coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> QMat[matq_count]; + uint ksh_offset[k_loops]; + bool mask_load_k[k_loops]; + + if (q_reg <= max_q_reg_staging) { + if (sg < qk_sg) { + if (aligned_q_stride) { + [[dont_unroll]] for (uint i = 0; i < q_reg / sg_mat_reduce / 8; ++i) { + const uint qsh_offset = (qsh_base + sg * q_reg * TN + lane_c * q_reg) / 8 + i * sg_mat_reduce + lane_r; + mat2x4 q_temp = data_qm2x4[q_offset]; + q_temp *= p.scale; + F16M2x4Poolsh.M2x4sh[qsh_offset] = f16mat2x4(q_temp); + q_offset += sg_mat_reduce; + } + } else { + [[dont_unroll]] for (uint i = 0; i < q_reg / sg_mat_reduce; ++i) { + const uint qsh_offset = (qsh_base + sg * q_reg * TN + lane_c * q_reg) + i * sg_mat_reduce + lane_r; + float q_temp = data_q[q_offset]; + q_temp *= p.scale; + F16Poolsh.F16sh[qsh_offset] = float16_t(q_temp); + q_offset += sg_mat_reduce; + } + } + } + + subgroupBarrier(); + + if (sg < qk_sg) { + const uint coord_q = sg * TN * q_reg + qsh_base; + [[unroll]] for (uint i = 0; i < matq_count; ++i) { + coopMatLoad(QMat[i], F16Poolsh.F16sh, coord_q + TK * i, q_reg, gl_CooperativeMatrixLayoutColumnMajor); + } + } + } else { + if (sg < qk_sg) { + const uint coord_q = sg * TN * TK + qsh_base; + [[unroll]] for (uint iq = 0; iq < q_reg / TK; ++iq) { + if (aligned_q_stride) { + [[dont_unroll]] for (uint idx = 0; idx < TK / sg_mat_reduce / 8; ++idx) { + const uint qsh_offset = (qsh_base + sg * TK * TN + lane_c * TK) / 8 + idx * sg_mat_reduce + lane_r; + mat2x4 q_temp = data_qm2x4[q_offset]; + q_temp *= p.scale; + F16M2x4Poolsh.M2x4sh[qsh_offset] = f16mat2x4(q_temp); + q_offset += sg_mat_reduce; + } + } else { + [[dont_unroll]] for (uint idx = 0; idx < TK / sg_mat_reduce; ++idx) { + const uint qsh_offset = (qsh_base + sg * TK * TN + lane_c * TK) + idx * sg_mat_reduce + lane_r; + float q_temp = data_q[q_offset]; + q_temp *= p.scale; + F16Poolsh.F16sh[qsh_offset] = float16_t(q_temp); + q_offset += sg_mat_reduce; + } + } + + subgroupBarrier(); + coopMatLoad(QMat[iq], F16Poolsh.F16sh, coord_q, TK, gl_CooperativeMatrixLayoutColumnMajor); + subgroupBarrier(); + } + } + } + + float Of[Bc / sg_mat_reduce]; + [[unroll]] for (uint i = 0; i < Bc / sg_mat_reduce; ++i) { + Of[i] = 0; + } + + barrier(); + + if (sg < qk_sg) { + if (aligned_q_stride) { + [[dont_unroll]] for (uint i = 0; i < q_shmem / sg_mat_reduce / 8; ++i) { + const uint qsh_offset = (qsh_base + sg * q_shmem * TN + lane_c * q_shmem) / 8 + i * sg_mat_reduce + lane_r; + mat2x4 q_temp = data_qm2x4[q_offset + i * sg_mat_reduce]; + q_temp *= p.scale; + F16M2x4Poolsh.M2x4sh[qsh_offset] = f16mat2x4(q_temp); + } + } else { + [[dont_unroll]] for (uint i = 0; i < q_shmem / sg_mat_reduce; ++i) { + const uint qsh_offset = (qsh_base + sg * q_shmem * TN + lane_c * q_shmem) + i * sg_mat_reduce + lane_r; + float q_temp = data_q[q_offset + i * sg_mat_reduce]; + q_temp *= p.scale; + F16Poolsh.F16sh[qsh_offset] = float16_t(q_temp); + } + } + } + + if (k_in_shmem) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + const uint flat_offset = idx * WorkGroupSize + sg * SubGroupSize + lane; + const uint row_idx = flat_offset / (HSK / k_col); + const uint col_idx = flat_offset % (HSK / k_col); + const uint sh_chunk_c = col_idx % (TK / k_col); + const uint sh_chunk_r = col_idx / (TK / k_col); + k_offset[idx] = k_base + row_idx * p.nbk1 / k_col + col_idx; + ksh_offset[idx] = (ksh_base + row_idx * TK + sh_chunk_r * TK * step_p) / k_col + sh_chunk_c; + if (k_boundary_check) { + mask_load_k[idx] = row_idx < step_p; + } + } + } + + const uint loop_count = (p.nek1 + step_p - 1) / step_p; + const uint k_seq_boundary = ((p.nek1 - 1) * p.nbk1 + batch_k_idx * p.nbk3 + head_k_idx * p.nbk2 + HSK) / k_col - 1; + float Mf = NEG_FLT_MAX_OVER_2; + float eMf = 0.0f; + float Lf = 0; + for (uint loop = 0; loop < loop_count; ++loop) { + if (k_in_shmem) { + if (k_boundary_check) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + if (seq_boundary_check && loop + 1 == loop_count) { + k_offset[idx] = min(k_offset[idx], k_seq_boundary); + } + + if (mask_load_k[idx]) { + if (k_col == 4) { + F16V4Poolsh.F16V4sh[ksh_offset[idx]] = data_kv4[k_offset[idx]]; + } else { + F16M2x4Poolsh.M2x4sh[ksh_offset[idx]] = data_km2x4[k_offset[idx]]; + } + } + k_offset[idx] += step_p * p.nbk1 / k_col; + } + } else { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + if (seq_boundary_check && loop + 1 == loop_count) { + k_offset[idx] = min(k_offset[idx], k_seq_boundary); + } + + if (k_col == 4) { + F16V4Poolsh.F16V4sh[ksh_offset[idx]] = data_kv4[k_offset[idx]]; + } else { + F16M2x4Poolsh.M2x4sh[ksh_offset[idx]] = data_km2x4[k_offset[idx]]; + } + k_offset[idx] += step_p * p.nbk1 / k_col; + } + } + } + + if (mask_in_shmem) { + if (seq_boundary_check) { + [[unroll]] for (uint idx = 0; idx < m_loops; ++idx) { + if (idx * m_hori_groups + loop * step_p + sg_m_c < p.nek1) { + F16Poolsh.F16sh[msh_store_offset + sg_m_c + idx * m_hori_groups] = mask_row_valid ? data_m[m_offset] : float16_t(0.0f); + m_offset += m_hori_groups; + } else { + F16Poolsh.F16sh[msh_store_offset + sg_m_c + idx * m_hori_groups] = NEG_F16_MAX_OVER_2; + } + } + } else if (m_loops % 4 == 0) { + [[unroll]] for (uint idx = 0; idx < m_loops / 4; ++idx) { + F16V4Poolsh.F16V4sh[msh_store_offset / 4 + sg_m_c + idx * m_hori_groups] = + mask_row_valid ? data_mv4[m_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + m_offset += m_hori_groups; + } + } else if (m_loops % 2 == 0) { + [[unroll]] for (uint idx = 0; idx < m_loops / 2; ++idx) { + F16V2Poolsh.F16V2sh[msh_store_offset / 2 + sg_m_c + idx * m_hori_groups] = mask_row_valid ? data_mv2[m_offset] : f16vec2(0.0f, 0.0f); + m_offset += m_hori_groups; + } + } else { + [[unroll]] for (uint idx = 0; idx < m_loops; ++idx) { + F16Poolsh.F16sh[msh_store_offset + sg_m_c + idx * m_hori_groups] = mask_row_valid ? data_m[m_offset] : float16_t(0.0f); + m_offset += m_hori_groups; + } + } + } + + barrier(); + if (qk_sg == sg_count || sg < qk_sg) { + { + coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> PMat[matp_count]; + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> KMat[matp_count]; + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + PMat[pc] = coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f); + } + + [[unroll]] for (uint d0 = 0; d0 < matq_count; ++d0) { + if (k_in_shmem) { + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + const uint coord = ksh_base + step_p * TK * d0 + TM * TK * pc; + coopMatLoad(KMat[pc], F16Poolsh.F16sh, coord, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + const uint k_base_row = loop * step_p + pc * TM; + if (seq_boundary_check && loop + 1 == loop_count) { + const uint k_base_offset = (k_base + k_base_row * p.nbk1 + TK * d0) / 4; + [[unroll]] for (uint idx = 0; idx < TM * TK / 4 / SubGroupSize; ++idx) { + const uint flat_offset = idx * SubGroupSize + lane; + const uint row_idx = flat_offset / (TK / 4); + const uint col_idx = flat_offset % (TK / 4); + const uint k_temp_offset = k_base_offset + row_idx * p.nbk1 / 4 + col_idx; + const uint ksh_temp_offset = ksh_base / 4 + sg * TM * TK / 4 + idx * SubGroupSize + lane; + f16vec4 kv4_temp = (k_base_row + row_idx) < p.nek1 ? data_kv4[k_temp_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[ksh_temp_offset] = kv4_temp; + } + + subgroupBarrier(); + coopMatLoad(KMat[pc], F16Poolsh.F16sh, ksh_base + sg * TM * TK, TK, gl_CooperativeMatrixLayoutRowMajor); + subgroupBarrier(); + } else { + const uint coord = k_base + k_base_row * p.nbk1 + TK * d0; + coopMatLoad(KMat[pc], data_k, coord, p.nbk1, gl_CooperativeMatrixLayoutRowMajor); + } + } + } + + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + PMat[pc] = coopMatMulAdd(KMat[pc], QMat[d0], PMat[pc]); + } + } + + [[unroll]] for (uint d0 = 0; d0 < matq_shmem_count; ++d0) { + coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> QMatsh; + if (k_in_shmem) { + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + const uint coord = ksh_base + matq_count * TK * step_p + step_p * TK * d0 + TM * TK * pc; + coopMatLoad(KMat[pc], F16Poolsh.F16sh, coord, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + const uint k_base_row = loop * step_p + pc * TM; + if (seq_boundary_check && loop + 1 == loop_count) { + const uint k_base_offset = (k_base + k_base_row * p.nbk1 + TK * d0 + TK * matq_count) / 4; + [[unroll]] for (uint idx = 0; idx < TM * TK / 4 / SubGroupSize; ++idx) { + const uint flat_offset = idx * SubGroupSize + lane; + const uint row_idx = flat_offset / (TK / 4); + const uint col_idx = flat_offset % (TK / 4); + const uint k_temp_offset = k_base_offset + row_idx * p.nbk1 / 4 + col_idx; + const uint ksh_temp_offset = ksh_base / 4 + sg * TM * TK / 4 + idx * SubGroupSize + lane; + f16vec4 kv4_temp = (k_base_row + row_idx) < p.nek1 ? data_kv4[k_temp_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[ksh_temp_offset] = kv4_temp; + } + + subgroupBarrier(); + coopMatLoad(KMat[pc], F16Poolsh.F16sh, ksh_base + sg * TM * TK, TK, gl_CooperativeMatrixLayoutRowMajor); + subgroupBarrier(); + } else { + const uint coord = k_base + k_base_row * p.nbk1 + TK * d0 + TK * matq_count; + coopMatLoad(KMat[pc], data_k, coord, p.nbk1, gl_CooperativeMatrixLayoutRowMajor); + } + } + } + + const uint coord = qsh_base + sg * TN * q_shmem + TK * d0; + coopMatLoad(QMatsh, F16Poolsh.F16sh, coord, q_shmem, gl_CooperativeMatrixLayoutColumnMajor); + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + PMat[pc] = coopMatMulAdd(KMat[pc], QMatsh, PMat[pc]); + } + } + + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + coopMatStore(PMat[pc], F32Poolsh.F32sh, psh_base + TM * pc + sg * TN * step_p, step_p, gl_CooperativeMatrixLayoutColumnMajor); + } + } + + subgroupBarrier(); + + float Pf[softmax_loops]; + float Mnewf = Mf; + + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + Pf[idx] = F32Poolsh.F32sh[psh_load_offset + idx]; + } + + if (mask_in_shmem) { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + Pf[idx] += float(F16Poolsh.F16sh[msh_load_offset + idx]); + } + } else { + if (seq_boundary_check) { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + if (loop * step_p + idx + lane_r * softmax_loops < p.nek1) { + Pf[idx] += mask_row_valid ? float(data_m[m_offset + idx]) : 0.0f; + } else { + Pf[idx] += NEG_FLT_MAX_OVER_2; + } + } + m_offset += step_p; + } else { + [[unroll]] for (uint idx = 0; idx < softmax_loops / 4; ++idx) { + f16vec4 mv4_temp = mask_row_valid ? data_mv4[m_offset + idx] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + float mask_f_temp[4] = float[4](float(mv4_temp.x), float(mv4_temp.y), float(mv4_temp.z), float(mv4_temp.w)); + [[unroll]] for (uint vec_idx = 0; vec_idx < 4; ++vec_idx) { + Pf[4 * idx + vec_idx] += mask_f_temp[vec_idx]; + } + } + m_offset += step_p / 4; + } + } + + if (seq_boundary_check && loop + 1 == loop_count) { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + if (loop * step_p + idx + lane_r * softmax_loops < p.nek1) { + Mnewf = max(Mnewf, Pf[idx]); + } + } + } else { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + Mnewf = max(Mnewf, Pf[idx]); + } + } + + [[unroll]] for (uint idx = 1; idx < sg_mat_reduce; ++idx) { + Mnewf = max(Mnewf, subgroupShuffleDown(Mnewf, idx * TN)); + } + + Mnewf = subgroupShuffle(Mnewf, lane & (TN - 1)); + eMf = exp(Mf - Mnewf); + Lf *= eMf; + + if (seq_boundary_check && loop + 1 == loop_count) { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + if (loop * step_p + idx + lane_r * softmax_loops < p.nek1) { + Pf[idx] = exp(Pf[idx] - Mnewf); + } else { + Pf[idx] = 0.0f; + } + Lf += Pf[idx]; + } + } else { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + Pf[idx] = exp(Pf[idx] - Mnewf); + Lf += Pf[idx]; + } + } + + if (qk_sg != sg_count && lane_r == 0) { + F32Poolsh.F32sh[emfsh_base + sg * TN + lane_c] = eMf; + } + + if (SubGroupSize > TN) { + subgroupBarrier(); + } + + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + F16Poolsh.F16sh[psh_store_offset + idx] = float16_t(Pf[idx]); + } + Mf = Mnewf; + } + + if (v_in_shmem && (qk_sg == sg_count || sg >= sg_count - loadv_sg)) { + if (seq_boundary_check && loop + 1 == loop_count) { + const uint v_global_coord_base = loop * step_p + sg_v_c * TK + lane_c; + if (v_boundary_check) { + [[dont_unroll]] for (uint c = 0; c < tk_sg_reduce; ++c) { + const uint v_coord_temp = v_global_coord_base + c * SubGroupSize; + uint v_oob_offset = (v_base + v_coord_temp * p.nbv1 + sg_v_r * TM) / 4 + lane_r; + uint vsh_temp_offset = vsh_offset + c * SubGroupSize + lane_c + lane_r * 4 * TK; + [[dont_unroll]] for (uint r = 0; r < v_loops - 1; ++r) { + [[dont_unroll]] for (uint idx = 0; idx < TM / sg_mat_reduce / 4; ++idx) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + idx * sg_mat_reduce] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 0 * TK] = vv4_temp.x; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 1 * TK] = vv4_temp.y; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 2 * TK] = vv4_temp.z; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 3 * TK] = vv4_temp.w; + } + v_oob_offset += TM * v_hori_groups / 4; + vsh_temp_offset += TM * TK * v_hori_groups; + } + } + + if ((sg_v_r + (v_loops - 1) * v_hori_groups) * TM < HSV) { + [[dont_unroll]] for (uint c = 0; c < tk_sg_reduce; ++c) { + const uint v_coord_temp = v_global_coord_base + c * SubGroupSize; + const uint v_oob_offset = (v_base + v_coord_temp * p.nbv1 + sg_v_r * TM + TM * (v_loops - 1) * v_hori_groups) / 4 + lane_r; + const uint vsh_temp_offset = vsh_offset + TM * (v_loops - 1) * TK * v_hori_groups + c * SubGroupSize + lane_c + lane_r * 4 * TK; + [[dont_unroll]] for (uint idx = 0; idx < TM / sg_mat_reduce / 4; ++idx) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + idx * sg_mat_reduce] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 0 * TK] = vv4_temp.x; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 1 * TK] = vv4_temp.y; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 2 * TK] = vv4_temp.z; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 3 * TK] = vv4_temp.w; + } + } + } + } else { + [[dont_unroll]] for (uint c = 0; c < tk_sg_reduce; ++c) { + const uint v_coord_temp = v_global_coord_base + c * SubGroupSize; + uint v_oob_offset = (v_base + v_coord_temp * p.nbv1 + sg_v_r * TM) / 4 + lane_r; + uint vsh_temp_offset = vsh_offset + c * SubGroupSize + lane_c + lane_r * 4 * TK; + [[dont_unroll]] for (uint r = 0; r < v_loops; ++r) { + [[dont_unroll]] for (uint idx = 0; idx < TM / sg_mat_reduce / 4; ++idx) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + idx * sg_mat_reduce] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 0 * TK] = vv4_temp.x; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 1 * TK] = vv4_temp.y; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 2 * TK] = vv4_temp.z; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 3 * TK] = vv4_temp.w; + } + v_oob_offset += TM * v_hori_groups / 4; + vsh_temp_offset += TM * TK * v_hori_groups; + } + } + } + } else { + if (v_boundary_check) { + [[unroll]] for (uint r = 0; r < v_loops - 1; ++r) { + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> VMat; + coopMatLoad(VMat, data_v, v_offset + TM * r * v_hori_groups, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + coopMatStore(VMat, F16Poolsh.F16sh, vsh_offset + TM * r * TK * v_hori_groups, TK, gl_CooperativeMatrixLayoutRowMajor); + } + + if ((sg_v_r + (v_loops - 1) * v_hori_groups) * TM < HSV) { + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> VMat; + coopMatLoad(VMat, data_v, v_offset + TM * (v_loops - 1) * v_hori_groups, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + coopMatStore(VMat, F16Poolsh.F16sh, vsh_offset + TM * (v_loops - 1) * TK * v_hori_groups, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + [[unroll]] for (uint r = 0; r < v_loops; ++r) { + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> VMat; + coopMatLoad(VMat, data_v, v_offset + TM * r * v_hori_groups, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + coopMatStore(VMat, F16Poolsh.F16sh, vsh_offset + TM * r * TK * v_hori_groups, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } + } + } + + barrier(); + { + const uint coord_p = psh_base * 2 + sg_o_r * TN * step_p * 2; + const uint coord_v_base = vsh_base + sg_o_c * mato_count * TM * TK; + const uint coord_pv = pvsh_base + sg * acc_count * TN * TM; + const uint pvsh_offset = pvsh_base + sg * acc_count * TN * TM + lane_c * acc_count * TM + lane_r * 4; + if (qk_sg != sg_count) { + eMf = F32Poolsh.F32sh[emfsh_base + sg_o_r * TN + lane_c]; + } + coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> PMat[step_p / TK]; + [[unroll]] for (uint d0 = 0; d0 < step_p / TK; ++d0) { + coopMatLoad(PMat[d0], F16Poolsh.F16sh, coord_p + TK * d0, 2 * step_p, gl_CooperativeMatrixLayoutColumnMajor); + } + + [[unroll]] for (uint pv_col = 0; pv_col < mato_count / acc_count; ++pv_col) { + const uint coord_v = coord_v_base + pv_col * acc_count * TM * TK; + coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> PVMat[acc_count]; + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> VMat[acc_count]; + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + PVMat[idx] = coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f); + } + + [[unroll]] for (uint d0 = 0; d0 < step_p / TK; ++d0) { + if (v_in_shmem) { + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + coopMatLoad(VMat[idx], F16Poolsh.F16sh, coord_v + HSV * TK * d0 + TM * TK * idx, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + if (seq_boundary_check && loop + 1 == loop_count) { + const uint v_global_coord_base = d0 * TK + loop * step_p + lane_c; + const uint vsh_staging_store = vsh_oob_staging_base + (sg * TM * TK + lane_c * TM) / 4 + lane_r; + const uint vsh_staging_load = vsh_oob_staging_base * 4 + sg * TM * TK; + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + subgroupBarrier(); + [[dont_unroll]] for (uint c = 0; c < tk_sg_reduce; ++c) { + const uint v_coord_temp = v_global_coord_base + c * SubGroupSize; + const uint v_oob_offset = (v_base + v_coord_temp * p.nbv1 + sg_o_c * Bc + pv_col * acc_count * TM + idx * TM) / 4 + lane_r; + + if (v_row_overshoot_check) { + const uint v_row_temp = sg_o_c * Bc + pv_col * acc_count * TM + idx * TM; + if (v_row_temp < HSV) { + [[dont_unroll]] for (uint r = 0; r < TM / sg_mat_reduce / 4; ++r) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + sg_mat_reduce * r] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[vsh_staging_store + c * SubGroupSize * TM / 4 + sg_mat_reduce * r] = vv4_temp; + } + } else { + [[dont_unroll]] for (uint r = 0; r < TM / sg_mat_reduce / 4; ++r) { + F16V4Poolsh.F16V4sh[vsh_staging_store + c * SubGroupSize * TM / 4 + sg_mat_reduce * r] = f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + } + } + } else { + [[dont_unroll]] for (uint r = 0; r < TM / sg_mat_reduce / 4; ++r) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + sg_mat_reduce * r] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[vsh_staging_store + c * SubGroupSize * TM / 4 + sg_mat_reduce * r] = vv4_temp; + } + } + } + subgroupBarrier(); + coopMatLoad(VMat[idx], F16Poolsh.F16sh, vsh_staging_load, TM, gl_CooperativeMatrixLayoutColumnMajor); + } + } else { + const uint v_coord_global = v_offset + pv_col * acc_count * TM + d0 * TK * p.nbv1; + if (v_row_overshoot_check) { + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + const uint v_row_temp = sg_o_c * Bc + pv_col * acc_count * TM + idx * TM; + if (v_row_temp < HSV) { + coopMatLoad(VMat[idx], data_v, v_coord_global + TM * idx, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + } else { + VMat[idx] = coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA>(0.0f); + } + } + } else { + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + coopMatLoad(VMat[idx], data_v, v_coord_global + TM * idx, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + } + } + } + } + + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + PVMat[idx] = coopMatMulAdd(VMat[idx], PMat[d0], PVMat[idx]); + } + } + + if (intel_gpu) { + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + [[unroll]] for (uint e = 0; e < CM1_ELEMS; ++e) { + Of[pv_col * acc_loops + idx * CM1_ELEMS + e] = fma(Of[pv_col * acc_loops + idx * CM1_ELEMS + e], + eMf, + PVMat[idx][e]); + } + } + } else { + barrier(); + + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + coopMatStore(PVMat[idx], F32Poolsh.F32sh, coord_pv + TM * idx, acc_count * TM, gl_CooperativeMatrixLayoutColumnMajor); + } + + subgroupBarrier(); + + [[unroll]] for (uint idx = 0; idx < acc_loops / 4; ++idx) { + [[unroll]] for (uint vec_idx = 0; vec_idx < 4; ++vec_idx) { + Of[pv_col * acc_loops + idx * 4 + vec_idx] = fma(Of[pv_col * acc_loops + idx * 4 + vec_idx], + eMf, + F32Poolsh.F32sh[pvsh_offset + idx * sg_mat_reduce * 4 + vec_idx]); + } + } + barrier(); + } + } + } + + v_offset += p.nbv1 * step_p; + barrier(); + } + + if (sg < qk_sg) { + [[unroll]] for (uint idx = 1; idx < sg_mat_reduce; ++idx) { + Lf += subgroupShuffleDown(Lf, idx * TN); + } + if (lane_r == 0) { + F32Poolsh.F32sh[sg * TN + lane_c] = Mf; + F32Poolsh.F32sh[qk_sg * TN + sg * TN + lane_c] = Lf; + } + } + + barrier(); + + Mf = F32Poolsh.F32sh[sg_o_r * TN + lane_c]; + Lf = F32Poolsh.F32sh[qk_sg * TN + sg_o_r * TN + lane_c]; + + float Lfrcp; + if ((p.flag & SINK_ENABLE_BIT) != 0) { + float sink_emf = Mf - sink; + sink_emf = exp(sink_emf); + float Lf_sink = Lf * sink_emf; + Lf += 1.0f / sink_emf; + Lf = 1.0f / Lf; + sink_emf /= (1.0f + Lf_sink); + Lfrcp = (Mf < sink) ? sink_emf : Lf; + } else { + Lfrcp = (Lf == 0.0f) ? 0.0f : (1.0f / Lf); + } + + [[unroll]] for (uint out_c = 0; out_c < Bc / sg_mat_reduce; ++out_c) { + Of[out_c] *= Lfrcp; + } + + const uint sg_out_r = out_r * folded_row_wg + sg_o_r_row * TN + lane_c; + [[unroll]] for (uint o_chunk = 0; o_chunk < Bc / sg_mat_reduce / 4; ++o_chunk) { + const uint out_c = sg_mat_reduce * 4 * o_chunk + sg_o_c * Bc + lane_r; + if (sg_out_r < p.neq1 && out_c < HSV) { + data_ov4[o_offset + o_chunk * sg_mat_reduce] = vec4(Of[4 * o_chunk], Of[4 * o_chunk + 1], Of[4 * o_chunk + 2], Of[4 * o_chunk + 3]); + } + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp b/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp index a2069964a..d279d4306 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp @@ -5,11 +5,13 @@ #extension GL_KHR_shader_subgroup_basic : enable #extension GL_KHR_shader_subgroup_shuffle : enable #endif +#extension GL_EXT_shader_16bit_storage : require layout(constant_id = 0) const uint BLOCK_SIZE = 32; layout(constant_id = 1) const uint N = 128; +layout(constant_id = 2) const uint ROWS = 4; -layout(local_size_x_id = 0, local_size_y = 4, local_size_z = 1) in; +layout(local_size_x_id = 0, local_size_y_id = 2, local_size_z = 1) in; layout(push_constant) uniform parameter { @@ -19,13 +21,13 @@ layout(push_constant) uniform parameter float scale; }; -layout(binding = 0, std430) readonly buffer A { float data_a[]; }; +layout(binding = 0, std430) readonly buffer A { A_TYPE data_a[]; }; layout(binding = 1, std430) writeonly buffer D { float data_d[]; }; const uint EL_W = N / BLOCK_SIZE; #ifdef FWHT_SHMEM -shared float shmem[4 * N]; +shared float shmem[ROWS * N]; #endif void main() { @@ -54,7 +56,7 @@ void main() { [[unroll]] for (uint i = 0; i < EL_W; ++i) { - reg[i] = row < n_rows ? data_a[src_offset + row_offset + i * BLOCK_SIZE + tid] * scale : 0.0; + reg[i] = row < n_rows ? float(data_a[src_offset + row_offset + i * BLOCK_SIZE + tid]) * scale : 0.0; } #ifdef FWHT_SHMEM diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp index bdf97dbb5..334321e7b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp @@ -18,8 +18,12 @@ void main() { const float eps = p.param1; const uint tid = gl_LocalInvocationID.x; - const uint start = gl_WorkGroupID.x * group_size + tid; - const uint end = (gl_WorkGroupID.x + 1) * group_size; + const uint group = gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.z * gl_NumWorkGroups.x * gl_NumWorkGroups.y; + if (group >= p.KY) { + return; + } + const uint start = group * group_size + tid; + const uint end = (group + 1) * group_size; tmp[tid] = 0.0f; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp index 9039ed1de..4ac040de9 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp @@ -14,6 +14,8 @@ void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; const uint tid = gl_LocalInvocationID.x; + if (row >= p.ne01 * p.ne02 * p.ne03) { return; } + const uint a_base = get_aoffset() + src0_idx(row * p.ne00); const uint d_base = get_doffset() + dst_idx(row * p.ne10); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl index 4aeda68c7..12d9cbf6b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl @@ -42,6 +42,7 @@ layout (push_constant) uniform parameter #ifdef MUL_MAT_ID uint expert_id; +bool expert_skip; // an id of -1 skips the slot, the result row is zeroed #endif void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) { @@ -63,7 +64,9 @@ void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) { batch_idx_a = i03 * p.ne02 + i02; } #else - expert_id = data_ids[expert_i0 + p.expert_i1 * p.nbi1]; + const int expert = data_ids[expert_i0 + p.expert_i1 * p.nbi1]; + expert_skip = expert == -1; + expert_id = expert_skip ? 0 : uint(expert); // clamp, so that the offsets below stay in range #endif a_offset = @@ -113,6 +116,9 @@ void reduce_result(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t const uint expert_i0 = gl_GlobalInvocationID.y; temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]); } + if (expert_skip) { + temp[j][n] = FLOAT_TYPE(0); + } #else if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) { temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]); @@ -168,6 +174,9 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs const uint expert_i0 = gl_GlobalInvocationID.y; temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]); } + if (expert_skip) { + temp[j][n] = FLOAT_TYPE(0); + } #else if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) { temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]); @@ -213,6 +222,9 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs const uint expert_i0 = gl_GlobalInvocationID.y; tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse1[expert_i0]); } + if (expert_skip) { + tmpsh[j][n][0] = FLOAT_TYPE(0); + } #else if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) { tmpsh[j][n][0] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 11098ee7b..19d37870b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -124,7 +124,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; #ifdef MUL_MAT_ID layout (binding = 3) readonly buffer IDS {int data_ids[];}; -layout (binding = 4) readonly buffer Counts {int data_expert_count[];}; +layout (binding = 4) readonly buffer RowMap {uint data_row_map[];}; #endif layout (push_constant) uniform parameter @@ -146,7 +146,7 @@ layout (push_constant) uniform parameter uint nbi1; uint ne11; uint n_experts; - uint hoist_row_ids; + uint row_ids_offset; #else uint base_work_group_z; uint num_batches; @@ -208,13 +208,17 @@ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS]; #endif void main() { - const uint ic = gl_WorkGroupID.y; - #ifdef MUL_MAT_ID - const uint expert_idx = gl_WorkGroupID.z; - if (ic * BN >= data_expert_count[expert_idx]) { + const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y; + if (tile_idx >= data_row_map[0]) { return; } + const uint expert_idx = data_row_map[1 + 2 * tile_idx]; + const uint row_begin = tile_idx * BN; + _ne1 = data_row_map[2 + 2 * tile_idx]; + const uint ic = 0; +#else + const uint ic = gl_WorkGroupID.y; #endif #if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT) init_iq_shmem(gl_WorkGroupSize); @@ -285,37 +289,9 @@ void main() { const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK; #ifdef MUL_MAT_ID - if (p.hoist_row_ids != 0) { - load_row_ids_hoisted(expert_idx, ic); - } else { -#ifdef MUL_MAT_ID_USE_SUBGROUPS - if (bitCount(p.nei0) == 1) { - load_row_ids(expert_idx, true, ic); - } else { - load_row_ids(expert_idx, false, ic); - } -#else - _ne1 = 0; - for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) { - for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) { - if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) { - if (_ne1 >= ic * BN) { - row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1); - } - _ne1++; - } - } - } - - barrier(); -#endif - } - - // Workgroup has no work - if (ic * BN >= _ne1) return; - - uint required_work_items = (_ne1 - ic * BN) * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B; - uint required_warp_c = (_ne1 - ic * BN + WN - 1) / WN; + load_row_ids(row_begin); + uint required_work_items = _ne1 * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B; + uint required_warp_c = (_ne1 + WN - 1) / WN; #endif #ifdef MUL_MAT_ID @@ -431,8 +407,7 @@ void main() { [[unroll]] for (uint cr = 0; cr < TM / 2; cr++) { // [WNITER][TN][WMITER][TM / 2] -> [wsic][cc][wsir][cr] const uint sums_idx = (wsic * TN + cc) * WMITER * (TM / 2) + wsir * (TM / 2) + cr; - sums[sums_idx].x = dot_product(cache_a[wsir * TM + 2 * cr ], cache_b, sums[sums_idx].x); - sums[sums_idx].y = dot_product(cache_a[wsir * TM + 2 * cr + 1], cache_b, sums[sums_idx].y); + sums[sums_idx] = dot_product_v2(cache_a[wsir * TM + 2 * cr], cache_a[wsir * TM + 2 * cr + 1], cache_b, sums[sums_idx]); } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 9b59e8bca..10dda7768 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -45,21 +45,17 @@ const uint BNover2 = BN / 2; const uint BNover4 = enable_smaller_matrices ? (BN / 4) : (BN / 2); layout (constant_id = 5) const uint ALIGNED = 0; layout (constant_id = 6) const uint subgroup_size = 32; +#define GROUP_A_BUDGET (8u * 1024u * 1024u) #ifdef MULMAT_QUANT uint mm_quant_k() { - switch (MmTypeA) { - case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - return 32u; - case GGML_TYPE_Q1_0: - return 128u; - case GGML_TYPE_Q2_0: - return 64u; - default: - return 256u; - } + // Use ternary expressions instead of a switch to keep qk as a specialization-constant expression. + return (MmTypeA == GGML_TYPE_Q1_0) ? 128u : + (MmTypeA == GGML_TYPE_Q2_0) ? 64u : + ((MmTypeA == GGML_TYPE_Q4_0 || MmTypeA == GGML_TYPE_Q4_1 || + MmTypeA == GGML_TYPE_Q5_0 || MmTypeA == GGML_TYPE_Q5_1 || + MmTypeA == GGML_TYPE_Q8_0) ? 32u : 256u); } #endif @@ -82,7 +78,7 @@ layout (push_constant) uniform parameter uint nbi1; uint ne11; uint n_experts; - uint hoist_row_ids; + uint row_ids_offset; #else uint base_work_group_z; uint num_batches; @@ -182,7 +178,7 @@ f16vec4 mmDecodeA_v(const in decodeBufA bl_in, const in uint blockCoords[2], con #ifdef MUL_MAT_ID layout (binding = 3) readonly buffer IDS {int data_ids[];}; -layout (binding = 4) readonly buffer Counts {int data_expert_count[];}; +layout (binding = 4) readonly buffer RowMap {uint data_row_map[];}; shared u16vec4 row_ids[BN]; @@ -191,7 +187,6 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB { }; uint _ne1; -shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size]; B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2]) { @@ -238,82 +233,9 @@ D_TYPE perElemOpD(const in uint32_t r, const in uint32_t c, const in D_TYPE elem return elem; } -void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) { - _ne1 = 0; - uint num_elements = p.nei1 * p.nei0; - uint nei0shift = findLSB(p.nei0); - - uint ids[16]; - uint iter = 0; - - uint expert_count = data_expert_count[expert_idx]; - - for (uint j = 0; j < num_elements; j += BLOCK_SIZE) { - // prefetch up to 16 elements - if (iter == 0) { - [[unroll]] for (uint k = 0; k < 16; ++k) { - uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE; - bool in_range = i < num_elements; - uint ii1; - if (nei0_is_pow2) { - ii1 = i >> nei0shift; - } else { - ii1 = i / p.nei0; - } - uint ii0 = i - ii1 * p.nei0; - ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0; - } - } - uint i = j + gl_LocalInvocationIndex; - bool in_range = i < num_elements; - uint ii1; - if (nei0_is_pow2) { - ii1 = i >> nei0shift; - } else { - ii1 = i / p.nei0; - } - uint ii0 = i - ii1 * p.nei0; - uint id = ids[iter++]; - uvec4 ballot = subgroupBallot(in_range && id == expert_idx); - - if (gl_SubgroupInvocationID == 0) { - ballots_sh[gl_SubgroupID] = ballot; - } - barrier(); - - uint subgroup_base = 0; - uint total = 0; - for (uint k = 0; k < gl_NumSubgroups; ++k) { - if (k == gl_SubgroupID) { - subgroup_base = total; - } - total += subgroupBallotBitCount(ballots_sh[k]); - } - barrier(); - - uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot); - if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) { - row_ids[_ne1 + idx - ic * BN] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0); - } - _ne1 += total; - iter &= 15; - if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) { - break; - } - } - barrier(); -} - -void load_row_ids_hoisted(uint expert_idx, uint ic) { - _ne1 = uint(data_expert_count[expert_idx]); - - const uint tile_begin = ic * BN; - const uint tile_count = tile_begin < _ne1 ? min(BN, _ne1 - tile_begin) : 0; - const uint expert_offset = uint(data_expert_count[p.n_experts + expert_idx]); - const uint row_ids_offset = 2 * p.n_experts + 1 + expert_offset + tile_begin; - - for (uint i = gl_LocalInvocationIndex; i < tile_count; i += BLOCK_SIZE) { - const uint packed_row_id = uint(data_expert_count[row_ids_offset + i]); +void load_row_ids(uint row_begin) { + for (uint i = gl_LocalInvocationIndex; i < _ne1; i += BLOCK_SIZE) { + const uint packed_row_id = data_row_map[p.row_ids_offset + row_begin + i]; const uint ii0 = packed_row_id & 0xffffu; const uint ii1 = packed_row_id >> 16; row_ids[i] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0); @@ -328,13 +250,18 @@ void load_row_ids_hoisted(uint expert_idx, uint ic) { void main() { const uint tid = gl_LocalInvocationIndex; - const uint ic = gl_WorkGroupID.y; - #ifdef MUL_MAT_ID - const uint expert_idx = gl_WorkGroupID.z; - if (ic * BN >= data_expert_count[expert_idx]) { + const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y; + if (tile_idx >= data_row_map[0]) { return; } + const uint expert_idx = data_row_map[1 + 2 * tile_idx]; + const uint row_begin = tile_idx * BN; + _ne1 = data_row_map[2 + 2 * tile_idx]; + const uint ic = 0; +#endif + +#ifdef MUL_MAT_ID // initialize to row 0 so we don't need to bounds check if (tid < BN) { row_ids[tid] = u16vec4(0); @@ -361,20 +288,35 @@ void main() { #endif const uint blocks_m = (p.M + BM - 1) / BM; - const uint ir = gl_WorkGroupID.x % blocks_m; const uint ik = gl_WorkGroupID.x / blocks_m; #ifdef MUL_MAT_ID - if (p.hoist_row_ids != 0) { - load_row_ids_hoisted(expert_idx, ic); - } else if (bitCount(p.nei0) == 1) { - load_row_ids(expert_idx, true, ic); - } else { - load_row_ids(expert_idx, false, ic); - } + const uint ir = gl_WorkGroupID.x % blocks_m; +#else + // L2-friendly workgroup scheduling + const uint blocks_n = (p.N + BN - 1) / BN; +#if defined(MULMAT_QUANT) + const uint a_panel_bytes = (BM * p.K / mm_quant_k()) * MmABlockBytes; +#elif defined(DATA_A_F32) + const uint a_panel_bytes = BM * p.K * 4; +#elif QUANT_K == 1 + const uint a_panel_bytes = BM * p.K * 2; +#else + const uint a_panel_bytes = (BM * p.K) / 2 + (BM * p.K) / 16; +#endif + const uint group_m = clamp(GROUP_A_BUDGET / max(a_panel_bytes, 1u), 1u, min(blocks_m, 32u)); + const uint tiles_per_group = group_m * blocks_n; + const uint lin = gl_WorkGroupID.y * blocks_m + (gl_WorkGroupID.x % blocks_m); + const uint group_id = lin / tiles_per_group; + const uint first_m = group_id * group_m; + const uint gsize = min(blocks_m - first_m, group_m); + const uint in_group = lin - group_id * tiles_per_group; + const uint ir = first_m + in_group % gsize; + const uint ic = in_group / gsize; +#endif - // Workgroup has no work - if (ic * BN >= _ne1) return; +#ifdef MUL_MAT_ID + load_row_ids(row_begin); #endif #ifdef MUL_MAT_ID diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl index 54ad60b2e..99a644e5a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl @@ -2,86 +2,9 @@ shared u16vec2 row_ids[BN]; uint _ne1; -#ifdef MUL_MAT_ID_USE_SUBGROUPS -shared uvec4 ballots_sh[NUM_WARPS]; - -void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) { - _ne1 = 0; - uint num_elements = p.nei1 * p.nei0; - uint nei0shift = findLSB(p.nei0); - - uint ids[16]; - uint iter = 0; - - uint expert_count = data_expert_count[expert_idx]; - - for (uint j = 0; j < num_elements; j += BLOCK_SIZE) { - // prefetch up to 16 elements - if (iter == 0) { - [[unroll]] for (uint k = 0; k < 16; ++k) { - uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE; - bool in_range = i < num_elements; - uint ii1; - if (nei0_is_pow2) { - ii1 = i >> nei0shift; - } else { - ii1 = i / p.nei0; - } - uint ii0 = i - ii1 * p.nei0; - ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0; - } - } - uint i = j + gl_LocalInvocationIndex; - bool in_range = i < num_elements; - uint ii1; - if (nei0_is_pow2) { - ii1 = i >> nei0shift; - } else { - ii1 = i / p.nei0; - } - uint ii0 = i - ii1 * p.nei0; - uint id = ids[iter++]; - uvec4 ballot = subgroupBallot(in_range && id == expert_idx); - - if (gl_SubgroupInvocationID == 0) { - ballots_sh[gl_SubgroupID] = ballot; - } - barrier(); - - uint subgroup_base = 0; - uint total = 0; - for (uint k = 0; k < gl_NumSubgroups; ++k) { - if (k == gl_SubgroupID) { - subgroup_base = total; - } - total += subgroupBallotBitCount(ballots_sh[k]); - } - barrier(); - - uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot); - if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) { - row_ids[_ne1 + idx - ic * BN] = u16vec2(ii0, ii1); - } - _ne1 += total; - iter &= 15; - if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) { - break; - } - } - barrier(); -} -#endif // MUL_MAT_ID_USE_SUBGROUPS - -void load_row_ids_hoisted(uint expert_idx, uint ic) { - _ne1 = uint(data_expert_count[expert_idx]); - - const uint tile_begin = ic * BN; - const uint tile_count = tile_begin < _ne1 ? min(BN, _ne1 - tile_begin) : 0; - const uint expert_offset = uint(data_expert_count[p.n_experts + expert_idx]); - const uint row_ids_offset = 2 * p.n_experts + 1 + expert_offset + tile_begin; - - for (uint i = gl_LocalInvocationIndex; i < tile_count; i += BLOCK_SIZE) { - const uint packed_row_id = uint(data_expert_count[row_ids_offset + i]); +void load_row_ids(uint row_begin) { + for (uint i = gl_LocalInvocationIndex; i < _ne1; i += BLOCK_SIZE) { + const uint packed_row_id = data_row_map[p.row_ids_offset + row_begin + i]; row_ids[i] = u16vec2(packed_row_id & 0xffffu, packed_row_id >> 16); } barrier(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp index 67908e704..1866386a7 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp @@ -37,7 +37,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; #ifdef MUL_MAT_ID layout (binding = 3) readonly buffer IDS {int data_ids[];}; -layout (binding = 4) readonly buffer Counts {int data_expert_count[];}; +layout (binding = 4) readonly buffer RowMap {uint data_row_map[];}; #endif layout (push_constant) uniform parameter @@ -59,7 +59,7 @@ layout (push_constant) uniform parameter uint nbi1; uint ne11; uint n_experts; - uint hoist_row_ids; + uint row_ids_offset; #else uint base_work_group_z; uint num_batches; @@ -105,19 +105,22 @@ block_b_cache cache_b; #define LOAD_VEC_A (4 * QUANT_R_MMQ) #define LOAD_VEC_B 16 -#define NUM_WARPS (BLOCK_SIZE / WARP) #include "mul_mm_id_funcs.glsl" #include "mul_mmq_funcs.glsl" void main() { - const uint ic = gl_WorkGroupID.y; - #ifdef MUL_MAT_ID - const uint expert_idx = gl_WorkGroupID.z; - if (ic * BN >= data_expert_count[expert_idx]) { + const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y; + if (tile_idx >= data_row_map[0]) { return; } + const uint expert_idx = data_row_map[1 + 2 * tile_idx]; + const uint row_begin = tile_idx * BN; + _ne1 = data_row_map[2 + 2 * tile_idx]; + const uint ic = 0; +#else + const uint ic = gl_WorkGroupID.y; #endif #ifdef NEEDS_INIT_IQ_SHMEM init_iq_shmem(gl_WorkGroupSize); @@ -161,34 +164,7 @@ void main() { const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK; #ifdef MUL_MAT_ID - if (p.hoist_row_ids != 0) { - load_row_ids_hoisted(expert_idx, ic); - } else { -#ifdef MUL_MAT_ID_USE_SUBGROUPS - if (bitCount(p.nei0) == 1) { - load_row_ids(expert_idx, true, ic); - } else { - load_row_ids(expert_idx, false, ic); - } -#else - _ne1 = 0; - for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) { - for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) { - if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) { - if (_ne1 >= ic * BN) { - row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1); - } - _ne1++; - } - } - } - - barrier(); -#endif - } - - // Workgroup has no work - if (ic * BN >= _ne1) return; + load_row_ids(row_begin); #endif #ifdef MUL_MAT_ID @@ -225,9 +201,7 @@ void main() { const uint iqs = loadr_a; [[unroll]] for (uint k_step = 0; k_step < BK_STEP; k_step++) { - if (block + k_step * BK < end_k) { - block_a_to_shmem(k_step * BM + buf_ib, ib + k_step, iqs); - } + block_a_to_shmem(k_step * BM + buf_ib, ib + k_step, iqs, block + k_step * BK < end_k); } } [[unroll]] for (uint l = 0; loadc_b + l < BN; l += loadstride_b) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp index 7cab9a119..eaf7eb65d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp @@ -39,7 +39,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; #ifdef MUL_MAT_ID layout (binding = 3) readonly buffer IDS {int data_ids[];}; -layout (binding = 4) readonly buffer Counts {int data_expert_count[];}; +layout (binding = 4) readonly buffer RowMap {uint data_row_map[];}; #endif layout (push_constant) uniform parameter @@ -61,7 +61,7 @@ layout (push_constant) uniform parameter uint nbi1; uint ne11; uint n_experts; - uint hoist_row_ids; + uint row_ids_offset; #else uint base_work_group_z; uint num_batches; @@ -142,13 +142,22 @@ ACC_TYPE cm1_accumulate(ACC_TYPE prev, int acc_e, float scale_a, float nbias_a, } #ifdef MUL_MAT_ID -#define NUM_WARPS (BLOCK_SIZE / WARP) #include "mul_mm_id_funcs.glsl" #endif #include "mul_mmq_cm1_funcs.glsl" void main() { +#ifdef MUL_MAT_ID + const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y; + if (tile_idx >= data_row_map[0]) { + return; + } + const uint expert_idx = data_row_map[1 + 2 * tile_idx]; + const uint row_begin = tile_idx * BN; + _ne1 = data_row_map[2 + 2 * tile_idx]; + const uint ic = 0; +#endif #if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) if (gl_LocalInvocationIndex < 16u) { cm1_kvalues[gl_LocalInvocationIndex] = kvalues_iq4nl_const[gl_LocalInvocationIndex]; @@ -164,7 +173,7 @@ void main() { cm1_kvalues[gl_LocalInvocationIndex] = kvalues_mxfp4_const[gl_LocalInvocationIndex]; } #if !defined(USE_OCP_FP4) - for (uint i = gl_LocalInvocationIndex; i < 128u; i += BLOCK_SIZE) { + for (uint i = gl_LocalInvocationIndex; i < 256u; i += BLOCK_SIZE) { ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i); } #endif @@ -175,12 +184,7 @@ void main() { const uint ik = gl_WorkGroupID.x / blocks_m; #ifdef MUL_MAT_ID - const uint ic = gl_WorkGroupID.y; const uint ir = gl_WorkGroupID.x % blocks_m; - const uint expert_idx = gl_WorkGroupID.z; - if (ic * BN >= data_expert_count[expert_idx]) { - return; - } #else // L2-friendly workgroup scheduling const uint blocks_n = (p.N + BN - 1) / BN; @@ -231,33 +235,7 @@ void main() { const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK; #ifdef MUL_MAT_ID - if (p.hoist_row_ids != 0) { - load_row_ids_hoisted(expert_idx, ic); - } else { -#ifdef MUL_MAT_ID_USE_SUBGROUPS - if (bitCount(p.nei0) == 1) { - load_row_ids(expert_idx, true, ic); - } else { - load_row_ids(expert_idx, false, ic); - } -#else - _ne1 = 0; - for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) { - for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) { - if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) { - if (_ne1 >= ic * BN) { - row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1); - } - _ne1++; - } - } - } - - barrier(); -#endif - } - - if (ic * BN >= _ne1) return; + load_row_ids(row_begin); #endif #ifdef MUL_MAT_ID diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1_funcs.glsl index 1760c138e..50c61f6cf 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1_funcs.glsl @@ -19,7 +19,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { uint32_t lo4 = blk.qs & 0x0F0F0F0F; uint32_t hi4 = (blk.qs >> 4) & 0x0F0F0F0F; lo4 = ((lo4 | 0x80808080) - 0x08080808) ^ 0x80808080; @@ -28,7 +28,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr + 4] = hi4; if (loadr == 0) { - buf_a_d[ks * BM + buf_ib] = float(blk.d); + buf_a_d[ks * BM + buf_ib] = in_bounds ? float(blk.d) : 0.0f; } } @@ -46,7 +46,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { // Store raw unsigned nibbles; the -8 offset is absorbed by the min term. uint32_t lo4 = blk.qs & 0x0F0F0F0F; uint32_t hi4 = (blk.qs >> 4) & 0x0F0F0F0F; @@ -54,7 +54,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr + 4] = hi4; if (loadr == 0) { - buf_a_dm[ks * BM + buf_ib] = vec2(float(blk.dm.x), float(blk.dm.y)); + buf_a_dm[ks * BM + buf_ib] = in_bounds ? vec2(float(blk.dm.x), float(blk.dm.y)) : vec2(0.0f); } } @@ -75,7 +75,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { uint32_t lo4 = blk.qs & 0x0F0F0F0F; uint32_t hi4 = (blk.qs >> 4) & 0x0F0F0F0F; lo4 |= ((blk.qh >> (4u * loadr )) & 0xFu) * 0x02040810u & 0x10101010u; @@ -86,7 +86,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr + 4] = hi4; if (loadr == 0) { - buf_a_d[ks * BM + buf_ib] = float(blk.d); + buf_a_d[ks * BM + buf_ib] = in_bounds ? float(blk.d) : 0.0f; } } @@ -106,7 +106,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { // Store raw unsigned 5-bit values; the -16 offset is absorbed by the min term. uint32_t lo4 = blk.qs & 0x0F0F0F0F; uint32_t hi4 = (blk.qs >> 4) & 0x0F0F0F0F; @@ -116,7 +116,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr + 4] = hi4; if (loadr == 0) { - buf_a_dm[ks * BM + buf_ib] = vec2(float(blk.dm.x), float(blk.dm.y)); + buf_a_dm[ks * BM + buf_ib] = in_bounds ? vec2(float(blk.dm.x), float(blk.dm.y)) : vec2(0.0f); } } @@ -135,11 +135,11 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr] = blk.qs; if (loadr == 0) { - buf_a_d[ks * BM + buf_ib] = float(blk.d); + buf_a_d[ks * BM + buf_ib] = in_bounds ? float(blk.d) : 0.0f; } } @@ -158,7 +158,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { const u8vec4 lo_idx = unpack8(blk.qs & 0x0F0F0F0F); const u8vec4 hi_idx = unpack8((blk.qs >> 4) & 0x0F0F0F0F); buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr ] = @@ -169,7 +169,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { cm1_kvalues[hi_idx.z], cm1_kvalues[hi_idx.w])); if (loadr == 0) { - buf_a_d[ks * BM + buf_ib] = float(blk.d); + buf_a_d[ks * BM + buf_ib] = in_bounds ? float(blk.d) : 0.0f; } } @@ -194,7 +194,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { const u8vec4 lo_idx = unpack8(blk.qs & 0x0F0F0F0F); const u8vec4 hi_idx = unpack8((blk.qs >> 4) & 0x0F0F0F0F); buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr ] = @@ -205,7 +205,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { cm1_kvalues[hi_idx.z], cm1_kvalues[hi_idx.w])); if (loadr == 0) { - buf_a_d[ks * BM + buf_ib] = blk.d; + buf_a_d[ks * BM + buf_ib] = in_bounds ? blk.d : 0.0f; } } @@ -226,7 +226,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { const u8vec4 lo_idx = unpack8(blk.qs & 0x0F0F0F0F); const u8vec4 hi_idx = unpack8((blk.qs >> 4) & 0x0F0F0F0F); buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr ] = @@ -237,7 +237,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { cm1_kvalues[hi_idx.z], cm1_kvalues[hi_idx.w])); if (loadr == 0) { - buf_a_d[ks * BM + buf_ib] = e8m0_to_fp32(blk.e) * 0.5; + buf_a_d[ks * BM + buf_ib] = in_bounds ? e8m0_to_fp32(blk.e) * 0.5 : 0.0f; } } @@ -271,7 +271,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { // Store raw unsigned nibbles (blk.qs already masked); no -8 recentering needed. buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr * 2 ] = blk.qs0; buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr * 2 + 1] = blk.qs1; @@ -287,7 +287,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { const uint mn_val = (sub < 4) ? (s_j4 & 0x3Fu) : ((s_j8 >> 4) | ((s_j4 >> 6) << 4)); vec2 dm = vec2(data_a_packed32[ib_k].dm); float d_scaled = dm.x * float(sc_val); - buf_a_dm[ks * BM + buf_ib] = vec2(d_scaled, -(dm.y * float(mn_val))); + buf_a_dm[ks * BM + buf_ib] = in_bounds ? vec2(d_scaled, -(dm.y * float(mn_val))) : vec2(0.0f); } } @@ -323,7 +323,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { // Store raw unsigned 5-bit values (qs nibble | qh bit); no -16 recentering needed. uint32_t v0 = blk.qs0 | blk.qh0; uint32_t v1 = blk.qs1 | blk.qh1; @@ -341,7 +341,7 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { const uint mn_val = (sub < 4) ? (s_j4 & 0x3Fu) : ((s_j8 >> 4) | ((s_j4 >> 6) << 4)); vec2 dm = vec2(data_a_packed32[ib_k].dm); float d_scaled = dm.x * float(sc_val); - buf_a_dm[ks * BM + buf_ib] = vec2(d_scaled, -(dm.y * float(mn_val))); + buf_a_dm[ks * BM + buf_ib] = in_bounds ? vec2(d_scaled, -(dm.y * float(mn_val))) : vec2(0.0f); } } @@ -388,7 +388,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { uint32_t v0 = ((blk.qs0 | 0x80808080) - 0x20202020) ^ 0x80808080; uint32_t v1 = ((blk.qs1 | 0x80808080) - 0x20202020) ^ 0x80808080; buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr * 2 ] = v0; @@ -398,8 +398,8 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { const uint ib_k = blk.ib / 8; const uint sub = blk.ib % 8; i8vec2 sc = unpack8(int32_t(int16_t(data_a_packed16[ib_k].scales[sub]))).xy; - buf_a_d[(ks * KSCALES ) * BM + buf_ib] = float(data_a_packed16[ib_k].d) * float(sc.x); - buf_a_d[(ks * KSCALES + 1) * BM + buf_ib] = float(data_a_packed16[ib_k].d) * float(sc.y); + buf_a_d[(ks * KSCALES ) * BM + buf_ib] = in_bounds ? float(data_a_packed16[ib_k].d) * float(sc.x) : 0.0f; + buf_a_d[(ks * KSCALES + 1) * BM + buf_ib] = in_bounds ? float(data_a_packed16[ib_k].d) * float(sc.y) : 0.0f; } } @@ -439,7 +439,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { uint32_t v0 = ((blk.qs0 | 0x80808080) - 0x04040404) ^ 0x80808080; uint32_t v1 = ((blk.qs1 | 0x80808080) - 0x04040404) ^ 0x80808080; buf_a_qs[buf_ib * QPITCH + ks * (BK / 4) + loadr * 2 ] = v0; @@ -456,8 +456,8 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { uint combined = lo | (hi << 4); i8vec2 sc = unpack8(int32_t(combined)).xy; float d = float(data_a_packed16[ib_k].d); - buf_a_d[(ks * KSCALES ) * BM + buf_ib] = d * float(int(sc.x) - 32); - buf_a_d[(ks * KSCALES + 1) * BM + buf_ib] = d * float(int(sc.y) - 32); + buf_a_d[(ks * KSCALES ) * BM + buf_ib] = in_bounds ? d * float(int(sc.x) - 32) : 0.0f; + buf_a_d[(ks * KSCALES + 1) * BM + buf_ib] = in_bounds ? d * float(int(sc.y) - 32) : 0.0f; } } @@ -486,7 +486,7 @@ block_a_prefetch block_a_load(uint ib, uint loadr) { return blk; } -void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { +void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr, bool in_bounds) { const u8vec4 lo_idx = unpack8(blk.qs & 0x0F0F0F0F); const u8vec4 hi_idx = unpack8((blk.qs >> 4) & 0x0F0F0F0F); const uint sub_base = (loadr >> 1) * 4; @@ -499,8 +499,8 @@ void block_a_to_shmem(block_a_prefetch blk, uint buf_ib, uint ks, uint loadr) { cm1_kvalues[hi_idx.z], cm1_kvalues[hi_idx.w])); if (loadr == 0) { - buf_a_d[(ks * KSCALES ) * BM + buf_ib] = ue4m3_to_fp32(blk.d0) * 0.5; - buf_a_d[(ks * KSCALES + 1) * BM + buf_ib] = ue4m3_to_fp32(blk.d1) * 0.5; + buf_a_d[(ks * KSCALES ) * BM + buf_ib] = in_bounds ? ue4m3_to_fp32(blk.d0) * 0.5 : 0.0f; + buf_a_d[(ks * KSCALES + 1) * BM + buf_ib] = in_bounds ? ue4m3_to_fp32(blk.d1) * 0.5 : 0.0f; } } @@ -559,7 +559,9 @@ void block_b_to_shmem(block_b_prefetch blk, uint buf_ib, uint ks, uint loadr, bo if (buf_ib < BM) { \ const uint ib = pos_a_ib + buf_ib * p.stride_a / BK; \ [[unroll]] for (uint ks = 0; ks < BK_STEP; ks++) { \ - pre_a[li * BK_STEP + ks] = block_a_load(ib + ks, loadr_a); \ + /* a k step past end_k re-reads an in-bounds block; its scale is zeroed below */ \ + const uint ib_k = ((blk) + ks * BK < end_k) ? (ib + ks) : ib; \ + pre_a[li * BK_STEP + ks] = block_a_load(ib_k, loadr_a); \ } \ } \ } \ @@ -579,7 +581,8 @@ void block_b_to_shmem(block_b_prefetch blk, uint buf_ib, uint ks, uint loadr, bo const uint buf_ib = loadc_a + li * loadstride_a; \ if (buf_ib < BM) { \ [[unroll]] for (uint ks = 0; ks < BK_STEP; ks++) { \ - block_a_to_shmem(pre_a[li * BK_STEP + ks], buf_ib, ks, loadr_a); \ + const bool in_bounds = (blk) + ks * BK < end_k; \ + block_a_to_shmem(pre_a[li * BK_STEP + ks], buf_ib, ks, loadr_a, in_bounds); \ } \ } \ } \ diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl index 136d5b745..d9f2f5748 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl @@ -7,18 +7,26 @@ // Each iqs value maps to a 32-bit integer #if defined(DATA_A_Q2_0) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - const uint block_idx = ib / 2; - const uint byte_idx = (ib & 1u) * 8u + iqs; - const uint bits = uint(data_a[block_idx].qs[byte_idx]); - buf_a[buf_ib].qs[iqs] = pack32(i8vec4( - int8_t(bits & 3u), - int8_t((bits >> 2u) & 3u), - int8_t((bits >> 4u) & 3u), - int8_t(bits >> 6u))); +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + const uint block_idx = ib / 2; + const uint byte_idx = (ib & 1u) * 8u + iqs; + const uint bits = uint(data_a[block_idx].qs[byte_idx]); + buf_a[buf_ib].qs[iqs] = pack32(i8vec4( + int8_t(bits & 3u), + int8_t((bits >> 2u) & 3u), + int8_t((bits >> 4u) & 3u), + int8_t(bits >> 6u))); + + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(data_a[block_idx].d); + } + } else { + buf_a[buf_ib].qs[iqs] = 0; - if (iqs == 0) { - buf_a[buf_ib].dm = FLOAT_TYPE(data_a[block_idx].d); + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(0.0f); + } } } @@ -43,21 +51,37 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_Q4_0) || defined(DATA_A_Q4_1) // 2-byte loads for Q4_0 blocks (18 bytes) // 4-byte loads for Q4_1 blocks (20 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + #ifdef DATA_A_Q4_0 + buf_a[buf_ib].qs[iqs] = pack32(u16vec2(data_a_packed16[ib].qs[iqs * 2], + data_a_packed16[ib].qs[iqs * 2 + 1])); + + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d); + } + #else // DATA_A_Q4_1 + buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs]; + + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm); + } + #endif + } else { #ifdef DATA_A_Q4_0 - buf_a[buf_ib].qs[iqs] = pack32(u16vec2(data_a_packed16[ib].qs[iqs * 2], - data_a_packed16[ib].qs[iqs * 2 + 1])); + buf_a[buf_ib].qs[iqs] = 0; - if (iqs == 0) { - buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d); - } + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(0.0f); + } #else // DATA_A_Q4_1 - buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs]; + buf_a[buf_ib].qs[iqs] = 0; - if (iqs == 0) { - buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm); - } + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPEV2(0.0f); + } #endif + } } void block_a_to_registers(const uint reg_ib, const uint buf_ib) { @@ -93,23 +117,41 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_Q5_0) || defined(DATA_A_Q5_1) // 2-byte loads for Q5_0 blocks (22 bytes) // 4-byte loads for Q5_1 blocks (24 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + #ifdef DATA_A_Q5_0 + buf_a[buf_ib].qs[iqs] = pack32(u16vec2(data_a_packed16[ib].qs[iqs * 2], + data_a_packed16[ib].qs[iqs * 2 + 1])); + + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d); + buf_a[buf_ib].qh = pack32(u16vec2(data_a_packed16[ib].qh[0], data_a_packed16[ib].qh[1])); + } + #else // DATA_A_Q5_1 + buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs]; + + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm); + buf_a[buf_ib].qh = data_a_packed32[ib].qh; + } + #endif + } else { #ifdef DATA_A_Q5_0 - buf_a[buf_ib].qs[iqs] = pack32(u16vec2(data_a_packed16[ib].qs[iqs * 2], - data_a_packed16[ib].qs[iqs * 2 + 1])); + buf_a[buf_ib].qs[iqs] = 0; - if (iqs == 0) { - buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d); - buf_a[buf_ib].qh = pack32(u16vec2(data_a_packed16[ib].qh[0], data_a_packed16[ib].qh[1])); - } + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(0.0f); + buf_a[buf_ib].qh = 0; + } #else // DATA_A_Q5_1 - buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs]; + buf_a[buf_ib].qs[iqs] = 0; - if (iqs == 0) { - buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm); - buf_a[buf_ib].qh = data_a_packed32[ib].qh; - } + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPEV2(0.0f); + buf_a[buf_ib].qh = 0; + } #endif + } } void block_a_to_registers(const uint reg_ib, const uint buf_ib) { @@ -148,12 +190,20 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_Q8_0) // 2-byte loads for Q8_0 blocks (34 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2], - data_a_packed16[ib].qs[iqs * 2 + 1])); +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2], + data_a_packed16[ib].qs[iqs * 2 + 1])); + + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d); + } + } else { + buf_a[buf_ib].qs[iqs] = 0; - if (iqs == 0) { - buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d); + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(0.0f); + } } } @@ -180,20 +230,29 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_MXFP4) // 1-byte loads for mxfp4 blocks (17 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - const uint32_t qs = pack32(u8vec4(data_a[ib].qs[iqs * 4 ], - data_a[ib].qs[iqs * 4 + 1], - data_a[ib].qs[iqs * 4 + 2], - data_a[ib].qs[iqs * 4 + 3])); +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + const uint32_t qs = pack32(u8vec4(data_a[ib].qs[iqs * 4 ], + data_a[ib].qs[iqs * 4 + 1], + data_a[ib].qs[iqs * 4 + 2], + data_a[ib].qs[iqs * 4 + 3])); - const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F); - const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F); + const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F); + const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F); - buf_a[buf_ib].qs[iqs ] = pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w])); - buf_a[buf_ib].qs[iqs + 4] = pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w])); + buf_a[buf_ib].qs[iqs ] = pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w])); + buf_a[buf_ib].qs[iqs + 4] = pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w])); + + if (iqs == 0) { + buf_a[buf_ib].d = FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e) * 0.5); + } + } else { + buf_a[buf_ib].qs[iqs] = 0; + buf_a[buf_ib].qs[iqs + 4] = 0; - if (iqs == 0) { - buf_a[buf_ib].d = FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e) * 0.5); + if (iqs == 0) { + buf_a[buf_ib].d = FLOAT_TYPE(0.0f); + } } } @@ -218,19 +277,28 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #endif #if defined(DATA_A_IQ4_XS) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - const uint ib_k = ib / 8; - const uint ib32 = ib % 8; - const uint32_t vui = data_a_packed32[ib_k].qs[4 * ib32 + iqs]; - const i32vec2 qs = iq4nl_to_i8x8(vui); +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + const uint ib_k = ib / 8; + const uint ib32 = ib % 8; + const uint32_t vui = data_a_packed32[ib_k].qs[4 * ib32 + iqs]; + const i32vec2 qs = iq4nl_to_i8x8(vui); + + buf_a[buf_ib].qs[iqs ] = qs.x; + buf_a[buf_ib].qs[iqs + 4] = qs.y; - buf_a[buf_ib].qs[iqs ] = qs.x; - buf_a[buf_ib].qs[iqs + 4] = qs.y; + if (iqs == 0) { + const uint sl = (data_a_packed32[ib_k].scales_l >> (4 * ib32)) & 0xF; + const uint sh = (data_a_packed32[ib_k].scales_h >> (2 * ib32)) & 3; + buf_a[buf_ib].d = FLOAT_TYPE(float(data_a[ib_k].d) * float(int(sl | (sh << 4)) - 32)); + } + } else { + buf_a[buf_ib].qs[iqs] = 0; + buf_a[buf_ib].qs[iqs + 4] = 0; - if (iqs == 0) { - const uint sl = (data_a_packed32[ib_k].scales_l >> (4 * ib32)) & 0xF; - const uint sh = (data_a_packed32[ib_k].scales_h >> (2 * ib32)) & 3; - buf_a[buf_ib].d = FLOAT_TYPE(float(data_a[ib_k].d) * float(int(sl | (sh << 4)) - 32)); + if (iqs == 0) { + buf_a[buf_ib].d = FLOAT_TYPE(0.0f); + } } } @@ -256,24 +324,33 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { // iqs still refers to a 32-bit integer, meaning 0..7 for 32-wide quants #if defined(DATA_A_Q2_K) // 4-byte loads for Q2_K blocks (84 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - const uint ib_k = ib / 8; - const uint iqs_k = (ib % 8) * 8 + iqs * QUANT_R_MMQ; +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + const uint ib_k = ib / 8; + const uint iqs_k = (ib % 8) * 8 + iqs * QUANT_R_MMQ; + + const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8); + const uint qs_shift = ((iqs_k % 32) / 8) * 2; - const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8); - const uint qs_shift = ((iqs_k % 32) / 8) * 2; + // Repack 4x4 quants into one int + const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x03030303; + const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x03030303; + const uint32_t vals2 = (data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x03030303; + const uint32_t vals3 = (data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x03030303; - // Repack 4x4 quants into one int - const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x03030303; - const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x03030303; - const uint32_t vals2 = (data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x03030303; - const uint32_t vals3 = (data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x03030303; + buf_a[buf_ib].qs[iqs] = vals0 | (vals1 << 2) | (vals2 << 4) | (vals3 << 6); - buf_a[buf_ib].qs[iqs] = vals0 | (vals1 << 2) | (vals2 << 4) | (vals3 << 6); + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib_k].dm); + buf_a[buf_ib].scales = unpack8(uint32_t(data_a_packed16[ib_k].scales[iqs_k / 8])).xy; // vec4 used due to #12147 + } + } else { + buf_a[buf_ib].qs[iqs] = 0; - if (iqs == 0) { - buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib_k].dm); - buf_a[buf_ib].scales = unpack8(uint32_t(data_a_packed16[ib_k].scales[iqs_k / 8])).xy; // vec4 used due to #12147 + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPEV2(0.0f); + buf_a[buf_ib].scales = u8vec2(0); + } } } @@ -305,35 +382,43 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_Q3_K) // 2-byte loads for Q3_K blocks (110 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - const uint ib_k = ib / 8; - const uint hm_idx = iqs * QUANT_R_MMQ; - const uint iqs_k = (ib % 8) * 8 + hm_idx; - - const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8); - const uint qs_shift = ((iqs_k % 32) / 8) * 2; - const uint hm_shift = iqs_k / 8; - - // Repack 2x4 quants into one int - // Add the 3rd bit instead of subtracting it to allow packing the quants - // vec4 for unpack8 used due to #12147 - const i8vec2 vals00 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 ] >> qs_shift) & uint16_t(0x0303)))).xy | - unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 ] >> hm_shift) & uint16_t(0x0101))) << 2)).xy; - const i8vec2 vals01 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 1 ] >> qs_shift) & uint16_t(0x0303)))).xy | - unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 1] >> hm_shift) & uint16_t(0x0101))) << 2)).xy; - const i8vec2 vals10 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 2 ] >> qs_shift) & uint16_t(0x0303)))).xy | - unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 2] >> hm_shift) & uint16_t(0x0101))) << 2)).xy; - const i8vec2 vals11 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 3 ] >> qs_shift) & uint16_t(0x0303)))).xy | - unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 3] >> hm_shift) & uint16_t(0x0101))) << 2)).xy; - buf_a[buf_ib].qs[iqs] = pack32(u8vec4(vals00.x, vals00.y, vals01.x, vals01.y)) | - (pack32(u8vec4(vals10.x, vals10.y, vals11.x, vals11.y)) << 4); - - if (iqs == 0) { - const uint is = iqs_k / 4; - const i8vec2 scales = i8vec2(unpack8(uint32_t(((data_a_packed16[ib_k].scales[(is % 8 ) / 2] >> (4 * (is / 8))) & 0x0F0F) | - (((data_a_packed16[ib_k].scales[(8 + (is % 4)) / 2] >> (2 * (is / 4))) & 0x0303) << 4))).xy); // vec4 used due to #12147 - - buf_a[buf_ib].d_scales = FLOAT_TYPEV2(float(data_a_packed16[ib_k].d) * vec2(scales - 32)); +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + const uint ib_k = ib / 8; + const uint hm_idx = iqs * QUANT_R_MMQ; + const uint iqs_k = (ib % 8) * 8 + hm_idx; + + const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8); + const uint qs_shift = ((iqs_k % 32) / 8) * 2; + const uint hm_shift = iqs_k / 8; + + // Repack 2x4 quants into one int + // Add the 3rd bit instead of subtracting it to allow packing the quants + // vec4 for unpack8 used due to #12147 + const i8vec2 vals00 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 ] >> qs_shift) & uint16_t(0x0303)))).xy | + unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 ] >> hm_shift) & uint16_t(0x0101))) << 2)).xy; + const i8vec2 vals01 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 1 ] >> qs_shift) & uint16_t(0x0303)))).xy | + unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 1] >> hm_shift) & uint16_t(0x0101))) << 2)).xy; + const i8vec2 vals10 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 2 ] >> qs_shift) & uint16_t(0x0303)))).xy | + unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 2] >> hm_shift) & uint16_t(0x0101))) << 2)).xy; + const i8vec2 vals11 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 3 ] >> qs_shift) & uint16_t(0x0303)))).xy | + unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 3] >> hm_shift) & uint16_t(0x0101))) << 2)).xy; + buf_a[buf_ib].qs[iqs] = pack32(u8vec4(vals00.x, vals00.y, vals01.x, vals01.y)) | + (pack32(u8vec4(vals10.x, vals10.y, vals11.x, vals11.y)) << 4); + + if (iqs == 0) { + const uint is = iqs_k / 4; + const i8vec2 scales = i8vec2(unpack8(uint32_t(((data_a_packed16[ib_k].scales[(is % 8 ) / 2] >> (4 * (is / 8))) & 0x0F0F) | + (((data_a_packed16[ib_k].scales[(8 + (is % 4)) / 2] >> (2 * (is / 4))) & 0x0303) << 4))).xy); // vec4 used due to #12147 + + buf_a[buf_ib].d_scales = FLOAT_TYPEV2(float(data_a_packed16[ib_k].d) * vec2(scales - 32)); + } + } else { + buf_a[buf_ib].qs[iqs] = 0; + + if (iqs == 0) { + buf_a[buf_ib].d_scales = FLOAT_TYPEV2(0.0f); + } } } @@ -371,39 +456,47 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) // 4-byte loads for Q4_K blocks (144 bytes) and Q5_K blocks (176 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - const uint ib_k = ib / 8; - const uint iqs_k = (ib % 8) * 8 + iqs * QUANT_R_MMQ; +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + const uint ib_k = ib / 8; + const uint iqs_k = (ib % 8) * 8 + iqs * QUANT_R_MMQ; - const uint qs_idx = (iqs_k / 16) * 8 + (iqs_k % 8); - const uint qs_shift = ((iqs_k % 16) / 8) * 4; + const uint qs_idx = (iqs_k / 16) * 8 + (iqs_k % 8); + const uint qs_shift = ((iqs_k % 16) / 8) * 4; - // Repack 2x4 quants into one int -#if defined(DATA_A_Q4_K) - const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x0F0F0F0F; - const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x0F0F0F0F; + // Repack 2x4 quants into one int + #if defined(DATA_A_Q4_K) + const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x0F0F0F0F; + const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x0F0F0F0F; - buf_a[buf_ib].qs[iqs] = vals0 | (vals1 << 4); -#else // defined(DATA_A_Q5_K) - const uint qh_idx = iqs * QUANT_R_MMQ; - const uint qh_shift = iqs_k / 8; + buf_a[buf_ib].qs[iqs] = vals0 | (vals1 << 4); + #else // defined(DATA_A_Q5_K) + const uint qh_idx = iqs * QUANT_R_MMQ; + const uint qh_shift = iqs_k / 8; - buf_a[buf_ib].qs[iqs] = int32_t(((data_a_packed32[ib_k].qs[qs_idx] >> qs_shift) & 0x0F0F0F0F) | - (((data_a_packed32[ib_k].qh[qh_idx] >> qh_shift) & 0x01010101) << 4)); -#endif + buf_a[buf_ib].qs[iqs] = int32_t(((data_a_packed32[ib_k].qs[qs_idx] >> qs_shift) & 0x0F0F0F0F) | + (((data_a_packed32[ib_k].qh[qh_idx] >> qh_shift) & 0x01010101) << 4)); + #endif - if (iqs == 0) { - // Scale index - const uint is = iqs_k / 8; - u8vec2 scale_dm; - if (is < 4) { - scale_dm = u8vec2(data_a[ib_k].scales[is] & 0x3F, data_a[ib_k].scales[is + 4] & 0x3F); - } else { - scale_dm = u8vec2((data_a[ib_k].scales[is+4] & 0xF) | ((data_a[ib_k].scales[is-4] & 0xC0) >> 2), - (data_a[ib_k].scales[is+4] >> 4) | ((data_a[ib_k].scales[is ] & 0xC0) >> 2)); + if (iqs == 0) { + // Scale index + const uint is = iqs_k / 8; + u8vec2 scale_dm; + if (is < 4) { + scale_dm = u8vec2(data_a[ib_k].scales[is] & 0x3F, data_a[ib_k].scales[is + 4] & 0x3F); + } else { + scale_dm = u8vec2((data_a[ib_k].scales[is+4] & 0xF) | ((data_a[ib_k].scales[is-4] & 0xC0) >> 2), + (data_a[ib_k].scales[is+4] >> 4) | ((data_a[ib_k].scales[is ] & 0xC0) >> 2)); + } + + buf_a[buf_ib].dm = FLOAT_TYPEV2(vec2(data_a_packed32[ib_k].dm) * vec2(scale_dm)); } + } else { + buf_a[buf_ib].qs[iqs] = 0; - buf_a[buf_ib].dm = FLOAT_TYPEV2(vec2(data_a_packed32[ib_k].dm) * vec2(scale_dm)); + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPEV2(0.0f); + } } } @@ -434,27 +527,35 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_Q6_K) // 2-byte loads for Q6_K blocks (210 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - const uint ib_k = ib / 8; - const uint iqs_k = (ib % 8) * 8 + iqs; +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + const uint ib_k = ib / 8; + const uint iqs_k = (ib % 8) * 8 + iqs; + + const uint ql_idx = (iqs_k / 32) * 16 + iqs_k % 16; + const uint ql_shift = ((iqs_k % 32) / 16) * 4; - const uint ql_idx = (iqs_k / 32) * 16 + iqs_k % 16; - const uint ql_shift = ((iqs_k % 32) / 16) * 4; + const uint qh_idx = (iqs_k / 32) * 8 + iqs; + const uint qh_shift = ((iqs_k % 32) / 8) * 2; - const uint qh_idx = (iqs_k / 32) * 8 + iqs; - const uint qh_shift = ((iqs_k % 32) / 8) * 2; + const i8vec2 vals00 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 ] >> ql_shift) & uint16_t(0x0F0F))).xy | + unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32); + const i8vec2 vals01 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 + 1] >> ql_shift) & uint16_t(0x0F0F))).xy | + unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32); + buf_a[buf_ib].qs[iqs] = pack32(i8vec4(vals00.x, vals00.y, vals01.x, vals01.y)); - const i8vec2 vals00 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 ] >> ql_shift) & uint16_t(0x0F0F))).xy | - unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32); - const i8vec2 vals01 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 + 1] >> ql_shift) & uint16_t(0x0F0F))).xy | - unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32); - buf_a[buf_ib].qs[iqs] = pack32(i8vec4(vals00.x, vals00.y, vals01.x, vals01.y)); + if (iqs == 0) { + const uint is = iqs_k / 4; + const i8vec2 scales = unpack8(int32_t(data_a_packed16[ib_k].scales[is / 2])).xy; - if (iqs == 0) { - const uint is = iqs_k / 4; - const i8vec2 scales = unpack8(int32_t(data_a_packed16[ib_k].scales[is / 2])).xy; + buf_a[buf_ib].d_scales = FLOAT_TYPEV2(float(data_a_packed16[ib_k].d) * vec2(scales)); + } + } else { + buf_a[buf_ib].qs[iqs] = 0; - buf_a[buf_ib].d_scales = FLOAT_TYPEV2(float(data_a_packed16[ib_k].d) * vec2(scales)); + if (iqs == 0) { + buf_a[buf_ib].d_scales = FLOAT_TYPEV2(0.0f); + } } } @@ -491,32 +592,41 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_IQ3_S) // 2-byte loads for IQ3_S blocks (110 bytes) -void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { - const uint ib_k = ib / 8; - const uint ib32 = ib % 8; +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { + if (is_in_bounds) { + const uint ib_k = ib / 8; + const uint ib32 = ib % 8; - // grid indices for qs[2 * iqs] and qs[2 * iqs + 1] - const uint qs = uint(data_a_packed16[ib_k].qs[ib32 * 4 + iqs]); - // their two high index bits - const uint qh = uint(data_a_packed16[ib_k].qh[ib32 / 2]) >> ((ib32 & 1) * 8 + 2 * iqs); - // one sign bit per value, 8 values - const uint signs = uint(data_a_packed16[ib_k].signs[ib32 * 2 + iqs / 2]) >> ((iqs & 1) * 8); + // grid indices for qs[2 * iqs] and qs[2 * iqs + 1] + const uint qs = uint(data_a_packed16[ib_k].qs[ib32 * 4 + iqs]); + // their two high index bits + const uint qh = uint(data_a_packed16[ib_k].qh[ib32 / 2]) >> ((ib32 & 1) * 8 + 2 * iqs); + // one sign bit per value, 8 values + const uint signs = uint(data_a_packed16[ib_k].signs[ib32 * 2 + iqs / 2]) >> ((iqs & 1) * 8); - // grid holds 4 values of 1..15, one per byte - const ivec4 vals0 = ivec4(unpack8(iq3s_grid[( qs & 0xFF) | ((qh & 1) << 8)])); - const ivec4 vals1 = ivec4(unpack8(iq3s_grid[((qs >> 8) & 0xFF) | ((qh & 2) << 7)])); + // grid holds 4 values of 1..15, one per byte + const ivec4 vals0 = ivec4(unpack8(iq3s_grid[( qs & 0xFF) | ((qh & 1) << 8)])); + const ivec4 vals1 = ivec4(unpack8(iq3s_grid[((qs >> 8) & 0xFF) | ((qh & 2) << 7)])); - // negate with (v ^ -s) - -s to avoid branches - const ivec4 m0 = -(ivec4(signs, signs >> 1, signs >> 2, signs >> 3) & 1); - const ivec4 m1 = -(ivec4(signs >> 4, signs >> 5, signs >> 6, signs >> 7) & 1); + // negate with (v ^ -s) - -s to avoid branches + const ivec4 m0 = -(ivec4(signs, signs >> 1, signs >> 2, signs >> 3) & 1); + const ivec4 m1 = -(ivec4(signs >> 4, signs >> 5, signs >> 6, signs >> 7) & 1); - buf_a[buf_ib].qs[2 * iqs ] = pack32(i8vec4((vals0 ^ m0) - m0)); - buf_a[buf_ib].qs[2 * iqs + 1] = pack32(i8vec4((vals1 ^ m1) - m1)); + buf_a[buf_ib].qs[2 * iqs ] = pack32(i8vec4((vals0 ^ m0) - m0)); + buf_a[buf_ib].qs[2 * iqs + 1] = pack32(i8vec4((vals1 ^ m1) - m1)); - if (iqs == 0) { - const uint scale = (uint(data_a_packed16[ib_k].scales[ib32 / 4]) >> ((ib32 & 3) * 4)) & 0xF; + if (iqs == 0) { + const uint scale = (uint(data_a_packed16[ib_k].scales[ib32 / 4]) >> ((ib32 & 3) * 4)) & 0xF; + + buf_a[buf_ib].d = FLOAT_TYPE(float(data_a_packed16[ib_k].d) * float(1 + 2 * scale)); + } + } else { + buf_a[buf_ib].qs[2 * iqs] = 0; + buf_a[buf_ib].qs[2 * iqs + 1] = 0; - buf_a[buf_ib].d = FLOAT_TYPE(float(data_a_packed16[ib_k].d) * float(1 + 2 * scale)); + if (iqs == 0) { + buf_a[buf_ib].d = FLOAT_TYPE(0.0f); + } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp index 792012d57..ebf9c4499 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp @@ -14,6 +14,8 @@ void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; const uint tid = gl_LocalInvocationID.x; + if (row >= p.ne01 * p.ne02 * p.ne03) { return; } + const uint a_base = get_aoffset() + src0_idx(row * p.ne00); const uint d_base = get_doffset() + dst_idx(row * p.ne10); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp b/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp index d9d7166e3..10eedcfd2 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp @@ -1,5 +1,10 @@ #version 450 +#if defined(USE_SUBGROUP) +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#endif + #include "types.glsl" #extension GL_EXT_shader_16bit_storage : require @@ -15,18 +20,27 @@ layout(push_constant) uniform parameter { int p0; int p1; } p; -#define BLOCK_SIZE 512 +layout(constant_id = 0) const uint BLOCK_SIZE = 512; #define FLT_MAX 3.402823466e+38F #define OP_POOL_MAX 0u #define OP_POOL_AVG 1u -layout (local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; +layout (local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; layout(binding = 0) readonly buffer X {A_TYPE data_a[];}; layout(binding = 1) writeonly buffer D {D_TYPE data_d[];}; void main() { - const uint idx = gl_GlobalInvocationID.x; + const uint workgroup = gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.x; +#if defined(USE_SUBGROUP) + const uint idx = workgroup * gl_NumSubgroups + gl_SubgroupID; + const int lane = int(gl_SubgroupInvocationID); + const int step = int(gl_SubgroupSize); +#else + const uint idx = workgroup * BLOCK_SIZE + gl_LocalInvocationID.x; + const int lane = 0; + const int step = 1; +#endif if (idx >= p.pelements) { return; } @@ -38,12 +52,12 @@ void main() { const uint cur_ow = (idx % O_HW) % p.OW; const int start_h = int(cur_oh) * p.s0 - p.p0; - const uint bh = max(start_h, 0); - const uint eh = min(start_h + p.k0, p.IH); + const int bh = max(start_h, 0); + const int eh = min(start_h + p.k0, int(p.IH)); const int start_w = int(cur_ow) * p.s1 - p.p1; - const uint bw = max(start_w, 0); - const uint ew = min(start_w + p.k1, p.IW); + const int bw = max(start_w, 0); + const int ew = min(start_w + p.k1, int(p.IW)); const float scale = 1.0 / float(p.k0 * p.k1); float res; @@ -57,18 +71,24 @@ void main() { } #pragma unroll - for (uint i = bh; i < eh; i++) { + for (int i = bh; i < eh; i++) { #pragma unroll - for (uint j = bw; j < ew; j++) { + for (int j = bw + lane; j < ew; j += step) { const float cur = D_TYPE(data_a[nc * p.IH * p.IW + i * p.IW + j]); if (p.op == OP_POOL_AVG) { - res += cur * scale; + res += cur; } else if (p.op == OP_POOL_MAX) { res = max(res, cur); } } } - data_d[nc * O_HW + cur_oh * p.OW + cur_ow] = res; +#if defined(USE_SUBGROUP) + res = p.op == OP_POOL_AVG ? subgroupAdd(res) : subgroupMax(res); + if (!subgroupElect()) { + return; + } +#endif + data_d[idx] = p.op == OP_POOL_AVG ? res * scale : res; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp index ee813842c..6a2525849 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp @@ -3,6 +3,15 @@ #include "generic_binary_head.glsl" #include "types.glsl" +#extension GL_EXT_control_flow_attributes : enable +#if USE_SUBGROUP_ADD +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#endif + +layout (constant_id = 3) const int BLOCK_SIZE = 512; +layout(local_size_x_id = 3, local_size_y = 1, local_size_z = 1) in; + #if RMS_NORM_ROPE_FUSION layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; @@ -11,7 +20,8 @@ layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; // data is passed from rms_norm -> rope through shared memory. // rms_norm calls this data_d, rope calls this rope_data_a. // Binding 2 is not used -shared FLOAT_TYPE rope_data_a[1024]; +// host picks a block size such that ncols <= 2*BLOCK_SIZE +shared FLOAT_TYPE rope_data_a[2*BLOCK_SIZE]; #define data_d rope_data_a layout (binding = 3) readonly buffer R_Y {int rope_data_pos[];}; @@ -38,116 +48,138 @@ layout (binding = 3) readonly buffer I {uvec2 data_i[];}; #endif -#extension GL_EXT_control_flow_attributes : enable -#define BLOCK_SIZE 512 - layout (constant_id = 1) const bool do_multiply = false; #if RMS_NORM_ADD_FUSION layout (constant_id = 2) const bool do_post_multiply = false; #endif -layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; - shared FLOAT_TYPE sumsh[BLOCK_SIZE]; void rms_norm(uint num_iters) { const uint ncols = p.ne00; const uint nrows = gl_NumWorkGroups.x; - const uint nchannels = gl_NumWorkGroups.y; + const uint nchannels = p.ne02; + const uint nsamples = p.ne03; const uint row = gl_WorkGroupID.x; - const uint channel = gl_WorkGroupID.y; - const uint samp = gl_WorkGroupID.z; const uint tid = gl_LocalInvocationID.x; const uint stride_row = p.nb01; const uint stride_channel = p.nb02; const uint stride_sample = p.nb03; - uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset(); - uint32_t b_offset = src1_idx(0, row, channel, samp) + get_boffset(); + // grid.y/z are clamped to the device workgroup limit, iterate over the excess channels/samples + for (uint samp = gl_WorkGroupID.z; samp < nsamples; samp += gl_NumWorkGroups.z) { + for (uint channel = gl_WorkGroupID.y; channel < nchannels; channel += gl_NumWorkGroups.y) { + barrier(); + + uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset(); + uint32_t b_offset = src1_idx(0, row, channel, samp) + get_boffset(); #if RMS_NORM_ROPE_FUSION - // Per-row offset in shared memory - uint32_t d_offset = 0; + // Per-row offset in shared memory + uint32_t d_offset = 0; #elif RMS_NORM_SET_ROWS_FUSION - uint32_t d_offset = data_i[channel].x*p.nb21 + row*ncols + get_doffset(); + uint32_t d_offset = data_i[channel].x*p.nb21 + row*ncols + get_doffset(); #else - uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset(); + uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset(); #endif - FLOAT_TYPE sum = FLOAT_TYPE(0.0f); // partial sum for thread in warp - - [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { - FLOAT_TYPE xi = FLOAT_TYPE(0); - if (col < ncols) { - xi = FLOAT_TYPE(data_a[a_offset + col]); - } - sum += xi * xi; - } - - sumsh[tid] = sum; - // sum up partial sums and write back result - barrier(); - [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { - if (tid < s) { - sum += sumsh[tid + s]; - sumsh[tid] = sum; - } - barrier(); - } - sum = sumsh[0]; + FLOAT_TYPE sum = FLOAT_TYPE(0.0f); // partial sum for thread in warp - const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols); - const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1)); - - if (do_multiply) { - if (ncols > p.ne10) { [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { - if (col >= ncols) { - continue; - } - FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]); -#if RMS_NORM_ADD_FUSION - value += FLOAT_TYPE(data_c[d_offset + col]); - if (do_post_multiply) { - value *= FLOAT_TYPE(data_e[0]); + FLOAT_TYPE xi = FLOAT_TYPE(0); + if (col < ncols) { + xi = FLOAT_TYPE(data_a[a_offset + col]); } -#endif - data_d[d_offset + col] = D_TYPE(value); + sum += xi * xi; } - } else { - [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { - if (col >= ncols) { - continue; + + #if USE_SUBGROUP_ADD + sum = subgroupAdd(sum); + if (subgroupElect()) { + sumsh[gl_SubgroupID] = sum; } - FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]); -#if RMS_NORM_ADD_FUSION - value += FLOAT_TYPE(data_c[d_offset + col]); - if (do_post_multiply) { - value *= FLOAT_TYPE(data_e[0]); + barrier(); + sum = FLOAT_TYPE(0.0f); + if(gl_NumSubgroups <= 4) { // fast path to skip the 2nd subgroupAdd + [[unroll]] for(uint i=0; i < gl_NumSubgroups; ++i) { + sum += sumsh[i]; + } + } + else { + // Each subgroup reads the partials, so one workgroup barrier is enough. + for (uint i = gl_SubgroupInvocationID; i < gl_NumSubgroups; i += gl_SubgroupSize) { + sum += sumsh[i]; + } + sum = subgroupAdd(sum); + } + #else + sumsh[tid] = sum; + // sum up partial sums and write back result + barrier(); + [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) { + sum += sumsh[tid + s]; + sumsh[tid] = sum; + } + barrier(); } + sum = sumsh[0]; + #endif + + const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols); + const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1)); + + if (do_multiply) { + if (ncols > p.ne10) { + [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { + if (col >= ncols) { + continue; + } + FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]); +#if RMS_NORM_ADD_FUSION + value += FLOAT_TYPE(data_c[d_offset + col]); + if (do_post_multiply) { + value *= FLOAT_TYPE(data_e[0]); + } #endif - data_d[d_offset + col] = D_TYPE(value); - } - } - } else { - [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { - if (col >= ncols) { - continue; + data_d[d_offset + col] = D_TYPE(value); + } + } else { + [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { + if (col >= ncols) { + continue; + } + FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]); +#if RMS_NORM_ADD_FUSION + value += FLOAT_TYPE(data_c[d_offset + col]); + if (do_post_multiply) { + value *= FLOAT_TYPE(data_e[0]); + } +#endif + data_d[d_offset + col] = D_TYPE(value); + } + } + } else { + [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { + if (col >= ncols) { + continue; + } + data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col])); + } } - data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col])); - } - } #if RMS_NORM_ROPE_FUSION - barrier(); - rope_params rp = p.rope; - for (uint t = 2*tid; t < ncols; t += 2*BLOCK_SIZE) { - if (rp.rope_mode == GGML_ROPE_TYPE_NEOX) { - rope_neox(t, row, channel, samp, rp); - } else if (rp.rope_mode == GGML_ROPE_TYPE_NORMAL) { - rope_norm(t, row, channel, samp, rp); + barrier(); + rope_params rp = p.rope; + for (uint t = 2*tid; t < ncols; t += 2*BLOCK_SIZE) { + if (rp.rope_mode == GGML_ROPE_TYPE_NEOX) { + rope_neox(t, row, channel, samp, rp); + } else if (rp.rope_mode == GGML_ROPE_TYPE_NORMAL) { + rope_norm(t, row, channel, samp, rp); + } + } +#endif } } -#endif } void main() { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp index 87707fc14..b8cada5ae 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp @@ -19,6 +19,8 @@ void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; const uint tid = gl_LocalInvocationID.x; + if (row >= p.KY) { return; } + // Compute derivative of x[i]/norm(x) = g[i]/norm(x) - x[i] dot(x,g)/KX / norm(x)^1.5 // partial sums for thread in warp diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp b/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp index 13ba2e99d..94bab1dfd 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp @@ -19,6 +19,8 @@ void main() { const uint col = gl_LocalInvocationID.x; const float weight = p.weight; + if (row >= p.ne01 * p.ne02 * p.ne03) { return; } + const uint i03 = fastdiv(row, p.ne0_12mp, p.ne0_12L); const uint i03_offset = i03 * p.ne01*p.ne02; const uint i02 = fastdiv(row - i03_offset, p.ne0_1mp, p.ne0_1L); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl index 1cb0f7827..b50db0a97 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl @@ -5,7 +5,7 @@ layout (push_constant) uniform parameter { uint n_cols; - uint ne01, ne02; + uint ne01, ne02, ne03; uint nb01, nb02, nb03; uint nb11, nb12, nb13; float weight; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp b/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp index 0b757f38e..3f6296bda 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp @@ -60,7 +60,13 @@ void topk(const uint row) { if (gl_GlobalInvocationID.x < p.ncols_input) { if (p.first_pass != 0) { const uint row_offset = row * p.ncols_input; - dst_row[tid] = ivec2(gl_GlobalInvocationID.x, floatBitsToInt(data_a[row_offset + gl_GlobalInvocationID.x])); + // NaN ranks lowest, like -inf, so that every value has a place in + // the ordering the search below counts + float a = float(data_a[row_offset + gl_GlobalInvocationID.x]); + if (isnan(a)) { + a = uintBitsToFloat(0xFF800000); + } + dst_row[tid] = ivec2(gl_GlobalInvocationID.x, floatBitsToInt(a)); } else { const uint row_offset = row * p.ncols_input; dst_row[tid] = data_s[row_offset + gl_GlobalInvocationID.x]; @@ -76,8 +82,10 @@ void topk(const uint row) { if (tid < s) { ivec2 a = dst_row[tid]; ivec2 b = dst_row[tid + s]; + // compare as floats: the bit patterns of negative values + // order the other way as integers if (a.x >= p.orig_ncols || - b.x < p.orig_ncols && b.y > a.y) { + b.x < p.orig_ncols && intBitsToFloat(b.y) > intBitsToFloat(a.y)) { dst_row[tid] = b; } } @@ -95,9 +103,11 @@ void topk(const uint row) { int shift = 32 - SUBGROUP_SIZE_LOG2; uint mask = ((1 << SUBGROUP_SIZE_LOG2) - 1) << shift; - // The current range. + // The current range, [range_min, range_max). It starts as every value + // (+inf maps to 0xFF800000 and NaN was replaced by -inf), so the + // buckets always hold at least limit values. uint range_min = 0; - uint range_max = 0xFF800000; + uint range_max = 0xFFFFFFFF; // How many are above the current range, and how many we need to find. uint total = 0; uint limit = min(p.k, p.ncols_input - gl_WorkGroupID.x * BLOCK_SIZE); @@ -138,8 +148,12 @@ void topk(const uint row) { total = sh_total; // Update the range, and break if we've found the K-th largest. - range_max = range_min + ((min_idx + 1) << shift); - range_min = range_min + (min_idx << shift); + // The end of the top bucket wraps past 2^32, clamp it instead. + range_min = range_min + (uint(min_idx) << shift); + range_max = range_min + (1u << shift); + if (range_max < range_min) { + range_max = 0xFFFFFFFF; + } if (total == p.k) { break; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index ec0a80c7e..99ce1458a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -1921,8 +1921,9 @@ shared int8_t kvalues_mxfp4[16]; #endif #if defined(DATA_A_NVFP4) && !defined(USE_OCP_FP4) -// UE4M3 scale in NVFP4 blocks use only 7 bits; sign (bit 7) is always zero. -shared float ue4m3_fp32_lut[128]; +// Cover all scale bytes accepted by ggml_validate_row_data. +// ue4m3_to_fp32_build() masks the exponent and mantissa, but checks the full byte for 0 and 0x7F. +shared float ue4m3_fp32_lut[256]; float ue4m3_to_fp32_build(uint u) { if (u == 0u || u == 127u) { @@ -1947,7 +1948,7 @@ void init_iq_shmem(uvec3 wgsize) kvalues_mxfp4[i] = kvalues_mxfp4_const[i]; } #if defined(DATA_A_NVFP4) - for (uint i = gl_LocalInvocationIndex.x; i < 128u; i += wgsize.x) { + for (uint i = gl_LocalInvocationIndex.x; i < 256u; i += wgsize.x) { ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i); } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 7597b3d33..7ccccfa1d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -485,6 +485,7 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c base_dict["ACC_TYPE_VEC4"] = f16acc ? "f16vec4" : "vec4"; if (f16acc) { base_dict["ACC_TYPE_MAX"] = "float16_t(65504.0)"; + base_dict["ACC_F16"] = "1"; } if (coopmat) { @@ -852,31 +853,37 @@ void process_shaders() { // Norms string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("rms_norm_mul_add_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}})); + for (bool use_subgroups : {false, true}) { + const std::string suffix = use_subgroups ? "_subgroup" : ""; + const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}}); + string_to_spv("rms_norm_f32" + suffix, "rms_norm.comp", rms_dict); + string_to_spv("rms_norm_mul_add_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_ADD_FUSION", "1"}})); + string_to_spv("rms_norm_set_rows_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SET_ROWS_FUSION", "1"}})); + string_to_spv("rms_norm_set_rows_f32_f16" + suffix, "rms_norm.comp", merge_maps({{"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}, rms_dict)); // keep rms_dict last so 'merge_maps' overwrites the value for key 'D_TYPE' + string_to_spv("rms_norm_mul_rope_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}})); + string_to_spv("rms_norm_mul_rope_f32_f16" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}})); + } string_to_spv("rms_norm_mul_add_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}})); - string_to_spv("rms_norm_set_rows_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}})); - string_to_spv("rms_norm_set_rows_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}})); string_to_spv("rms_norm_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("rms_norm_mul_rope_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}})); - string_to_spv("rms_norm_mul_rope_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}})); string_to_spv("rms_norm_back_f32", "rms_norm_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("l2_norm_f32", "l2_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("cpy_f32_f32", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); + string_to_spv("cpy_32", "copy.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}}); string_to_spv("cpy_f32_f16", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float16_t"}}); - string_to_spv("cpy_f16_f16", "copy.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OPTIMIZATION_ERROR_WORKAROUND", "1"}}); + string_to_spv("cpy_16", "copy.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); string_to_spv("cpy_f16_f32", "copy.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float"}, {"OPTIMIZATION_ERROR_WORKAROUND", "1"}}); string_to_spv("cpy_f32_bf16","copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_D_BF16", "1"}}); string_to_spv("cpy_bf16_f32","copy.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}}); - string_to_spv("contig_cpy_f32_f32", "contig_copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); + string_to_spv("cpy_bf16_f16","copy.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "float16_t"}, {"DATA_A_BF16", "1"}}); + string_to_spv("contig_cpy_32", "contig_copy.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}}); string_to_spv("contig_cpy_f32_i32", "contig_copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "int"}}); string_to_spv("contig_cpy_i32_f32", "contig_copy.comp", {{"A_TYPE", "int"}, {"D_TYPE", "float"}}); string_to_spv("contig_cpy_f32_f16", "contig_copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float16_t"}}); - string_to_spv("contig_cpy_f16_f16", "contig_copy.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OPTIMIZATION_ERROR_WORKAROUND", "1"}}); + string_to_spv("contig_cpy_16", "contig_copy.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); string_to_spv("contig_cpy_f16_f32", "contig_copy.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float"}, {"OPTIMIZATION_ERROR_WORKAROUND", "1"}}); string_to_spv("contig_cpy_f32_bf16","contig_copy.comp",{{"A_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_D_BF16", "1"}}); string_to_spv("contig_cpy_bf16_f32","contig_copy.comp",{{"A_TYPE", "uint16_t"}, {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}}); + string_to_spv("contig_cpy_bf16_f16","contig_copy.comp",{{"A_TYPE", "uint16_t"}, {"D_TYPE", "float16_t"}, {"DATA_A_BF16", "1"}}); string_to_spv("cpy_f32_i32", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "int"}}); string_to_spv("cpy_i32_f32", "copy.comp", {{"A_TYPE", "int"}, {"D_TYPE", "float"}}); @@ -932,6 +939,7 @@ void process_shaders() { #if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) string_to_spv("fa_decode_ph1", "flash_attn_decode_phase_1.comp", {}, true, true, false, false); string_to_spv("fa_decode_ph2", "flash_attn_decode_phase_2.comp", {}, true, true, false, false); + string_to_spv("fa_prefill", "flash_attn_prefill.comp", {}, true, true, false, false); #endif string_to_spv("fa_sparse_compact", "flash_attn_sparse_compact.comp", {}); @@ -962,6 +970,9 @@ void process_shaders() { string_to_spv("concat_i16", "concat.comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); string_to_spv("concat_i32", "concat.comp", {{"A_TYPE", "uint"}, {"B_TYPE", "uint"}, {"D_TYPE", "uint"}}); string_to_spv("concat_i64", "concat.comp", {{"A_TYPE", "uvec2"}, {"B_TYPE", "uvec2"}, {"D_TYPE", "uvec2"}}); + string_to_spv("concat_transpose_i8", "concat_transpose.comp", {{"A_TYPE", "uint8_t"}, {"B_TYPE", "uint8_t"}, {"D_TYPE", "uint8_t"}}); + string_to_spv("concat_transpose_i16", "concat_transpose.comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); + string_to_spv("concat_transpose_i32", "concat_transpose.comp", {{"A_TYPE", "uint"}, {"B_TYPE", "uint"}, {"D_TYPE", "uint"}}); string_to_spv("upscale_f32", "upscale.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}); @@ -1064,8 +1075,6 @@ void process_shaders() { string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("diag_mask_inf_f32", "diag_mask_inf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("soft_max_f32", "soft_max.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("soft_max_f32_f16", "soft_max.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float"}})); string_to_spv("soft_max_back_f32", "soft_max_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); @@ -1103,8 +1112,10 @@ void process_shaders() { string_to_spv("sum_rows_f32", "sum_rows.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cross_entropy_loss_f32", "cross_entropy_loss.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cross_entropy_loss_back_f32", "cross_entropy_loss_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("fwht_f32", "fwht.comp", {}); - string_to_spv("fwht_shmem_f32", "fwht.comp", {{"FWHT_SHMEM", "1"}}); + string_to_spv("fwht_f32", "fwht.comp", {{"A_TYPE", "float"}}); + string_to_spv("fwht_shmem_f32", "fwht.comp", {{"A_TYPE", "float"}, {"FWHT_SHMEM", "1"}}); + string_to_spv("fwht_f16", "fwht.comp", {{"A_TYPE", "float16_t"}}); + string_to_spv("fwht_shmem_f16", "fwht.comp", {{"A_TYPE", "float16_t"}, {"FWHT_SHMEM", "1"}}); string_to_spv("count_equal_i32", "count_equal.comp", merge_maps(base_dict, {{"A_TYPE", "int"}, {"B_TYPE", "int"}, {"D_TYPE", "int"}})); string_to_spv("dsv4_hc_comb_f32", "dsv4_hc_comb.comp", {}); string_to_spv("dsv4_hc_pre_f32", "dsv4_hc_pre.comp", {}); @@ -1140,6 +1151,7 @@ void process_shaders() { string_to_spv("pool1d_f32", "pool1d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("pool2d_f32", "pool2d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); + string_to_spv("pool2d_subgroup_f32", "pool2d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP", "1"}})); string_to_spv("rwkv_wkv6_f32", "wkv6.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp index 65556f352..3cc6a70c8 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp @@ -19,6 +19,7 @@ #define GGML_WEBGPU_I32_SIZE_BYTES 4 #define GGML_WEBGPU_FLASH_ATTN_PREFERRED_KV_SG_TILES 8u #define GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN 20u +#define GGML_WEBGPU_MUL_MAT_VEC_MAX_ROWS 4 #define GGML_WEBGPU_FLASH_ATTN_VEC_MAX_KV_TILE 32u #define GGML_WEBGPU_FLASH_ATTN_TILE_MAX_KV_TILE 64u #define GGML_WEBGPU_FLASH_ATTN_PREFERRED_WG_SIZE 128u @@ -26,6 +27,7 @@ #define GGML_WEBGPU_KV_SEQ_PAD 256u #define GGML_WEBGPU_ARGSORT_MERGE_MAX_WG_SIZE 512u +#define GGML_WEBGPU_COUNT_EQUAL_WG_SIZE 256u // Matrix multiplication parameters @@ -250,6 +252,31 @@ struct ggml_webgpu_get_rows_pipeline_key_hash { } }; +/** Group Norm **/ + +struct ggml_webgpu_group_norm_pipeline_key { + ggml_type src_type; + ggml_type dst_type; + bool inplace; + uint32_t wg_size; + + bool operator==(const ggml_webgpu_group_norm_pipeline_key & other) const { + return src_type == other.src_type && dst_type == other.dst_type && inplace == other.inplace && + wg_size == other.wg_size; + } +}; + +struct ggml_webgpu_group_norm_pipeline_key_hash { + size_t operator()(const ggml_webgpu_group_norm_pipeline_key & key) const { + size_t seed = 0; + ggml_webgpu_hash_combine(seed, key.src_type); + ggml_webgpu_hash_combine(seed, key.dst_type); + ggml_webgpu_hash_combine(seed, key.inplace); + ggml_webgpu_hash_combine(seed, key.wg_size); + return seed; + } +}; + /** Row Norm **/ struct ggml_webgpu_row_norm_pipeline_key { @@ -306,14 +333,18 @@ struct ggml_webgpu_rms_norm_mul_shader_decisions { /** Pad **/ struct ggml_webgpu_pad_pipeline_key { bool circular; + bool reflect; - bool operator==(const ggml_webgpu_pad_pipeline_key & other) const { return circular == other.circular; } + bool operator==(const ggml_webgpu_pad_pipeline_key & other) const { + return circular == other.circular && reflect == other.reflect; + } }; struct ggml_webgpu_pad_pipeline_key_hash { size_t operator()(const ggml_webgpu_pad_pipeline_key & key) const { size_t seed = 0; ggml_webgpu_hash_combine(seed, key.circular); + ggml_webgpu_hash_combine(seed, key.reflect); return seed; } }; @@ -1053,6 +1084,22 @@ struct ggml_webgpu_mul_mat_shader_decisions { /** MUL_MAT_ID **/ +struct ggml_webgpu_mul_mat_id_gather_pipeline_key { + uint32_t n_experts; + + bool operator==(const ggml_webgpu_mul_mat_id_gather_pipeline_key & other) const { + return n_experts == other.n_experts; + } +}; + +struct ggml_webgpu_mul_mat_id_gather_pipeline_key_hash { + size_t operator()(const ggml_webgpu_mul_mat_id_gather_pipeline_key & key) const { + size_t seed = 0; + ggml_webgpu_hash_combine(seed, key.n_experts); + return seed; + } +}; + struct ggml_webgpu_mul_mat_id_pipeline_key { ggml_type src0_type; ggml_type src1_type; @@ -1173,9 +1220,10 @@ struct ggml_webgpu_soft_max_pipeline_key_hash { inline bool ggml_webgpu_can_use_mmvq(const ggml_tensor * src0, const ggml_tensor * src1, + int64_t n_rows, bool supports_dot_product, const std::string & vendor) { - if (src1->ne[1] <= 4) { + if (n_rows <= GGML_WEBGPU_MUL_MAT_VEC_MAX_ROWS) { bool supports_dp4a = vendor == "amd" || vendor == "intel" || vendor == "nvidia"; if (supports_dp4a && supports_dot_product) { switch (src1->type) { @@ -1207,14 +1255,19 @@ inline bool ggml_webgpu_can_use_mmvq(const ggml_tensor * src0, } class ggml_webgpu_shader_lib { + std::unordered_map<ggml_webgpu_group_norm_pipeline_key, webgpu_pipeline, ggml_webgpu_group_norm_pipeline_key_hash> + group_norm_pipelines; + wgpu::Device device; pre_wgsl::Preprocessor preprocessor; - std::unordered_map<int, webgpu_pipeline> sum_rows_pipelines; // key is fixed, no variants yet + std::unordered_map<int, webgpu_pipeline> sum_rows_pipelines; // key is whether to compute the mean std::unordered_map<int, webgpu_pipeline> argmax_pipelines; // key is vec4 std::unordered_map<int, webgpu_pipeline> argsort_pipelines; // key is order std::unordered_map<int, webgpu_pipeline> argsort_merge_pipelines; // key is order std::unordered_map<int, webgpu_pipeline> cumsum_pipelines; // key is fixed, no variants yet + std::unordered_map<int, webgpu_pipeline> arange_pipelines; // key is fixed, no variants yet + std::unordered_map<int, webgpu_pipeline> count_equal_pipelines; // key is src_overlap std::unordered_map<ggml_webgpu_row_norm_pipeline_key, webgpu_pipeline, ggml_webgpu_row_norm_pipeline_key_hash> row_norm_pipelines; // op/inplace @@ -1263,12 +1316,15 @@ class ggml_webgpu_shader_lib { std::unordered_map<ggml_webgpu_mul_mat_pipeline_key, webgpu_pipeline, ggml_webgpu_mul_mat_pipeline_key_hash> mul_mat_fast_pipelines; // fast mat-mat (reg-tile or subgroup) std::unordered_map<ggml_webgpu_quantize_q8_pipeline_key, webgpu_pipeline, ggml_webgpu_quantize_q8_pipeline_key_hash> - quantize_q8_pipelines; - std::unordered_map<int, webgpu_pipeline> mul_mat_id_gather_pipelines; // key is fixed + quantize_q8_pipelines; + std::unordered_map<ggml_webgpu_mul_mat_id_gather_pipeline_key, + webgpu_pipeline, + ggml_webgpu_mul_mat_id_gather_pipeline_key_hash> + mul_mat_id_gather_pipelines; // n_experts std::unordered_map<ggml_webgpu_mul_mat_id_pipeline_key, webgpu_pipeline, ggml_webgpu_mul_mat_id_pipeline_key_hash> - mul_mat_id_pipelines; // src0_type/src1_type + mul_mat_id_pipelines; // src0_type/src1_type std::unordered_map<ggml_webgpu_mul_mat_id_pipeline_key, webgpu_pipeline, ggml_webgpu_mul_mat_id_pipeline_key_hash> - mul_mat_id_vec_pipelines; // src0_type/src1_type + mul_mat_id_vec_pipelines; // src0_type/src1_type std::unordered_map<ggml_webgpu_set_rows_pipeline_key, webgpu_pipeline, ggml_webgpu_set_rows_pipeline_key_hash> set_rows_pipelines; @@ -1296,17 +1352,70 @@ class ggml_webgpu_shader_lib { public: ggml_webgpu_shader_lib(wgpu::Device device) { this->device = device; } + webgpu_pipeline get_count_equal_pipeline(bool src_overlap) { + const int key = src_overlap; + auto it = count_equal_pipelines.find(key); + if (it != count_equal_pipelines.end()) { + return it->second; + } + std::vector<std::string> defines = { "WG_SIZE=" + std::to_string(GGML_WEBGPU_COUNT_EQUAL_WG_SIZE) }; + if (src_overlap) { + defines.push_back("SRC_OVERLAP"); + } + auto processed = preprocessor.preprocess(wgsl_count_equal, defines); + count_equal_pipelines[key] = ggml_webgpu_create_pipeline(device, processed, "count_equal"); + return count_equal_pipelines[key]; + } + webgpu_pipeline get_sum_rows_pipeline(const ggml_webgpu_shader_lib_context & context) { - auto it = sum_rows_pipelines.find(1); + const bool mean = context.dst->op == GGML_OP_MEAN; + auto it = sum_rows_pipelines.find(mean); if (it != sum_rows_pipelines.end()) { return it->second; } std::vector<std::string> defines; defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size)); + if (mean) { + defines.push_back("MEAN"); + } - auto processed = preprocessor.preprocess(wgsl_sum_rows, defines); - sum_rows_pipelines[1] = ggml_webgpu_create_pipeline(device, processed, "sum_rows"); - return sum_rows_pipelines[1]; + auto processed = preprocessor.preprocess(wgsl_sum_rows, defines); + sum_rows_pipelines[mean] = ggml_webgpu_create_pipeline(device, processed, mean ? "mean" : "sum_rows"); + return sum_rows_pipelines[mean]; + } + + webgpu_pipeline get_group_norm_pipeline(const ggml_webgpu_shader_lib_context & context) { + const ggml_webgpu_group_norm_pipeline_key key = { + context.src0->type, + context.dst->type, + ggml_webgpu_tensor_equal(context.src0, context.dst), + std::min(context.max_wg_size, 128u), + }; + auto it = group_norm_pipelines.find(key); + if (it != group_norm_pipelines.end()) { + return it->second; + } + + std::vector<std::string> defines = { + "WG_SIZE=" + std::to_string(key.wg_size), + key.src_type == GGML_TYPE_F16 ? "SRC_F16" : "SRC_F32", + key.dst_type == GGML_TYPE_F16 ? "DST_F16" : "DST_F32", + }; + std::string variant = + std::string("group_norm_src_") + ggml_type_name(key.src_type) + "_dst_" + ggml_type_name(key.dst_type); + if (key.inplace) { + GGML_ASSERT(key.src_type == key.dst_type); + defines.push_back("INPLACE"); + variant += "_inplace"; + } + auto processed = preprocessor.preprocess(wgsl_group_norm, defines); + auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>(); + decisions->wg_size = key.wg_size; + decisions->inplace = key.inplace; + auto pipeline = ggml_webgpu_create_pipeline(device, processed, variant); + pipeline.context = decisions; + group_norm_pipelines[key] = pipeline; + return pipeline; } webgpu_pipeline get_row_norm_pipeline(const ggml_webgpu_shader_lib_context & context) { @@ -1636,6 +1745,7 @@ class ggml_webgpu_shader_lib { switch (key.src_type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q5_0: case GGML_TYPE_Q8_0: @@ -1680,7 +1790,7 @@ class ggml_webgpu_shader_lib { } else if ((key.src_type >= GGML_TYPE_Q4_0 && key.src_type <= GGML_TYPE_Q8_1) || key.src_type == GGML_TYPE_IQ4_NL || key.src_type == GGML_TYPE_MXFP4) { defines.push_back("BLOCK_SIZE=32u"); - } else if (key.src_type == GGML_TYPE_NVFP4) { + } else if (key.src_type == GGML_TYPE_Q2_0 || key.src_type == GGML_TYPE_NVFP4) { defines.push_back("BLOCK_SIZE=64u"); } else if (key.src_type >= GGML_TYPE_Q2_K) { defines.push_back("BLOCK_SIZE=256u"); @@ -1921,9 +2031,28 @@ class ggml_webgpu_shader_lib { return gated_delta_net_pipelines[key]; } + webgpu_pipeline get_arange_pipeline(const ggml_webgpu_shader_lib_context & context) { + auto it = arange_pipelines.find(1); + if (it != arange_pipelines.end()) { + return it->second; + } + + std::vector<std::string> defines; + defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size)); + + auto processed = preprocessor.preprocess(wgsl_arange, defines); + auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>(); + decisions->wg_size = context.max_wg_size; + webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, "arange"); + pipeline.context = decisions; + arange_pipelines[1] = pipeline; + return arange_pipelines[1]; + } + webgpu_pipeline get_pad_pipeline(const ggml_webgpu_shader_lib_context & context) { ggml_webgpu_pad_pipeline_key key = {}; - key.circular = ggml_get_op_params_i32(context.dst, 8) != 0; + key.reflect = context.dst->op == GGML_OP_PAD_REFLECT_1D; + key.circular = !key.reflect && ggml_get_op_params_i32(context.dst, 8) != 0; auto it = pad_pipelines.find(key); if (it != pad_pipelines.end()) { @@ -1933,6 +2062,10 @@ class ggml_webgpu_shader_lib { std::vector<std::string> defines; std::string variant = "pad"; + if (key.reflect) { + defines.push_back("REFLECT"); + variant += "_reflect"; + } if (key.circular) { defines.push_back("CIRCULAR"); variant += "_circular"; @@ -1991,13 +2124,13 @@ class ggml_webgpu_shader_lib { ggml_webgpu_mul_mat_vec_pipeline_key key = {}; key.src0_type = context.src0->type; key.src1_type = context.src1->type; - key.vectorized = (context.src0->ne[0] % 4 == 0 && - (context.src0->type == GGML_TYPE_F32 || context.src0->type == GGML_TYPE_F16)) ? - 1 : - 0; - key.num_cols = context.dst->ne[1]; - key.use_mmvq = - ggml_webgpu_can_use_mmvq(context.src0, context.src1, context.supports_dot_product, context.vendor); + key.vectorized = (context.src0->ne[0] % 4 == 0 && + (context.src0->type == GGML_TYPE_F32 || context.src0->type == GGML_TYPE_F16)) ? + 1 : + 0; + key.num_cols = context.dst->ne[1]; + key.use_mmvq = ggml_webgpu_can_use_mmvq(context.src0, context.src1, context.src1->ne[1], + context.supports_dot_product, context.vendor); key.src_overlap = src_overlap; auto it = mul_mat_vec_pipelines.find(key); @@ -2325,20 +2458,26 @@ class ggml_webgpu_shader_lib { } webgpu_pipeline get_mul_mat_id_gather_pipeline(const ggml_webgpu_shader_lib_context & context) { - auto it = mul_mat_id_gather_pipelines.find(1); + ggml_webgpu_mul_mat_id_gather_pipeline_key key = {}; + key.n_experts = (uint32_t) context.src0->ne[2]; + + auto it = mul_mat_id_gather_pipelines.find(key); if (it != mul_mat_id_gather_pipelines.end()) { return it->second; } std::vector<std::string> defines; - defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size)); + defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size) + "u"); + defines.push_back(std::string("N_EXPERT=") + std::to_string(key.n_experts) + "u"); + defines.push_back(std::string("TILE_N_SLOTS=") + + std::to_string(WEBGPU_MUL_MAT_TILE_N * WEBGPU_MUL_MAT_WG_SIZE_N) + "u"); auto processed = preprocessor.preprocess(wgsl_mul_mat_id_gather, defines); auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>(); decisions->wg_size = context.max_wg_size; - webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, "mul_mat_id_gather"); - pipeline.context = decisions; - mul_mat_id_gather_pipelines[1] = pipeline; + webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, "mul_mat_id_gather"); + pipeline.context = decisions; + mul_mat_id_gather_pipelines[key] = pipeline; return pipeline; } diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index a7986b9c7..c08aa415f 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -182,7 +182,6 @@ struct webgpu_capabilities { uint32_t subgroup_size = 0; uint32_t min_subgroup_size = 0; uint32_t max_subgroup_size = 0; - size_t memset_bytes_per_thread; }; // Stores global webgpu members @@ -203,6 +202,11 @@ struct webgpu_global_context_struct { wgpu::Buffer memset_params_buf; webgpu_pipeline memset_pipeline; + // Packed source data and write descriptors for set_tensor_n + wgpu::Buffer set_tensor_n_src_buf; + wgpu::Buffer set_tensor_n_descs_buf; + webgpu_pipeline set_tensor_n_pipeline; + std::string vendor; // TODO: We should rework the CPU profiling time handling to make it more useful. ref: https://github.com/ggml-org/llama.cpp/pull/22050 @@ -227,6 +231,14 @@ struct webgpu_global_context_struct { this->memset_params_buf.Destroy(); this->memset_params_buf = nullptr; } + if (this->set_tensor_n_src_buf) { + this->set_tensor_n_src_buf.Destroy(); + this->set_tensor_n_src_buf = nullptr; + } + if (this->set_tensor_n_descs_buf) { + this->set_tensor_n_descs_buf.Destroy(); + this->set_tensor_n_descs_buf = nullptr; + } #ifdef GGML_WEBGPU_DEBUG if (this->debug_host_buf) { this->debug_host_buf.Destroy(); @@ -256,8 +268,6 @@ struct webgpu_context_struct { wgpu::ComputePassEncoder active_compute_pass; bool batch_compute_passes = true; - size_t memset_bytes_per_thread; - #ifdef GGML_WEBGPU_GPU_PROFILE // Profiling: per-shader GPU time in ms std::unordered_map<std::string, double> shader_gpu_time_ms; @@ -670,38 +680,56 @@ static void ggml_backend_webgpu_buffer_memset(webgpu_global_context & ctx, uint32_t value, size_t offset, size_t size) { - std::vector<uint32_t> params = { (uint32_t) offset, (uint32_t) size, value }; - std::vector<wgpu::BindGroupEntry> entries = { ggml_webgpu_make_bind_group_entry(0, buf, 0, buf.GetSize()) }; - size_t bytes_per_wg = - ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * ctx->capabilities.memset_bytes_per_thread; - uint32_t wg_x = CEIL_DIV(size + 3, bytes_per_wg); - - ctx->queue.WriteBuffer(ctx->memset_params_buf, 0, params.data(), params.size() * sizeof(uint32_t)); - - wgpu::BindGroupEntry params_entry = {}; - params_entry.binding = 1; - params_entry.buffer = ctx->memset_params_buf; - params_entry.offset = 0; - params_entry.size = WEBGPU_PARAMS_BUF_SIZE_BYTES; - entries.push_back(params_entry); - - wgpu::BindGroupDescriptor bind_group_desc; - bind_group_desc.layout = ctx->memset_pipeline.pipeline.GetBindGroupLayout(0); - bind_group_desc.entryCount = entries.size(); - bind_group_desc.entries = entries.data(); - bind_group_desc.label = ctx->memset_pipeline.name.c_str(); - wgpu::BindGroup bind_group = ctx->device.CreateBindGroup(&bind_group_desc); - - wgpu::CommandEncoder encoder = ctx->device.CreateCommandEncoder(); - wgpu::ComputePassEncoder pass = encoder.BeginComputePass(); - pass.SetPipeline(ctx->memset_pipeline.pipeline); - pass.SetBindGroup(0, bind_group); - pass.DispatchWorkgroups(wg_x, 1, 1); - pass.End(); - - wgpu::CommandBuffer command = encoder.Finish(); - std::vector<wgpu::CommandBuffer> commands = { command }; - ctx->queue.Submit(commands.size(), commands.data()); + std::lock_guard<std::recursive_mutex> lock(ctx->mutex); + + const size_t max_bind_size = ctx->capabilities.limits.maxStorageBufferBindingSize; + const size_t max_wg_size = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + + while (size > 0) { + const size_t align_offset = offset & ~(ctx->capabilities.limits.minStorageBufferOffsetAlignment - 1); + const size_t lead = offset - align_offset; + const size_t chunk_size = std::min(size, max_bind_size - lead); + const size_t bind_size = ROUNDUP_POW2(chunk_size + lead, WEBGPU_STORAGE_BUF_BINDING_MULT); + + std::vector<uint32_t> params = { (uint32_t) lead, (uint32_t) chunk_size, value }; + std::vector<wgpu::BindGroupEntry> entries = { ggml_webgpu_make_bind_group_entry(0, buf, align_offset, + bind_size) }; + uint32_t wg_x; + uint32_t wg_y; + uint32_t n_values = CEIL_DIV(chunk_size + lead, sizeof(uint32_t)); + uint32_t total_wg = CEIL_DIV(n_values, max_wg_size); + compute_2d_workgroups(total_wg, ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + + ctx->queue.WriteBuffer(ctx->memset_params_buf, 0, params.data(), params.size() * sizeof(uint32_t)); + + wgpu::BindGroupEntry params_entry = {}; + params_entry.binding = 1; + params_entry.buffer = ctx->memset_params_buf; + params_entry.offset = 0; + params_entry.size = WEBGPU_PARAMS_BUF_SIZE_BYTES; + entries.push_back(params_entry); + + wgpu::BindGroupDescriptor bind_group_desc; + bind_group_desc.layout = ctx->memset_pipeline.pipeline.GetBindGroupLayout(0); + bind_group_desc.entryCount = entries.size(); + bind_group_desc.entries = entries.data(); + bind_group_desc.label = ctx->memset_pipeline.name.c_str(); + wgpu::BindGroup bind_group = ctx->device.CreateBindGroup(&bind_group_desc); + + wgpu::CommandEncoder encoder = ctx->device.CreateCommandEncoder(); + wgpu::ComputePassEncoder pass = encoder.BeginComputePass(); + pass.SetPipeline(ctx->memset_pipeline.pipeline); + pass.SetBindGroup(0, bind_group); + pass.DispatchWorkgroups(wg_x, wg_y, 1); + pass.End(); + + wgpu::CommandBuffer command = encoder.Finish(); + std::vector<wgpu::CommandBuffer> commands = { command }; + ctx->queue.Submit(commands.size(), commands.data()); + + offset += chunk_size; + size -= chunk_size; + } } /** End WebGPU Actions */ @@ -847,8 +875,37 @@ static webgpu_encoded_op ggml_webgpu_set(webgpu_context & ctx, entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, binding_index, src1)); entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, binding_index + 1, dst)); - uint32_t wg_x = CEIL_DIV(ne, decisions->wg_size); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); +} + +static webgpu_encoded_op ggml_webgpu_arange(webgpu_context & ctx, ggml_tensor * dst) { + ggml_webgpu_shader_lib_context shader_lib_ctx = {}; + shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + + webgpu_pipeline pipeline = ctx->shader_lib->get_arange_pipeline(shader_lib_ctx); + auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get()); + + const uint32_t ne = (uint32_t) ggml_nelements(dst); + std::vector<uint32_t> params = { + ne, + (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)), + ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 0)), + ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 2)), + }; + + std::vector<wgpu::BindGroupEntry> entries = { + ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, dst), + }; + + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_pad(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { @@ -897,8 +954,11 @@ static webgpu_encoded_op ggml_webgpu_pad(webgpu_context & ctx, ggml_tensor * src ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst), }; - uint32_t wg_x = CEIL_DIV(ne, decisions->wg_size); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_solve_tri(webgpu_context & ctx, @@ -1495,8 +1555,11 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_set_rows(webgpu_context & ct } else { threads = src->ne[0] * src->ne[1] * src->ne[2] * src->ne[3]; } - uint32_t wg_x = CEIL_DIV(threads, decisions->wg_size); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, 1); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(threads, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } // Workgroup size is a common constant @@ -1557,9 +1620,11 @@ static webgpu_encoded_op ggml_webgpu_get_rows(webgpu_context & ctx, uint32_t blocks_per_row = (uint32_t) (dst->ne[0] / (decisions->vectorized ? 4 : 1)); uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); uint32_t total_threads = float_parallel ? blocks_per_row * total_rows : total_rows; - uint32_t wg_x = CEIL_DIV(total_threads, decisions->wg_size); - - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(total_threads, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static void ggml_webgpu_quantize_q8_dispatch(webgpu_context & ctx, @@ -1621,11 +1686,11 @@ static webgpu_encoded_op ggml_webgpu_mul_mat(webgpu_context & ctx, ggml_tensor * src1, ggml_tensor * dst) { // Determine if this is a mat-vec operation - bool use_mat_vec = (dst->ne[1] <= 4); + bool use_mat_vec = (dst->ne[1] <= GGML_WEBGPU_MUL_MAT_VEC_MAX_ROWS); // use MMVQ path for mat-vec - bool use_mmvq = ggml_webgpu_can_use_mmvq(src0, src1, ctx->global_ctx->capabilities.supports_dot_product, - ctx->global_ctx->vendor); + bool use_mmvq = ggml_webgpu_can_use_mmvq( + src0, src1, src1->ne[1], ctx->global_ctx->capabilities.supports_dot_product, ctx->global_ctx->vendor); ggml_webgpu_shader_lib_context shader_lib_ctx = {}; @@ -1806,6 +1871,19 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id_vec(webgpu_context & ctx, return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } +// upper bound on the number of slot tiles emitted by mul_mat_id_gather.wgsl +static uint32_t ggml_webgpu_mul_mat_id_max_tiles(const ggml_tensor * dst) { + const uint32_t n_expert = (uint32_t) dst->src[0]->ne[2]; + const uint32_t n_slots = (uint32_t) (dst->ne[1] * dst->ne[2]); // n_expert_used * n_tokens + + return CEIL_DIV(n_slots, WEBGPU_MUL_MAT_TILE_N * WEBGPU_MUL_MAT_WG_SIZE_N) + std::min(n_expert, n_slots); +} + +// tile count followed by (expert, first slot, n slots) per tile +static size_t ggml_webgpu_mul_mat_id_tiles_nbytes(const ggml_tensor * dst) { + return (1 + 3 * (size_t) ggml_webgpu_mul_mat_id_max_tiles(dst)) * sizeof(uint32_t); +} + static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx, ggml_tensor * src0, ggml_tensor * src1, @@ -1839,45 +1917,39 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx, // params for mul_mat_id_gather.wgsl std::vector<uint32_t> gather_params = { (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src2) / ggml_type_size(src2->type)), + (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)), param_n_expert, param_n_expert_used, param_n_tokens, (uint32_t) (src2->nb[1] / ggml_type_size(src2->type)), + (uint32_t) dst->ne[0], }; - const size_t dst_offset = ggml_webgpu_tensor_offset(dst); - const size_t gathered_buf_nbytes = src0->ne[2] * src1->ne[2] * sizeof(uint32_t); + const size_t dst_offset = ggml_webgpu_tensor_offset(dst); + const size_t ids_cache_nbytes = src2->ne[0] * src1->ne[2] * sizeof(uint32_t); + const size_t ids_tiles_nbytes = ggml_webgpu_mul_mat_id_tiles_nbytes(dst); - const size_t gathered_expert_used_align_offset = ROUNDUP_POW2( + const size_t ids_cache_align_offset = ROUNDUP_POW2( dst_offset + ggml_nbytes(dst), ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); - const size_t gathered_tokens_align_offset = - ROUNDUP_POW2(gathered_expert_used_align_offset + gathered_buf_nbytes, - ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); - const size_t gathered_count_ids_align_offset = - ROUNDUP_POW2(gathered_tokens_align_offset + gathered_buf_nbytes, + const size_t ids_tiles_align_offset = + ROUNDUP_POW2(ids_cache_align_offset + ids_cache_nbytes, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); - const size_t gathered_binding_size = ROUNDUP_POW2(gathered_buf_nbytes, WEBGPU_STORAGE_BUF_BINDING_MULT); - const size_t gathered_count_ids_binding_size = - ROUNDUP_POW2(src0->ne[2] * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT); + const size_t ids_cache_bind_size = ROUNDUP_POW2(ids_cache_nbytes, WEBGPU_STORAGE_BUF_BINDING_MULT); + const size_t ids_tiles_bind_size = ROUNDUP_POW2(ids_tiles_nbytes, WEBGPU_STORAGE_BUF_BINDING_MULT); // bind group entries for mul_mat_id_gather.wgsl std::vector<wgpu::BindGroupEntry> gather_entries = { ggml_webgpu_make_bind_group_entry(0, ggml_webgpu_tensor_buf(src2), ggml_webgpu_tensor_align_offset(ctx, src2), ggml_webgpu_tensor_binding_size(ctx, src2)), - ggml_webgpu_make_bind_group_entry(1, ggml_webgpu_tensor_buf(dst), gathered_expert_used_align_offset, - gathered_binding_size), - ggml_webgpu_make_bind_group_entry(2, ggml_webgpu_tensor_buf(dst), gathered_tokens_align_offset, - gathered_binding_size), - ggml_webgpu_make_bind_group_entry(3, ggml_webgpu_tensor_buf(dst), gathered_count_ids_align_offset, - gathered_count_ids_binding_size), + ggml_webgpu_make_bind_group_entry(1, ggml_webgpu_tensor_buf(dst), ids_cache_align_offset, ids_cache_bind_size), + ggml_webgpu_make_bind_group_entry(2, ggml_webgpu_tensor_buf(dst), ids_tiles_align_offset, ids_tiles_bind_size), + ggml_webgpu_make_bind_group_entry(3, ggml_webgpu_tensor_buf(dst), ggml_webgpu_tensor_align_offset(ctx, dst), + ggml_webgpu_tensor_binding_size(ctx, dst)), }; - // n_expert is much less than maxComputeWorkgroupsPerDimension (e.g., n_exeprt=256 at Qwen3.5-35B-A3B) - const uint32_t gather_wg_x = param_n_expert; - dispatches.push_back({ - gather_pipeline, std::move(gather_params), std::move(gather_entries), { gather_wg_x, 1 } + gather_pipeline, std::move(gather_params), std::move(gather_entries), { 1, 1 } }); // params for mul_mat_id.wgsl @@ -1905,12 +1977,8 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx, ggml_webgpu_tensor_binding_size(ctx, src1)), ggml_webgpu_make_bind_group_entry(2, ggml_webgpu_tensor_buf(dst), ggml_webgpu_tensor_align_offset(ctx, dst), ggml_webgpu_tensor_binding_size(ctx, dst)), - ggml_webgpu_make_bind_group_entry(3, ggml_webgpu_tensor_buf(dst), gathered_expert_used_align_offset, - gathered_binding_size), - ggml_webgpu_make_bind_group_entry(4, ggml_webgpu_tensor_buf(dst), gathered_tokens_align_offset, - gathered_binding_size), - ggml_webgpu_make_bind_group_entry(5, ggml_webgpu_tensor_buf(dst), gathered_count_ids_align_offset, - gathered_count_ids_binding_size), + ggml_webgpu_make_bind_group_entry(3, ggml_webgpu_tensor_buf(dst), ids_cache_align_offset, ids_cache_bind_size), + ggml_webgpu_make_bind_group_entry(4, ggml_webgpu_tensor_buf(dst), ids_tiles_align_offset, ids_tiles_bind_size), }; // Calculate workgroup dimensions @@ -1919,15 +1987,12 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx, auto * main_decisions = static_cast<ggml_webgpu_mul_mat_shader_decisions *>(main_pipeline.context.get()); - uint32_t wg_m; + // must match `TILE_N_SLOTS` of mul_mat_id_gather.wgsl + GGML_ASSERT(main_decisions->tile_n * main_decisions->wg_size_n == WEBGPU_MUL_MAT_TILE_N * WEBGPU_MUL_MAT_WG_SIZE_N); - uint32_t tile_m_s = main_decisions->tile_m * main_decisions->wg_size_m; - uint32_t tile_n_s = main_decisions->tile_n * main_decisions->wg_size_n; - wg_m = CEIL_DIV(dst->ne[0], tile_m_s); - uint32_t total_gathered = dst->ne[1] * dst->ne[2]; - uint32_t max_active_experts = std::min((uint32_t) src0->ne[2], total_gathered); - uint32_t max_wg_n = CEIL_DIV(total_gathered, tile_n_s) + max_active_experts; - uint32_t total_wg = wg_m * max_wg_n; + const uint32_t tile_m_s = main_decisions->tile_m * main_decisions->wg_size_m; + const uint32_t wg_m = CEIL_DIV(dst->ne[0], tile_m_s); + const uint32_t total_wg = wg_m * ggml_webgpu_mul_mat_id_max_tiles(dst); compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); @@ -1952,7 +2017,8 @@ struct ggml_webgpu_flash_attn_op { static bool ggml_webgpu_flash_attn_use_vec_path(const webgpu_global_context & global_ctx, const ggml_tensor * Q, const ggml_tensor * K, - const ggml_tensor * V) { + const ggml_tensor * V, + int64_t n_q) { const size_t storage_offset_alignment = global_ctx->capabilities.limits.minStorageBufferOffsetAlignment; const bool k_float_vec4_aligned = (K->type != GGML_TYPE_F16 && K->type != GGML_TYPE_F32) || @@ -1966,7 +2032,7 @@ static bool ggml_webgpu_flash_attn_use_vec_path(const webgpu_global_context & gl ggml_is_quantized(V->type) ? ggml_blck_size(V->type) : GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH; const bool kv_vec_head_dims_aligned = Q->ne[0] % k_vec_head_align == 0 && V->ne[0] % v_vec_head_align == 0; - return global_ctx->capabilities.supports_subgroups && (Q->ne[1] < GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN) && + return global_ctx->capabilities.supports_subgroups && (n_q < GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN) && kv_vec_head_dims_aligned && k_float_vec4_aligned && v_float_vec4_aligned; } @@ -2275,7 +2341,7 @@ static webgpu_encoded_op ggml_webgpu_flash_attn(webgpu_context & ctx, ggml_tensor * sinks, ggml_tensor * dst) { ggml_webgpu_flash_attn_op op = ggml_webgpu_flash_attn_prepare(ctx, Q, K, V, mask, sinks, dst); - if (ggml_webgpu_flash_attn_use_vec_path(ctx->global_ctx, Q, K, V)) { + if (ggml_webgpu_flash_attn_use_vec_path(ctx->global_ctx, Q, K, V, Q->ne[1])) { return ggml_webgpu_flash_attn_vec(ctx, Q, K, V, mask, sinks, dst, std::move(op)); } return ggml_webgpu_flash_attn_direct(ctx, op); @@ -2347,7 +2413,8 @@ static webgpu_encoded_op ggml_webgpu_unary_op(webgpu_context & ctx, ggml_tensor entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst)); } - uint32_t wg_x, wg_y; + uint32_t wg_x; + uint32_t wg_y; uint32_t total_wg = CEIL_DIV(ggml_nelements(dst), decisions->wg_size); compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); @@ -2425,7 +2492,8 @@ static webgpu_encoded_op ggml_webgpu_binary_op(webgpu_context & ctx, } } - uint32_t wg_x, wg_y; + uint32_t wg_x; + uint32_t wg_y; uint32_t total_wg = CEIL_DIV(ggml_nelements(dst), decisions->wg_size); compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); @@ -2556,8 +2624,11 @@ static webgpu_encoded_op ggml_webgpu_concat(webgpu_context & ctx, entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, dst)); } - uint32_t wg_x = CEIL_DIV(ne, decisions->wg_size); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_repeat(webgpu_context & ctx, ggml_tensor * src0, ggml_tensor * dst) { @@ -2591,8 +2662,12 @@ static webgpu_encoded_op ggml_webgpu_repeat(webgpu_context & ctx, ggml_tensor * webgpu_pipeline pipeline = ctx->shader_lib->get_repeat_pipeline(shader_lib_ctx); auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get()); - uint32_t wg_x = CEIL_DIV(ne, decisions->wg_size); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static std::optional<webgpu_encoded_op> ggml_webgpu_rms_norm_mul(webgpu_context & ctx, @@ -2637,6 +2712,7 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_rms_norm_mul(webgpu_context (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], + (uint32_t) dst->ne[3], ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(rn_dst, 0)) // epsilon, treated as f32 in the shader }; @@ -2676,7 +2752,52 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_rms_norm_mul(webgpu_context entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, dst)); } - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, ggml_nrows(dst)); + uint32_t wg_x; + uint32_t wg_y; + compute_2d_workgroups(ggml_nrows(dst), ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, + wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); +} + +static webgpu_encoded_op ggml_webgpu_group_norm(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { + const size_t src_type_size = ggml_type_size(src->type); + const size_t dst_type_size = ggml_type_size(dst->type); + const uint32_t num_groups = (uint32_t) ggml_get_op_params_i32(dst, 0); + std::vector<uint32_t> params = { + (uint32_t) src->ne[0], + (uint32_t) src->ne[1], + (uint32_t) src->ne[2], + (uint32_t) src->ne[3], + num_groups, + ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 1)), + (uint32_t) (src->nb[1] / src_type_size), + (uint32_t) (src->nb[0] / src_type_size), + (uint32_t) (src->nb[2] / src_type_size), + (uint32_t) (src->nb[3] / src_type_size), + (uint32_t) (dst->nb[1] / dst_type_size), + (uint32_t) (dst->nb[0] / dst_type_size), + (uint32_t) (dst->nb[2] / dst_type_size), + (uint32_t) (dst->nb[3] / dst_type_size), + (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / src_type_size), + (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / dst_type_size), + }; + + ggml_webgpu_shader_lib_context shader_lib_ctx = {}; + shader_lib_ctx.src0 = src; + shader_lib_ctx.dst = dst; + shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + webgpu_pipeline pipeline = ctx->shader_lib->get_group_norm_pipeline(shader_lib_ctx); + auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get()); + + std::vector<wgpu::BindGroupEntry> entries = { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src) }; + if (!decisions->inplace) { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst)); + } + uint32_t wg_x; + uint32_t wg_y; + compute_2d_workgroups(num_groups * (uint32_t) src->ne[3], + ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_row_norm(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { @@ -2692,6 +2813,7 @@ static webgpu_encoded_op ggml_webgpu_row_norm(webgpu_context & ctx, ggml_tensor (uint32_t) src->ne[0], (uint32_t) src->ne[1], (uint32_t) src->ne[2], + (uint32_t) src->ne[3], ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 0)) // epsilon, treated as f32 in the shader }; @@ -2707,7 +2829,12 @@ static webgpu_encoded_op ggml_webgpu_row_norm(webgpu_context & ctx, ggml_tensor if (!decisions->inplace) { entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst)); } - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, ggml_nrows(src)); + + uint32_t wg_x; + uint32_t wg_y; + compute_2d_workgroups(ggml_nrows(src), ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, + wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx, @@ -2753,7 +2880,8 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx, float theta_scale = powf(freq_base, -2.0f / n_dims); float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(dst), + corr_dims); std::vector<uint32_t> params = { (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)), @@ -2796,8 +2924,11 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx, entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, dst_binding, dst)); } - uint32_t wg_x = CEIL_DIV(ggml_nelements(dst), decisions->wg_size); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ggml_nelements(dst), decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_glu(webgpu_context & ctx, @@ -2870,8 +3001,11 @@ static webgpu_encoded_op ggml_webgpu_glu(webgpu_context & ctx, } entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, dst_binding, dst)); - uint32_t wg_x = CEIL_DIV(ggml_nelements(dst), decisions->wg_size); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ggml_nelements(dst), decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_scale(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { @@ -2909,7 +3043,8 @@ static webgpu_encoded_op ggml_webgpu_scale(webgpu_context & ctx, ggml_tensor * s entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst)); } - uint32_t wg_x, wg_y; + uint32_t wg_x; + uint32_t wg_y; uint32_t total_wg = CEIL_DIV(ggml_nelements(dst), decisions->wg_size); compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); @@ -2982,7 +3117,11 @@ static webgpu_encoded_op ggml_webgpu_soft_max(webgpu_context & ctx, entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, binding_num, dst)); } - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, ggml_nrows(dst)); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = ggml_nrows(dst); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_argmax(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { @@ -3003,6 +3142,52 @@ static webgpu_encoded_op ggml_webgpu_argmax(webgpu_context & ctx, ggml_tensor * return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); } +static webgpu_encoded_op ggml_webgpu_count_equal(webgpu_context & ctx, + ggml_tensor * src0, + ggml_tensor * src1, + ggml_tensor * dst) { + const uint32_t ne = ggml_nelements(src0); + const uint32_t offset_dst = ggml_webgpu_tensor_misalignment(ctx, dst); + std::vector<webgpu_dispatch_desc> dispatches; + // Reset both words of the I64 result before accumulating partial counts. + dispatches.push_back({ + ctx->global_ctx->memset_pipeline, + { offset_dst, sizeof(int64_t), 0 }, + { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, dst) }, + { 1, 1 } + }); + + if (ne == 0) { + return ggml_backend_webgpu_build_multi(ctx, dispatches); + } + + const bool src_overlap = ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src0, src1); + webgpu_pipeline pipeline = ctx->shader_lib->get_count_equal_pipeline(src_overlap); + + uint32_t offset_src0 = ggml_webgpu_tensor_misalignment(ctx, src0) / sizeof(int32_t); + uint32_t offset_src1 = ggml_webgpu_tensor_misalignment(ctx, src1) / sizeof(int32_t); + std::vector<wgpu::BindGroupEntry> entries; + if (src_overlap) { + const ggml_webgpu_merged_binding_range range = ggml_webgpu_tensor_merged_binding_range(ctx, { src0, src1 }); + offset_src0 = ggml_webgpu_tensor_merged_element_offset(src0, range); + offset_src1 = ggml_webgpu_tensor_merged_element_offset(src1, range); + entries.push_back(ggml_webgpu_make_bind_group_entry(0, ggml_webgpu_tensor_buf(src0), range.offset, range.size)); + } else { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0)); + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, src1)); + } + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, entries.size(), dst)); + std::vector<uint32_t> params = { offset_src0, offset_src1, offset_dst / (uint32_t) sizeof(uint32_t), ne }; + uint32_t wg_x; + uint32_t wg_y; + const uint32_t total_wg = CEIL_DIV(ne, GGML_WEBGPU_COUNT_EQUAL_WG_SIZE); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + dispatches.push_back({ + pipeline, std::move(params), std::move(entries), { wg_x, wg_y } + }); + return ggml_backend_webgpu_build_multi(ctx, dispatches); +} + static webgpu_encoded_op ggml_webgpu_argsort(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { bool is_top_k = dst->op == GGML_OP_TOP_K; @@ -3165,8 +3350,11 @@ static webgpu_encoded_op ggml_webgpu_cumsum(webgpu_context & ctx, ggml_tensor * shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; webgpu_pipeline pipeline = ctx->shader_lib->get_cumsum_pipeline(shader_lib_ctx); - uint32_t wg_x = ggml_nrows(dst); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = ggml_nrows(dst); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static webgpu_encoded_op ggml_webgpu_sum_rows(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { @@ -3178,7 +3366,8 @@ static webgpu_encoded_op ggml_webgpu_sum_rows(webgpu_context & ctx, ggml_tensor total_sum ? 0 : (uint32_t) (src->nb[3] / ggml_type_size(src->type)), total_sum ? static_cast<uint32_t>(ggml_nelements(src)) : (uint32_t) src->ne[0], total_sum ? 1 : (uint32_t) src->ne[1], - total_sum ? 1 : (uint32_t) src->ne[2] }; + total_sum ? 1 : (uint32_t) src->ne[2], + total_sum ? 1 : (uint32_t) ggml_nrows(dst) }; std::vector<wgpu::BindGroupEntry> entries = { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src), ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst) }; @@ -3190,8 +3379,11 @@ static webgpu_encoded_op ggml_webgpu_sum_rows(webgpu_context & ctx, ggml_tensor webgpu_pipeline pipeline = ctx->shader_lib->get_sum_rows_pipeline(shader_lib_ctx); - uint32_t wg_x = total_sum ? 1 : ggml_nrows(dst); - return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = total_sum ? 1 : ggml_nrows(dst); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } static bool ggml_webgpu_can_fuse_rms_norm_mul(const struct ggml_cgraph * cgraph, int node_idx) { @@ -3356,6 +3548,7 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_encode(webgpu_context ctx, case GGML_OP_TRANSPOSE: case GGML_OP_RESHAPE: return std::nullopt; + case GGML_OP_DUP: case GGML_OP_CPY: case GGML_OP_CONT: return ggml_webgpu_cpy(ctx, src0, node); @@ -3382,6 +3575,8 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_encode(webgpu_context ctx, return ggml_webgpu_concat(ctx, src0, src1, node); case GGML_OP_REPEAT: return ggml_webgpu_repeat(ctx, src0, node); + case GGML_OP_GROUP_NORM: + return ggml_webgpu_group_norm(ctx, src0, node); case GGML_OP_RMS_NORM: if (ggml_webgpu_can_fuse_rms_norm_mul(cgraph, node_idx)) { num_encoded_ops = 2; @@ -3428,8 +3623,13 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_encode(webgpu_context ctx, return ggml_webgpu_gated_delta_net(ctx, src0, src1, src2, node->src[3], node->src[4], node->src[5], node, nullptr); } + case GGML_OP_ARANGE: + return ggml_webgpu_arange(ctx, node); + case GGML_OP_PAD_REFLECT_1D: case GGML_OP_PAD: return ggml_webgpu_pad(ctx, src0, node); + case GGML_OP_COUNT_EQUAL: + return ggml_webgpu_count_equal(ctx, src0, src1, node); case GGML_OP_ARGMAX: return ggml_webgpu_argmax(ctx, src0, node); case GGML_OP_ARGSORT: @@ -3440,6 +3640,7 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_encode(webgpu_context ctx, return ggml_webgpu_cumsum(ctx, src0, node); case GGML_OP_SUM: case GGML_OP_SUM_ROWS: + case GGML_OP_MEAN: return ggml_webgpu_sum_rows(ctx, src0, node); case GGML_OP_CONV_2D: return ggml_webgpu_conv_2d(ctx, src0, src1, node); @@ -3807,6 +4008,112 @@ static void ggml_backend_webgpu_buffer_set_tensor(ggml_backend_buffer_t buffer, WEBGPU_CPU_PROFILE_TOTAL_END(set_tensor, buf_ctx->global_ctx); } +static void ggml_backend_webgpu_buffer_set_tensor_n(ggml_backend_buffer_t buffer, + struct ggml_tensor * const * tensors, + const void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + WEBGPU_CPU_PROFILE_TOTAL_START(set_tensor_n); + ggml_backend_webgpu_buffer_context * buf_ctx = (ggml_backend_webgpu_buffer_context *) buffer->context; + webgpu_global_context & ctx = buf_ctx->global_ctx; + + WEBGPU_LOG_DEBUG("ggml_backend_webgpu_buffer_set_tensor_n(" << buf_ctx->label << ", " << (const void *) tensors + << ", " << (const void *) data << ", " << offsets + << ", " << sizes << ", " << n_tensors << ")"); + + const size_t max_binding_size = ctx->capabilities.limits.maxStorageBufferBindingSize; + const size_t max_wg_per_dim = ctx->capabilities.limits.maxComputeWorkgroupsPerDimension; + + std::lock_guard<std::recursive_mutex> lock(ctx->mutex); + + std::vector<uint8_t> src; // we use src to copy the data of many tensors + std::vector<uint32_t> descs; // [dst offset in bytes, src offset in words, size in bytes, unused] + wgpu::Buffer dst_buf; + + auto grow_buffer = [&ctx](wgpu::Buffer & buf, size_t size, const char * label) { + if (buf == nullptr || buf.GetSize() < size) { + if (buf) { + buf.Destroy(); + } + ggml_webgpu_create_buffer(ctx->device, buf, size, wgpu::BufferUsage::CopyDst | wgpu::BufferUsage::Storage, + label); + } + }; + + auto flush_tensors = [&]() { + if (descs.empty()) { + return; + } + + src.resize(ROUNDUP_POW2(src.size(), 4)); + const size_t descs_size = descs.size() * sizeof(uint32_t); + const size_t n_dsts = descs.size() / 4; + + grow_buffer(ctx->set_tensor_n_src_buf, src.size(), "set_tensor_n_src_buf"); + grow_buffer(ctx->set_tensor_n_descs_buf, descs_size, "set_tensor_n_descs_buf"); + ctx->queue.WriteBuffer(ctx->set_tensor_n_src_buf, 0, src.data(), src.size()); + ctx->queue.WriteBuffer(ctx->set_tensor_n_descs_buf, 0, descs.data(), descs_size); + + std::vector<wgpu::BindGroupEntry> entries = { + ggml_webgpu_make_bind_group_entry(0, dst_buf, 0, dst_buf.GetSize()), + ggml_webgpu_make_bind_group_entry(1, ctx->set_tensor_n_src_buf, 0, src.size()), + ggml_webgpu_make_bind_group_entry(2, ctx->set_tensor_n_descs_buf, 0, descs_size), + }; + + wgpu::BindGroupDescriptor bind_group_desc; + bind_group_desc.layout = ctx->set_tensor_n_pipeline.pipeline.GetBindGroupLayout(0); + bind_group_desc.entryCount = entries.size(); + bind_group_desc.entries = entries.data(); + bind_group_desc.label = ctx->set_tensor_n_pipeline.name.c_str(); + wgpu::BindGroup bind_group = ctx->device.CreateBindGroup(&bind_group_desc); + + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = n_dsts; + compute_2d_workgroups(total_wg, max_wg_per_dim, wg_x, wg_y); + + wgpu::CommandEncoder encoder = ctx->device.CreateCommandEncoder(); + wgpu::ComputePassEncoder pass = encoder.BeginComputePass(); + pass.SetPipeline(ctx->set_tensor_n_pipeline.pipeline); + pass.SetBindGroup(0, bind_group); + pass.DispatchWorkgroups(wg_x, wg_y, 1); + pass.End(); + + wgpu::CommandBuffer commands = encoder.Finish(); + ctx->queue.Submit(1, &commands); + + src.clear(); + descs.clear(); + }; + + if (n_tensors > 0) { + dst_buf = ggml_webgpu_tensor_buf(tensors[0]); + } + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + wgpu::Buffer buf = ggml_webgpu_tensor_buf(tensors[i]); + size_t offset = ggml_webgpu_tensor_offset(tensors[i]) + offsets[i]; + if (!descs.empty() && (dst_buf.Get() != buf.Get() || src.size() + 4 + sizes[i] > max_binding_size || + (descs.size() + 4) * sizeof(uint32_t) > max_binding_size)) { + flush_tensors(); + } + dst_buf = buf; + + // Place the data at the same in-word byte lane as its destination + size_t src_offset = ROUNDUP_POW2(src.size(), 4) + offset % 4; + src.resize(src_offset + sizes[i]); + memcpy(src.data() + src_offset, data[i], sizes[i]); + descs.insert(descs.end(), { (uint32_t) offset, (uint32_t) (src_offset / 4), (uint32_t) sizes[i], 0 }); + } + flush_tensors(); + + WEBGPU_CPU_PROFILE_TOTAL_END(set_tensor_n, ctx); +} + static void ggml_backend_webgpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, @@ -3867,6 +4174,117 @@ static void ggml_backend_webgpu_buffer_get_tensor(ggml_backend_buffer_t buffer, WEBGPU_CPU_PROFILE_TOTAL_END(get_tensor, buf_ctx->global_ctx); } +static void ggml_backend_webgpu_buffer_get_tensor_n(ggml_backend_buffer_t buffer, + const struct ggml_tensor * const * tensors, + void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + WEBGPU_CPU_PROFILE_TOTAL_START(get_tensor_n); + ggml_backend_webgpu_buffer_context * buf_ctx = (ggml_backend_webgpu_buffer_context *) buffer->context; + WEBGPU_LOG_DEBUG("ggml_backend_webgpu_buffer_get_tensor_n(" << buf_ctx->label << ", " << (const void *) tensors + << ", " << (const void *) data << ", " << offsets + << ", " << sizes << ", " << n_tensors << ")"); + webgpu_global_context global_ctx = buf_ctx->global_ctx; + wgpu::Device device = global_ctx->device; + + typedef struct { + size_t staging_size; + size_t align_offset; + size_t local_offset; + } tensor_desc; + + std::vector<tensor_desc> t_descs; + std::vector<std::pair<size_t, size_t>> + staging_chunk_descs; // pair (boundary id, sum of the tensor size to the bounary id) + t_descs.reserve(n_tensors); + + size_t sum_tensor_size = 0; + for (size_t i = 0; i < n_tensors; i++) { + const ggml_tensor * tensor = tensors[i]; + tensor_desc td; + + td.align_offset = ggml_webgpu_tensor_offset(tensor) + offsets[i]; + + td.local_offset = td.align_offset % 4; + if (td.local_offset != 0) { + // If offset is not a multiple of 4, we need to round it down to the previous + // multiple of 4 + td.align_offset -= td.local_offset; + } + + td.staging_size = sizes[i] + td.local_offset; + if (td.staging_size % 4 != 0) { + // If size is not a multiple of 4, we need to round it up to the next + // multiple of 4 + td.staging_size += 4 - (td.staging_size % 4); + } + + t_descs.push_back(td); + + if (sum_tensor_size + td.staging_size > global_ctx->capabilities.limits.maxBufferSize) { + if (i == 0) { + GGML_ABORT("it is not expected that one tensor size exceed the one wgpu::Buffer limit size."); + } + staging_chunk_descs.push_back({ i - 1, sum_tensor_size }); + sum_tensor_size = td.staging_size; + } else { + sum_tensor_size += td.staging_size; + } + } + staging_chunk_descs.push_back({ n_tensors - 1, sum_tensor_size }); + + std::lock_guard<std::recursive_mutex> lock(global_ctx->mutex); + + size_t tsi = 0; + for (auto & [tei, sum_chunk_size] : staging_chunk_descs) { + if (global_ctx->get_tensor_staging_buf == nullptr || + global_ctx->get_tensor_staging_buf.GetSize() < sum_chunk_size) { + // Create a new staging buffer if it doesn't exist or is too small + if (global_ctx->get_tensor_staging_buf) { + global_ctx->get_tensor_staging_buf.Destroy(); + } + ggml_webgpu_create_buffer(device, global_ctx->get_tensor_staging_buf, sum_chunk_size, + wgpu::BufferUsage::CopyDst | wgpu::BufferUsage::MapRead, + "get_tensor_staging_buf"); + } + + wgpu::CommandEncoder encoder = device.CreateCommandEncoder(); + + // Copy the data of the each tensor from the buffer to the staging buffer + size_t i_offset = 0; + for (size_t i = tsi; i <= tei; i++) { + encoder.CopyBufferToBuffer(ggml_webgpu_tensor_buf(tensors[i]), t_descs[i].align_offset, + global_ctx->get_tensor_staging_buf, i_offset, t_descs[i].staging_size); + i_offset += t_descs[i].staging_size; + } + + wgpu::CommandBuffer commands = encoder.Finish(); + + // Submit the command buffer to the queue + global_ctx->queue.Submit(1, &commands); + + // Map the staging buffer to read the data + ggml_backend_webgpu_map_buffer(global_ctx, global_ctx->get_tensor_staging_buf, wgpu::MapMode::Read, 0, + sum_chunk_size); + // Must specify size here since the staging buffer might be larger than the tensor size + const void * mapped_range = global_ctx->get_tensor_staging_buf.GetConstMappedRange(0, sum_chunk_size); + + i_offset = 0; + for (size_t i = tsi; i <= tei; i++) { + // Copy the data from the mapped range to the output buffer + std::memcpy(data[i], (const void *) ((const char *) mapped_range + i_offset + t_descs[i].local_offset), + sizes[i]); + i_offset += t_descs[i].staging_size; + } + + tsi = tei + 1; + global_ctx->get_tensor_staging_buf.Unmap(); + } + + WEBGPU_CPU_PROFILE_TOTAL_END(get_tensor_n, buf_ctx->global_ctx); +} + static void ggml_backend_webgpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { WEBGPU_LOG_DEBUG("ggml_backend_webgpu_buffer_clear(" << buffer << ", " << (uint32_t) value << ")"); WEBGPU_CPU_PROFILE_TOTAL_START(clear); @@ -3887,7 +4305,9 @@ static ggml_backend_buffer_i ggml_backend_webgpu_buffer_interface = { /* .cpy_tensor = */ NULL, // TODO: optional, implement this /* .clear = */ ggml_backend_webgpu_buffer_clear, /* .reset = */ NULL, // TODO: optional, think it coordinates with - // .init_tensor + /* .set_tensor_n = */ ggml_backend_webgpu_buffer_set_tensor_n, + /* .get_tensor_n = */ ggml_backend_webgpu_buffer_get_tensor_n, + /* .cpy_tensor_n = */ NULL, }; /* End GGML Backend Buffer Interface */ @@ -3924,12 +4344,10 @@ static size_t ggml_backend_webgpu_buffer_type_get_alignment(ggml_backend_buffer_ return dev_ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment; } -// maxBufferSize might be larger, but you can't bind more than -// maxStorageBufferBindingSize to a single binding. static size_t ggml_backend_webgpu_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { ggml_backend_webgpu_device_context * dev_ctx = static_cast<ggml_backend_webgpu_device_context *>(buft->device->context); - return dev_ctx->webgpu_global_ctx->capabilities.limits.maxStorageBufferBindingSize; + return dev_ctx->webgpu_global_ctx->capabilities.limits.maxBufferSize; } static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, @@ -3959,7 +4377,10 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer const ggml_tensor * V = tensor->src[2]; const ggml_tensor * mask = tensor->src[3]; const auto & capabilities = ctx->webgpu_global_ctx->capabilities; - if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V)) { + // the vec path scratch grows with the query rows, so a longer batch reserves the scratch of the + // longest vec batch and the size stays monotonic in the batch length + const int64_t n_q = std::min<int64_t>(Q->ne[1], GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN - 1); + if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V, n_q)) { const bool kv_direct = ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH) || ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH); @@ -3971,7 +4392,7 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer uint32_t nwg = ggml_webgpu_flash_attn_vec_nwg(vec_nwg_cap, kv_tile, (uint32_t) K->ne[1]); const size_t align = capabilities.limits.minStorageBufferOffsetAlignment; - const uint64_t nrows = (uint64_t) Q->ne[1] * Q->ne[2] * Q->ne[3]; + const uint64_t nrows = (uint64_t) n_q * Q->ne[2] * Q->ne[3]; if (nwg > 1u) { const uint64_t tmp_data_elems = nrows * (uint64_t) V->ne[0] * nwg; const uint64_t tmp_stats_elems = nrows * 2u * nwg; @@ -3983,7 +4404,7 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer } if (mask != nullptr) { const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile); - const uint32_t blk_nblk1 = CEIL_DIV((uint32_t) Q->ne[1], 1u); + const uint32_t blk_nblk1 = (uint32_t) n_q; const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type)); const uint32_t blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u; const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count; @@ -3999,11 +4420,15 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer { const ggml_tensor * src0 = tensor->src[0]; const ggml_tensor * src1 = tensor->src[1]; - bool use_mmvq = - ggml_webgpu_can_use_mmvq(src0, src1, ctx->webgpu_global_ctx->capabilities.supports_dot_product, - ctx->webgpu_global_ctx->vendor); + + // the q8 copy of src1 grows with its rows, so a longer batch reserves the copy of the longest + // mmvq batch and the size stays monotonic in the batch length + const int64_t n_rows = std::min<int64_t>(src1->ne[1], GGML_WEBGPU_MUL_MAT_VEC_MAX_ROWS); + const bool use_mmvq = ggml_webgpu_can_use_mmvq( + src0, src1, n_rows, ctx->webgpu_global_ctx->capabilities.supports_dot_product, + ctx->webgpu_global_ctx->vendor); if (use_mmvq) { - const size_t q8_src1_size = src1->ne[3] * src1->ne[2] * src1->ne[1] * + const size_t q8_src1_size = src1->ne[3] * src1->ne[2] * n_rows * (36 /* sizeof(q8_1) */ * (src1->ne[0] / /* block_size */ 32)); res = ROUNDUP_POW2(res + q8_src1_size + ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment, @@ -4016,11 +4441,11 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer const ggml_tensor * src0 = tensor->src[0]; const ggml_tensor * src1 = tensor->src[1]; if (src0 && src1) { - const size_t gathered_size = sizeof(uint32_t) * tensor->src[0]->ne[2] * tensor->src[1]->ne[2]; - const size_t gathered_count_ids_size = sizeof(uint32_t) * tensor->src[0]->ne[2]; - res = ROUNDUP_POW2( - res + gathered_size * 2 + gathered_count_ids_size + - ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment * 3, + const size_t ids_cache_size = sizeof(uint32_t) * tensor->ne[1] * tensor->ne[2]; + const size_t ids_tiles_size = ggml_webgpu_mul_mat_id_tiles_nbytes(tensor); + res = ROUNDUP_POW2( + res + ids_cache_size + ids_tiles_size + + ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment * 2, WEBGPU_STORAGE_BUF_BINDING_MULT); } } @@ -4087,24 +4512,23 @@ static ggml_guid_t ggml_backend_webgpu_guid(void) { } static void ggml_webgpu_init_memset_pipeline(webgpu_global_context & ctx) { - // we use the maximum workgroup size for the memset pipeline - size_t max_threads = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * - ctx->capabilities.limits.maxComputeWorkgroupsPerDimension; - // Size the bytes_per_thread so that the largest buffer size can be handled - ctx->capabilities.memset_bytes_per_thread = - CEIL_DIV(ctx->capabilities.limits.maxStorageBufferBindingSize, max_threads); - std::vector<wgpu::ConstantEntry> constants(2); + std::vector<wgpu::ConstantEntry> constants(1); constants[0].key = "wg_size"; constants[0].value = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; - constants[1].key = "bytes_per_thread"; - constants[1].value = ctx->capabilities.memset_bytes_per_thread; ctx->memset_pipeline = ggml_webgpu_create_pipeline(ctx->device, wgsl_memset, "memset", constants); } +static void ggml_webgpu_init_scatter_write_pipeline(webgpu_global_context & ctx) { + std::vector<wgpu::ConstantEntry> constants(1); + constants[0].key = "wg_size"; + constants[0].value = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + ctx->set_tensor_n_pipeline = ggml_webgpu_create_pipeline(ctx->device, wgsl_set_tensor_n, "set_tensor_n", constants); +} + static void ggml_backend_webgpu_request_adapter(wgpu::Instance & instance, wgpu::Adapter & adapter) { wgpu::RequestAdapterOptions options = {}; -#ifndef __EMSCRIPTEN__ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) // TODO: track need for these toggles: https://issues.chromium.org/issues/42251215 const char * const adapterEnabledToggles[] = { "vulkan_enable_f16_on_nvidia", "use_vulkan_memory_model" }; wgpu::DawnTogglesDescriptor adapterTogglesDesc; @@ -4133,7 +4557,7 @@ static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { ctx->webgpu_global_ctx->adapter.GetLimits(&ctx->webgpu_global_ctx->capabilities.limits); wgpu::AdapterInfo info{}; -#ifndef __EMSCRIPTEN__ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) wgpu::AdapterPropertiesSubgroupMatrixConfigs subgroup_matrix_configs{}; if (ctx->webgpu_global_ctx->adapter.HasFeature(wgpu::FeatureName::ChromiumExperimentalSubgroupMatrix)) { info.nextInChain = &subgroup_matrix_configs; @@ -4150,7 +4574,7 @@ static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { wgpu::WGSLLanguageFeatureName::Packed4x8IntegerDotProduct); bool valid_subgroup_matrix_config = false; -#ifndef __EMSCRIPTEN__ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) // Accept f16 subgroup matrix configurations (square or non-square). // NVIDIA GPUs typically report square configs (e.g. 16x16x16), // while Intel Xe2 GPUs report non-square configs (e.g. 8x16x16). @@ -4178,7 +4602,7 @@ static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { // Initialize device std::vector<wgpu::FeatureName> required_features = { wgpu::FeatureName::ShaderF16 }; -#ifndef __EMSCRIPTEN__ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) required_features.push_back(wgpu::FeatureName::ImplicitDeviceSynchronization); if (ctx->webgpu_global_ctx->capabilities.supports_subgroup_matrix) { required_features.push_back(wgpu::FeatureName::ChromiumExperimentalSubgroupMatrix); @@ -4214,7 +4638,7 @@ static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { std::string(message).c_str()); }); -#ifndef __EMSCRIPTEN__ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) // Enable Dawn-specific toggles to increase native performance // TODO: Maybe WebGPU needs a "fast" mode where you can request compilers skip adding checks like these, // only for native performance? @@ -4244,6 +4668,7 @@ static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { GGML_ASSERT(ctx->webgpu_global_ctx->device != nullptr); ggml_webgpu_init_memset_pipeline(ctx->webgpu_global_ctx); + ggml_webgpu_init_scatter_write_pipeline(ctx->webgpu_global_ctx); ggml_webgpu_create_buffer(ctx->webgpu_global_ctx->device, ctx->webgpu_global_ctx->memset_params_buf, WEBGPU_PARAMS_BUF_SIZE_BYTES, wgpu::BufferUsage::CopyDst | wgpu::BufferUsage::Uniform, "memset_params_buf"); @@ -4350,6 +4775,7 @@ static bool ggml_backend_webgpu_device_supports_buft(ggml_backend_dev_t dev, ggm static bool ggml_webgpu_supported_qtype(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4419,6 +4845,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const supports_op = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_I32 || src0->type == GGML_TYPE_I16); break; + case GGML_OP_DUP: case GGML_OP_CPY: case GGML_OP_CONT: supports_op = (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32) && @@ -4454,6 +4881,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4483,6 +4911,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const default: break; } + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + supports_op = false; + } break; } case GGML_OP_MUL_MAT_ID: @@ -4526,7 +4957,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const default: break; } - if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { supports_op = false; } break; @@ -4534,7 +4965,8 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const { // conservative support checks for whether the more resource-intensive shader paths // can be used, to avoid cases where flash_attn is assigned to the CPU later on - supports_op = src0->type == GGML_TYPE_F32 && + // K/V row lists (ggml_flash_attn_ext_rows) are not supported + supports_op = op->src[5] == nullptr && src0->type == GGML_TYPE_F32 && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_Q4_0 || src1->type == GGML_TYPE_Q8_0) && (src2->type == GGML_TYPE_F32 || src2->type == GGML_TYPE_F16 || @@ -4544,7 +4976,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const break; } if (ggml_webgpu_tensor_binding_overlap(ctx->webgpu_global_ctx, src1, src2) && - src1->type != src2->type && !ggml_is_quantized(src1->type) && !ggml_is_quantized(src2->type)) { + src1->type != src2->type && !(ggml_is_quantized(src1->type) && ggml_is_quantized(src2->type))) { supports_op = false; break; } @@ -4592,6 +5024,18 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const supports_op = max_kv_tile > 0; break; } + case GGML_OP_GROUP_NORM: + { + bool valid_type = (src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) || + (src0->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16); + // One workgroup per normalization group within each batch item. + const uint64_t num_workgroups = uint64_t(ggml_get_op_params_i32(op, 0)) * uint64_t(src0->ne[3]); + const uint64_t max_per_dim = + ctx->webgpu_global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension; + const bool dispatch_fits = num_workgroups <= max_per_dim * max_per_dim; + supports_op = valid_type && dispatch_fits; + break; + } case GGML_OP_RMS_NORM: case GGML_OP_NORM: case GGML_OP_L2_NORM: @@ -4719,6 +5163,12 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const case GGML_OP_COS: supports_op = (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && (src0->type == op->type); break; + case GGML_OP_ARANGE: + supports_op = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + break; + case GGML_OP_PAD_REFLECT_1D: + supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32; + break; case GGML_OP_PAD: supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32; break; @@ -4735,7 +5185,14 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type; break; case GGML_OP_SUM: + supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguously_allocated(src0); + break; + case GGML_OP_COUNT_EQUAL: + supports_op = src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32 && ggml_is_contiguous(src0) && + ggml_is_contiguous(src1); + break; case GGML_OP_SUM_ROWS: + case GGML_OP_MEAN: supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguous_rows(src0); break; case GGML_OP_UPSCALE: @@ -4865,7 +5322,7 @@ ggml_backend_reg_t ggml_backend_webgpu_reg() { instance_descriptor.requiredFeatures = instance_features.data(); instance_descriptor.requiredFeatureCount = instance_features.size(); -#ifndef __EMSCRIPTEN__ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) const char * const instanceEnabledToggles[] = { "allow_unsafe_apis" }; wgpu::DawnTogglesDescriptor instanceTogglesDesc; instanceTogglesDesc.enabledToggles = instanceEnabledToggles; @@ -4885,7 +5342,7 @@ ggml_backend_reg_t ggml_backend_webgpu_reg() { // WebGPU backend requires f16 support and, on native, implicit device synchronization. if (adapter != nullptr && adapter.HasFeature(wgpu::FeatureName::ShaderF16) -#ifndef __EMSCRIPTEN__ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) && adapter.HasFeature(wgpu::FeatureName::ImplicitDeviceSynchronization) #endif ) { diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl index 2573926cb..1e377bff5 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl @@ -46,12 +46,24 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>, let i2 = wg_linear / params.ne1; let i1 = wg_linear % params.ne1; - let i11 = u32(ids[params.offset_ids + i1 * params.nb20 + i2 * params.nb21]); + let i11 = i32(ids[params.offset_ids + i1 * params.nb20 + i2 * params.nb21]); let src0_row = params.offset_src0 + i1 * params.nb01 + i2 * params.nb02; - let src1_row = params.offset_src1 + i11 * params.nb11; let dst_row = params.offset_dst + i1 * params.ne0 + i2 * (params.ne0 * params.ne1); + if (i11 == -1) { // skipped slot, add nothing +#ifdef INPLACE + // src0 already holds the result +#else + for (var i = thread_id;i < params.ne0; i += WG_SIZE) { + dst[dst_row + i] = src0[src0_row + i]; + } +#endif + return; + } + + let src1_row = params.offset_src1 + u32(i11) * params.nb11; + for (var i = thread_id;i < params.ne0; i += WG_SIZE) { #ifdef INPLACE src0[src0_row + i] = src0[src0_row + i] + src1[src1_row + i]; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl new file mode 100644 index 000000000..707a66536 --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl @@ -0,0 +1,22 @@ +@group(0) @binding(0) +var<storage, read_write> dst: array<f32>; + +struct Params { + ne: u32, + offset_dst: u32, + start: f32, + step: f32, +}; + +@group(0) @binding(1) +var<uniform> params: Params; + +@compute @workgroup_size(WG_SIZE) +fn main(@builtin(num_workgroups) num_wg: vec3<u32>, @builtin(global_invocation_id) gid: vec3<u32>) { + let i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (i >= params.ne) { + return; + } + + dst[params.offset_dst + i] = params.start + params.step * f32(i); +} diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl index 7ccad73f4..de6d6e6c7 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl @@ -53,10 +53,12 @@ var<storage, read_write> dst: array<DataType>; var<uniform> params: Params; #endif @compute @workgroup_size(WG_SIZE) -fn main(@builtin(global_invocation_id) gid: vec3<u32>) { +fn main(@builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(global_invocation_id) gid: vec3<u32>) { - if (gid.x < params.ne) { - var i = gid.x; + let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (gid_i < params.ne) { + var i = gid_i; let i3 = i / (params.ne2 * params.ne1 * params.ne0); i = i % (params.ne2 * params.ne1 * params.ne0); let i2 = i / (params.ne1 * params.ne0); @@ -72,9 +74,9 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) { ni[2] * params.stride_src0_2 + ni[3] * params.stride_src0_3; #ifdef SRC_OVERLAP - dst[params.offset_dst + gid.x] = merged_src[params.offset_src0 + src_i]; + dst[params.offset_dst + gid_i] = merged_src[params.offset_src0 + src_i]; #else - dst[params.offset_dst + gid.x] = src0[params.offset_src0 + src_i]; + dst[params.offset_dst + gid_i] = src0[params.offset_src0 + src_i]; #endif } else { ni[params.dim] -= params.src0_nedim; @@ -83,9 +85,9 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) { ni[2] * params.stride_src1_2 + ni[3] * params.stride_src1_3; #ifdef SRC_OVERLAP - dst[params.offset_dst + gid.x] = merged_src[params.offset_src1 + src_i]; + dst[params.offset_dst + gid_i] = merged_src[params.offset_src1 + src_i]; #else - dst[params.offset_dst + gid.x] = src1[params.offset_src1 + src_i]; + dst[params.offset_dst + gid_i] = src1[params.offset_src1 + src_i]; #endif } } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/count_equal.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/count_equal.wgsl new file mode 100644 index 000000000..1ab6a635b --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/count_equal.wgsl @@ -0,0 +1,57 @@ +struct Params { + offset_src0: u32, + offset_src1: u32, + offset_dst: u32, + ne: u32, +}; + +#ifdef SRC_OVERLAP +@group(0) @binding(0) +var<storage, read_write> merged_src: array<i32>; + +@group(0) @binding(1) +var<storage, read_write> dst: array<atomic<u32>>; + +@group(0) @binding(2) +var<uniform> params: Params; +#else +@group(0) @binding(0) +var<storage, read_write> src0: array<i32>; + +@group(0) @binding(1) +var<storage, read_write> src1: array<i32>; + +@group(0) @binding(2) +var<storage, read_write> dst: array<atomic<u32>>; + +@group(0) @binding(3) +var<uniform> params: Params; +#endif + +var<workgroup> shared_count: array<u32, WG_SIZE>; + +@compute @workgroup_size(WG_SIZE) +fn main(@builtin(workgroup_id) wid: vec3<u32>, + @builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(local_invocation_id) lid: vec3<u32>) { + var count = 0u; + let i = (wid.x + wid.y * num_wg.x) * WG_SIZE + lid.x; + if (i < params.ne) { +#ifdef SRC_OVERLAP + count += select(0u, 1u, merged_src[params.offset_src0 + i] == merged_src[params.offset_src1 + i]); +#else + count += select(0u, 1u, src0[params.offset_src0 + i] == src1[params.offset_src1 + i]); +#endif + } + shared_count[lid.x] = count; + workgroupBarrier(); + for (var offset: u32 = WG_SIZE >> 1; offset > 0; offset >>= 1) { + if (lid.x < offset) { + shared_count[lid.x] += shared_count[lid.x + offset]; + } + workgroupBarrier(); + } + if (lid.x == 0) { + atomicAdd(&dst[params.offset_dst], shared_count[0]); + } +} diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl index e622552c4..58c703dcf 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl @@ -17,8 +17,11 @@ var<workgroup> shared_sum: array<f32, WG_SIZE>; @compute @workgroup_size(WG_SIZE) fn main(@builtin(workgroup_id) wid: vec3<u32>, + @builtin(num_workgroups) num_wg: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) { - let row_idx = params.offset_src + wid.x * params.ne0; + + let wid_i = wid.x + wid.y * num_wg.x; + let row_idx = params.offset_src + wid_i * params.ne0; let elems = (params.ne0 + WG_SIZE - 1) / WG_SIZE; var local_sum: f32 = 0.0; for (var col = lid.x * elems; col < (lid.x + 1) * elems && col < params.ne0; col ++) { diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl index a3114a4be..4bf3c9347 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl @@ -51,6 +51,20 @@ fn copy_elements(src_base: u32, dst_base: u32, offset: u32) { } #endif +#ifdef Q2_0 +fn copy_elements(src_base: u32, dst_base: u32, offset: u32) { + let block_byte_base = (src_base + offset) * 18u; + let d = load_f16_as_f32_at_src(block_byte_base); + for (var j = 0u; j < 4u; j++) { + let q_packed = load_u32_at_src(block_byte_base + 2u + j * 4u); + for (var k = 0u; k < 16u; k++) { + let q = (q_packed >> (2u * k)) & 3u; + dst[dst_base + offset * 64u + j * 16u + k] = (f32(q) - 1.0) * d; + } + } +} +#endif + #ifdef Q4_0 fn copy_elements(src_base: u32, dst_base: u32, offset: u32) { let block_byte_base = (src_base + offset) * 18; // Block stride: 18 bytes diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl index 6bbed5d3b..6f1c367d5 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl @@ -157,12 +157,15 @@ fn b_value(base: u32) -> DataType { #endif @compute @workgroup_size(WG_SIZE) -fn main(@builtin(global_invocation_id) gid: vec3<u32>) { - if (gid.x >= params.ne) { +fn main(@builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(global_invocation_id) gid: vec3<u32>) { + + let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (gid_i >= params.ne) { return; } - var i = gid.x; + var i = gid_i; let i3 = i / (params.ne2 * params.ne1 * params.ne0); i = i % (params.ne2 * params.ne1 * params.ne0); let i2 = i / (params.ne1 * params.ne0); diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/group_norm.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/group_norm.wgsl new file mode 100644 index 000000000..a2dcd42c0 --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/group_norm.wgsl @@ -0,0 +1,154 @@ +#if defined(SRC_F16) || defined(DST_F16) +enable f16; +#endif + +#ifdef SRC_F16 +#define SRC_TYPE f16 +#else +#define SRC_TYPE f32 +#endif + +#ifdef DST_F16 +#define DST_TYPE f16 +#else +#define DST_TYPE f32 +#endif + +struct Params { + width: u32, + height: u32, + num_channels: u32, + batch_size: u32, + num_groups: u32, + epsilon: f32, + // input strides + row_stride: u32, + col_stride: u32, + channel_stride: u32, + batch_stride: u32, + // output strides + output_row_stride: u32, + output_col_stride: u32, + output_channel_stride: u32, + output_batch_stride: u32, + // offsets + input_offset: u32, + output_offset: u32, +}; + +@group(0) @binding(0) var<storage, read_write> input: array<SRC_TYPE>; +#ifdef INPLACE +@group(0) @binding(1) var<uniform> params: Params; +#else +@group(0) @binding(1) var<storage, read_write> output: array<DST_TYPE>; +@group(0) @binding(2) var<uniform> params: Params; +#endif + +var<workgroup> partials: array<f32, WG_SIZE>; +var<workgroup> mean: f32; +var<workgroup> variance: f32; + +fn input_index(logical_position: u32, group_base: u32) -> u32 { + let col: u32 = logical_position % params.width; + let row: u32 = (logical_position / params.width) % params.height; + let channel: u32 = (logical_position / params.width) / params.height; + return group_base + + (row * params.row_stride) + + (col * params.col_stride) + + (channel * params.channel_stride); +} + +fn output_index(logical_position: u32, group_base: u32) -> u32 { + let col: u32 = logical_position % params.width; + let row: u32 = (logical_position / params.width) % params.height; + let channel: u32 = (logical_position / params.width) / params.height; + return group_base + + (row * params.output_row_stride) + + (col * params.output_col_stride) + + (channel * params.output_channel_stride); +} + +@compute @workgroup_size(WG_SIZE) +fn main( + @builtin(workgroup_id) workgroup_id: vec3<u32>, + @builtin(num_workgroups) num_workgroups: vec3<u32>, + @builtin(local_invocation_index) lane: u32 +) { + if (params.num_groups == 0u) { + return; + } + let linear_group_id = workgroup_id.x + workgroup_id.y * num_workgroups.x; + let batch_index = linear_group_id / params.num_groups; + if (batch_index >= params.batch_size) { + return; + } + let group_index = linear_group_id % params.num_groups; + let channels_per_group = params.num_channels / params.num_groups + + select(0u, 1u, params.num_channels % params.num_groups != 0u); + if (group_index > params.num_channels / max(channels_per_group, 1u)) { + return; + } + let first_channel = group_index * channels_per_group; + if (first_channel >= params.num_channels) { + return; + } + let group_channels = min(channels_per_group, params.num_channels - first_channel); + let num_elems_per_group = group_channels * params.width * params.height; + let batch_base = params.input_offset + batch_index * params.batch_stride; + let group_base = batch_base + first_channel * params.channel_stride; + + var partial_sum : f32 = 0.0f; + for (var i = lane; i < num_elems_per_group; i += WG_SIZE) { + partial_sum += f32(input[input_index(i, group_base)]); + } + partials[lane] = partial_sum; + + workgroupBarrier(); + + if (lane == 0) { + var total_sum : f32 = 0.0f; + for (var i: u32 = 0u; i < WG_SIZE; i++) { + total_sum += partials[i]; + } + mean = total_sum / f32(num_elems_per_group); + } + + workgroupBarrier(); + + var partial_sq_dev : f32 = 0.0f; + for (var i = lane; i < num_elems_per_group; i += WG_SIZE) { + let delta = f32(input[input_index(i, group_base)]) - mean; + partial_sq_dev += delta * delta; + } + partials[lane] = partial_sq_dev; + + workgroupBarrier(); + + if (lane == 0) { + var total_sum : f32 = 0.0f; + for (var i = 0u; i < WG_SIZE; i++) { + total_sum += partials[i]; + } + variance = total_sum / f32(num_elems_per_group); + } + + workgroupBarrier(); + + let scale = inverseSqrt(variance + params.epsilon); +#ifndef INPLACE + let output_group_base = params.output_offset + + batch_index * params.output_batch_stride + + first_channel * params.output_channel_stride; +#endif + for (var i = lane; i < num_elems_per_group; i += WG_SIZE) { + let index = input_index(i, group_base); + let normalized = (f32(input[index]) - mean) * scale; +#ifdef INPLACE + input[index] = SRC_TYPE(normalized); +#else + output[output_index(i, output_group_base)] = DST_TYPE(normalized); +#endif + } + + +} diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl index 194d2d6f5..af5a23bc0 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl @@ -11,30 +11,33 @@ struct Params { var<uniform> params: Params; override wg_size: u32; -override bytes_per_thread: u32; @compute @workgroup_size(wg_size) -fn main(@builtin(global_invocation_id) gid: vec3<u32>) { - let i = gid.x * bytes_per_thread; +fn main(@builtin(workgroup_id) wid: vec3<u32>, + @builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(local_invocation_id) lid: vec3<u32>) { + + let i = (wid.x + wid.y * num_wg.x) * wg_size + lid.x; let start = params.offset; let end = params.offset + params.size; + let id = (start >> 2u) + i; + let id_u32 = id * 4; + + if (id_u32 >= end) { + return; + } + + let lo = max(start, id_u32) - id_u32; + let hi = min(end, id_u32 + 4u) - id_u32; + var mask = 0xFFFFFFFFu << (8u * lo); + if (hi < 4u) { + mask &= (1u << (8u * hi)) - 1u; + } - for (var j: u32 = 0u; j < bytes_per_thread; j += 4) { - let byte_index = start + i + j; - if (byte_index + 4 <= end) { - output_buffer[byte_index >> 2] = params.value; - } else { - // Handle tail (unaligned) - for (var k: u32 = 0; k < 4; k++) { - let idx = byte_index + k; - if (idx < end) { - let word_idx = idx >> 2; - let bit_offset = (idx & 3) * 8u; - let mask = ~(0xffu << bit_offset); - let existing = output_buffer[word_idx]; - output_buffer[word_idx] = (existing & mask) | (params.value & (0xffu << bit_offset)); - } - } - } + if (mask == 0xFFFFFFFFu) { + output_buffer[id] = params.value; + } else { + let existing = output_buffer[id]; + output_buffer[id] = (existing & ~(mask)) | (params.value & mask); } } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl index 83f6fa172..1af6286ac 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl @@ -111,6 +111,29 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 } #endif // INIT_SRC0_SHMEM_Q1_0 +#ifdef INIT_SRC0_SHMEM_Q2_0 +const BLOCK_SIZE = 64u; +const BLOCK_SIZE_BYTES = 18u; +const NQ = 16u; + +fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) { + for (var i = thread_id * NQ; i < TILE_SRC0_SHMEM; i += TOTAL_WORKGROUP_SIZE * NQ) { + let global_m = offset_m + i / TILE_K; + let global_k = k_outer + i % TILE_K; + if (global_m < params.m && global_k < params.k) { + let block_idx = batch_offset + global_m * params.stride_01 + global_k / BLOCK_SIZE; + let block_byte_base = block_idx * BLOCK_SIZE_BYTES; + let d = load_f16_at_src0(block_byte_base); + let q_packed = load_u32_at_src0(block_byte_base + 2u + (global_k % BLOCK_SIZE) / 4u); + for (var k = 0u; k < NQ; k++) { + let q = (q_packed >> (2u * k)) & 3u; + shmem[i + k] = (f16(q) - 1.0) * d; + } + } + } +} +#endif // INIT_SRC0_SHMEM_Q2_0 + // legacy-quants #if defined(INIT_SRC0_SHMEM_Q4_0) || defined(INIT_SRC0_SHMEM_Q4_1) || defined(INIT_SRC0_SHMEM_Q5_0) || defined(INIT_SRC0_SHMEM_Q5_1) || defined(INIT_SRC0_SHMEM_Q8_0) || defined(INIT_SRC0_SHMEM_Q8_1) || defined(INIT_SRC0_SHMEM_MXFP4) const BLOCK_SIZE = 32u; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl index 91039ff25..a3c25f656 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl @@ -38,11 +38,11 @@ struct MulMatIdParams { @group(0) @binding(0) var<storage, read_write> src0: array<SRC0_TYPE>; // [cols, rows, n_expert] @group(0) @binding(1) var<storage, read_write> src1: array<SRC1_TYPE>; // [cols, b_ne1, n_tokens] @group(0) @binding(2) var<storage, read_write> dst: array<DST_TYPE>; // [rows, n_expert_used, n_tokens] -@group(0) @binding(3) var<storage, read_write> global_gathered_expert_used: array<u32>; // [n_expert][n_tokens] -@group(0) @binding(4) var<storage, read_write> global_gathered_tokens: array<u32>; // [n_expert][n_tokens] -@group(0) @binding(5) var<storage, read_write> gathered_count_ids: array<u32>; // [n_expert] -@group(0) @binding(6) var<uniform> params: MulMatIdParams; +@group(0) @binding(3) var<storage, read_write> ids_slot_pos: array<u32>; // [n_expert_used * n_tokens] +@group(0) @binding(4) var<storage, read_write> ids_tiles: array<u32>; // [n_tiles, (expert, first slot, n slots) * n_tiles] + +@group(0) @binding(5) var<uniform> params: MulMatIdParams; fn get_local_n(thread_id: u32) -> u32 { return thread_id / WORKGROUP_SIZE_M; @@ -90,44 +90,29 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>, let local_m = get_local_m(thread_id); let local_n = get_local_n(thread_id); - var expert_idx:u32 = 0xFFFFFFFFu; - var wg_in_batch:u32 = 0; - var wg_sum:u32 = 0; let wg_m_count = (params.m + WORKGROUP_SIZE_M * TILE_M - 1u) / (WORKGROUP_SIZE_M * TILE_M); let wg_linear = wg_id.y * num_wg.x + wg_id.x; - for (var i = 0u;i < params.n_expert;i += 1) { - let wg_n_count = (gathered_count_ids[i] + WORKGROUP_SIZE_N * TILE_N - 1u) / (WORKGROUP_SIZE_N * TILE_N); - let wg_per_matrix = wg_m_count * wg_n_count; - if (wg_sum <= wg_linear && wg_linear < wg_sum + wg_per_matrix) { - expert_idx = i; - wg_in_batch = wg_linear - wg_sum; - break; - } - wg_sum += wg_per_matrix; - } + // the slot tile of this workgroup, described by mul_mat_id_gather.wgsl + let tile = wg_linear / wg_m_count; + let wg_m = wg_linear % wg_m_count; - let is_valid = expert_idx != 0xFFFFFFFFu; + let is_valid = tile < ids_tiles[0]; - var wg_m: u32 = 0; - var wg_n: u32 = 0; - var offset_wg_m: u32 = 0; - var offset_wg_n: u32 = 0; + var expert_idx: u32 = 0; var rest_token_n: u32 = 0; var src0_batch_offset: u32 = 0; - wg_m = wg_in_batch % wg_m_count; - wg_n = wg_in_batch / wg_m_count; - - offset_wg_m = wg_m * WORKGROUP_SIZE_M * TILE_M; - offset_wg_n = wg_n * WORKGROUP_SIZE_N * TILE_N; + let offset_wg_m = wg_m * WORKGROUP_SIZE_M * TILE_M; if (is_valid) { - rest_token_n = gathered_count_ids[expert_idx] - offset_wg_n; - let global_gathered_base = expert_idx * params.n_tokens + offset_wg_n; - for (var i = thread_id; i < TILE_N * WORKGROUP_SIZE_N && offset_wg_n + i < gathered_count_ids[expert_idx]; i += TOTAL_WORKGROUP_SIZE) { - gathered_expert_used[i] = global_gathered_expert_used[global_gathered_base + i]; - gathered_tokens[i] = global_gathered_tokens[global_gathered_base + i]; + expert_idx = ids_tiles[1 + 3 * tile]; + let slot_base = ids_tiles[1 + 3 * tile + 1]; + rest_token_n = ids_tiles[1 + 3 * tile + 2]; + for (var i = thread_id; i < rest_token_n; i += TOTAL_WORKGROUP_SIZE) { + let pos_in_ids = ids_slot_pos[slot_base + i]; + gathered_expert_used[i] = pos_in_ids % params.n_expert_used; + gathered_tokens[i] = pos_in_ids / params.n_expert_used; } src0_batch_offset = params.offset_src0 + expert_idx * params.stride_02; } @@ -135,7 +120,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>, workgroupBarrier(); let output_row_base = offset_wg_m + local_m * TILE_M; - let output_col_base = offset_wg_n + local_n * TILE_N; + let output_col_base = local_n * TILE_N; let dst2_stride = params.m * params.n_expert_used; let dst1_stride = params.m; @@ -178,9 +163,9 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>, if (is_valid) { for (var tn = 0u; tn < TILE_N; tn++) { let n_idx = output_col_base + tn; - if (n_idx < gathered_count_ids[expert_idx]) { - let dst1_idx = gathered_expert_used[n_idx - offset_wg_n]; - let dst2_idx = gathered_tokens[n_idx - offset_wg_n]; + if (n_idx < rest_token_n) { + let dst1_idx = gathered_expert_used[n_idx]; + let dst2_idx = gathered_tokens[n_idx]; let dst12_offset = params.offset_dst + dst2_idx * dst2_stride + dst1_idx * dst1_stride; for (var tm = 0u; tm < TILE_M; tm += VEC_SIZE) { let global_row = output_row_base + tm; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl index 581e92270..11fcc6ba8 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl @@ -2,51 +2,99 @@ enable f16; struct MulMatIdGatherParams { offset_ids: u32, + offset_dst: u32, n_expert: u32, n_expert_used: u32, n_tokens: u32, stride_ids_1: u32, + + m: u32, }; @group(0) @binding(0) var<storage, read_write> ids: array<i32>; // [n_expert_used, n_tokens] -@group(0) @binding(1) var<storage, read_write> global_gathered_expert_used: array<u32>; // [n_expert][n_tokens] -@group(0) @binding(2) var<storage, read_write> global_gathered_tokens: array<u32>; // [n_expert][n_tokens] -@group(0) @binding(3) var<storage, read_write> gathered_count_ids: array<u32>; // [n_expert] +@group(0) @binding(1) var<storage, read_write> ids_slot_pos: array<u32>; // [n_expert_used * n_tokens] +@group(0) @binding(2) var<storage, read_write> ids_tiles: array<u32>; // [n_tiles, (expert, first slot, n slots) * n_tiles] + +@group(0) @binding(3) var<storage, read_write> dst: array<f32>; // [m, n_expert_used, n_tokens] @group(0) @binding(4) var<uniform> params: MulMatIdGatherParams; -var<workgroup> count:atomic<u32>; +// per expert: the slot count, replaced by the first slot of the expert in ids_slot_pos by the prefix sum, then advanced as its write cursor +var<workgroup> ids_count: array<atomic<u32>, N_EXPERT>; +// launched as a single workgroup, where the threads process the elements of ids in parallel with atomics @compute @workgroup_size(WG_SIZE) -fn main(@builtin(workgroup_id) wg_id: vec3<u32>, - @builtin(local_invocation_id) local_id: vec3<u32>) { +fn main(@builtin(local_invocation_id) local_id: vec3<u32>) { let thread_id = local_id.x; - let own_expert = wg_id.x; // the expert assigned to this workgroup + let ids_slots_num = params.n_expert_used * params.n_tokens; - if (thread_id == 0u) { - atomicStore(&count, 0); + for (var e = thread_id; e < N_EXPERT; e += WG_SIZE) { + atomicStore(&ids_count[e], 0u); } workgroupBarrier(); - for (var i = thread_id;i < params.n_expert_used * params.n_tokens;i += WG_SIZE) { + // count the slots of each expert + for (var i = thread_id; i < ids_slots_num; i += WG_SIZE) { let row = i / params.n_expert_used; let col = i % params.n_expert_used; - let expert = u32(ids[params.offset_ids + row * params.stride_ids_1 + col]); - if (own_expert == expert) { - let pos = atomicAdd(&count, 1u); - let gathered_id = own_expert * params.n_tokens + pos; - global_gathered_expert_used[gathered_id] = col; - global_gathered_tokens[gathered_id] = row; + let expert = ids[params.offset_ids + row * params.stride_ids_1 + col]; + if (expert != -1) { // skipped slot, owned by no expert + atomicAdd(&ids_count[u32(expert)], 1u); } } workgroupBarrier(); + // exclusive prefix sum over the counts, and emit one tile descriptor per `TILE_N_SLOTS` slots of each expert if (thread_id == 0u) { - gathered_count_ids[own_expert] = atomicLoad(&count); + var base: u32 = 0; + var tile: u32 = 0; + + for (var e: u32 = 0; e < N_EXPERT; e += 1) { + let n = atomicLoad(&ids_count[e]); + + atomicStore(&ids_count[e], base); + + for (var k: u32 = 0; k < n; k += TILE_N_SLOTS) { + ids_tiles[1 + 3 * tile] = e; + ids_tiles[1 + 3 * tile + 1] = base + k; + ids_tiles[1 + 3 * tile + 2] = min(n - k, TILE_N_SLOTS); + tile += 1; + } + + base += n; + } + + ids_tiles[0] = tile; + } + + workgroupBarrier(); + + // store the position in ids tensor to ids_slot_pos + for (var i = thread_id; i < ids_slots_num; i += WG_SIZE) { + let row = i / params.n_expert_used; + let col = i % params.n_expert_used; + let expert = ids[params.offset_ids + row * params.stride_ids_1 + col]; + if (expert != -1) { + let pos = atomicAdd(&ids_count[u32(expert)], 1u); + ids_slot_pos[pos] = i; + } + } + + // the matmul writes only the rows that an expert owns, so zero the rows of the skipped slots + for (var i: u32 = 0; i < ids_slots_num; i += 1) { + let row = i / params.n_expert_used; + let col = i % params.n_expert_used; + if (ids[params.offset_ids + row * params.stride_ids_1 + col] != -1) { + continue; + } + let dst_row = params.offset_dst + i * params.m; + for (var r = thread_id; r < params.m; r += WG_SIZE) { + dst[dst_row + r] = 0.0f; + } } } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl index 78ae955e6..e87cc6b2d 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl @@ -59,47 +59,29 @@ fn main( let thread_id = local_id.x; - for (var i = thread_id;i < params.n_expert;i += WG_SIZE) { - gathered_count_ids[i] = 0; - } - - workgroupBarrier(); - - // gather the selected experts for the target token. - for (var col = thread_id;col < params.n_expert_used;col += WG_SIZE) { - let expert = ids[params.offset_ids + col]; - gathered_count_ids[expert] = 1; - gathered_expert_used[expert] = col; - } - - workgroupBarrier(); - - let output_groups:u32 = (params.m + OUTPUTS_PER_WG - 1u) / OUTPUTS_PER_WG; + let output_groups: u32 = (params.m + OUTPUTS_PER_WG - 1u) / OUTPUTS_PER_WG; let wg_linear = wg_id.y * num_wg.x + wg_id.x; - var own_expert:u32 = 0; - var wg_in_batch:u32 = 0; - var wg_sum:u32 = 0; - - for (var i = 0u;i < params.n_expert;i += 1) { - let wg_vec_count = gathered_count_ids[i]; // 1 or 0 - let wg_per_matrix = output_groups * wg_vec_count; - if (wg_sum <= wg_linear && wg_linear < wg_sum + wg_per_matrix) { - own_expert = i; - wg_in_batch = wg_linear - wg_sum; - break; - } - wg_sum += wg_per_matrix; + if (wg_linear >= output_groups * params.n_expert_used) { + return; } - let row_base = (wg_linear % output_groups) * OUTPUTS_PER_WG; - let dst1_stride = params.m; + let slot = wg_linear / output_groups; + let own_expert = ids[params.offset_ids + slot]; + let row_base = (wg_linear % output_groups) * OUTPUTS_PER_WG; - let src0_batch_offset = params.offset_src0 + own_expert * params.stride_02; - let src1_idx_base = params.offset_src1 + (gathered_expert_used[own_expert] % params.b_ne1) * params.stride_11; - let dst_idx_base = params.offset_dst + gathered_expert_used[own_expert] * dst1_stride + row_base; + // a skipped slot (id -1) keeps the zero accumulator, so the reduction below writes zeros + // no early return: the barriers below must stay in uniform control flow + let skipped = i32(own_expert) == -1; - let acc = accumulate_vec_dot(thread_id, row_base, src0_batch_offset, src1_idx_base); + let src0_batch_offset = params.offset_src0 + select(own_expert, 0u, skipped) * params.stride_02; + let src1_idx_base = params.offset_src1 + (slot % params.b_ne1) * params.stride_11; + let dst_idx_base = params.offset_dst + slot * params.m + row_base; + + var acc: array<array<f32, OUTPUTS_PER_WG>, NUM_COLS>; + if (!skipped) { + acc = accumulate_vec_dot(thread_id, row_base, src0_batch_offset, src1_idx_base); + } #ifdef USE_SUBGROUP_REDUCTION for (var row = 0u; row < OUTPUTS_PER_WG; row++) { diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl index 841777df9..2c116aad6 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl @@ -119,6 +119,46 @@ fn accumulate_vec_dot(thread_id: u32, row_base: u32, src0_batch_offset: u32, src } #endif +#ifdef MUL_ACC_Q2_0 +#define BLOCK_SIZE 64 +#define BLOCK_SIZE_BYTES 18 +#define THREADS_PER_BLOCK 4 +#define ELEMS_PER_THREAD (BLOCK_SIZE/THREADS_PER_BLOCK) +fn accumulate_vec_dot(thread_id: u32, row_base: u32, src0_batch_offset: u32, src1_idx_base: u32) -> array<array<f32, OUTPUTS_PER_WG>, NUM_COLS> { + var acc: array<array<f32, OUTPUTS_PER_WG>, NUM_COLS>; + + let num_blocks = params.k / BLOCK_SIZE; + let thread_within_block = thread_id % THREADS_PER_BLOCK; + for (var block = thread_id / THREADS_PER_BLOCK; block < num_blocks; block += WG_SIZE / THREADS_PER_BLOCK) { + let x_base = src1_idx_base + block * BLOCK_SIZE + thread_within_block * ELEMS_PER_THREAD; + var x_block: array<array<f32, ELEMS_PER_THREAD>, NUM_COLS>; + for (var col = 0u; col < NUM_COLS; col++) { + for (var i = 0u; i < ELEMS_PER_THREAD; i++) { + x_block[col][i] = f32(SRC1[x_base + col * params.stride_11 + i]); + } + } + for (var row = 0u; row < OUTPUTS_PER_WG; row++) { + let output_row = row_base + row; + if (output_row < params.m) { + let block_byte_base = (src0_batch_offset + output_row * params.stride_01 + block) * BLOCK_SIZE_BYTES; + let d = f32(load_f16_at_src0(block_byte_base)); + let q_packed = load_u32_at_src0(block_byte_base + 2u + 4u * thread_within_block); + for (var col = 0u; col < NUM_COLS; col++) { + var row_sum = 0.0; + for (var i = 0u; i < ELEMS_PER_THREAD; i++) { + let q = (q_packed >> (2u * i)) & 3u; + row_sum += (f32(q) - 1.0) * x_block[col][i]; + } + acc[col][row] += d * row_sum; + } + } + } + } + + return acc; +} +#endif + #ifdef MUL_ACC_Q4_0 #define BLOCK_SIZE 32 #define BLOCK_SIZE_BYTES 18 diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl index ea63b9a73..6c7b2e0ac 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl @@ -45,12 +45,15 @@ fn wrap_around(idx: i32, n: u32) -> u32 { } @compute @workgroup_size(WG_SIZE) -fn main(@builtin(global_invocation_id) gid: vec3<u32>) { - if (gid.x >= params.ne) { +fn main(@builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(global_invocation_id) gid: vec3<u32>) { + + let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (gid_i >= params.ne) { return; } - var i = gid.x; + var i = gid_i; let dst_plane = params.dst_ne2 * params.dst_ne1 * params.dst_ne0; let i3 = i / dst_plane; i = i % dst_plane; @@ -61,6 +64,18 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) { var value: f32 = 0.0; +#ifdef REFLECT + var x: u32; + if (i0 < params.lp0) { + x = params.lp0 - i0; + } else if (i0 < params.lp0 + params.src_ne0) { + x = i0 - params.lp0; + } else { + x = params.src_ne0 - 2 - (i0 - params.lp0 - params.src_ne0); + } + value = src[params.offset_src + x * params.stride_src0 + i1 * params.stride_src1 + + i2 * params.stride_src2 + i3 * params.stride_src3]; +#else #ifdef CIRCULAR let ci0 = wrap_around(i32(i0) - i32(params.lp0), params.src_ne0); let ci1 = wrap_around(i32(i1) - i32(params.lp1), params.src_ne1); @@ -82,5 +97,6 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) { } #endif - dst[params.offset_dst + gid.x] = value; +#endif + dst[params.offset_dst + gid_i] = value; } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl index 43b883e67..9debcd9c5 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl @@ -45,9 +45,12 @@ var<storage, read_write> dst: array<DataType>; var<uniform> params: Params; @compute @workgroup_size(WG_SIZE) -fn main(@builtin(global_invocation_id) gid: vec3<u32>) { - if (gid.x < params.ne) { - var i = gid.x; +fn main(@builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(global_invocation_id) gid: vec3<u32>) { + + let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (gid_i < params.ne) { + var i = gid_i; let i3 = i / (params.ne2 * params.ne1 * params.ne0); i = i % (params.ne2 * params.ne1 * params.ne0); let i2 = i / (params.ne1 * params.ne0); @@ -65,6 +68,6 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) { a_i2 * params.stride_src0_2 + a_i3 * params.stride_src0_3; - dst[params.offset_dst + gid.x] = src0[params.offset_src0 + a_index]; + dst[params.offset_dst + gid_i] = src0[params.offset_src0 + a_index]; } } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl index c9e424ffc..46a208029 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl @@ -88,6 +88,7 @@ struct Params { ne0: u32, ne1: u32, ne2: u32, + ne3: u32, eps: f32 }; @@ -96,10 +97,14 @@ var<workgroup> scratch: array<f32, WG_SIZE>; @compute @workgroup_size(WG_SIZE) fn main(@builtin(workgroup_id) wid: vec3<u32>, + @builtin(num_workgroups) num_wg: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) { - // one thread per row - var i = wid.x; + // one workgroup per row + var i = wid.x + wid.y * num_wg.x; + if (i >= params.ne1 * params.ne2 * params.ne3) { + return; + } let i3 = i / (params.ne2 * params.ne1); i = i % (params.ne2 * params.ne1); let i2 = i / params.ne1; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl index 6ff53088c..31a73c613 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl @@ -145,9 +145,12 @@ fn pair_offset(is_neox: bool, is_mrope: bool, is_vision: bool) -> u32 { } @compute @workgroup_size(WG_SIZE) -fn main(@builtin(global_invocation_id) gid: vec3<u32>) { +fn main(@builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(global_invocation_id) gid: vec3<u32>) { + + let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; // two elements per n_threads - if (gid.x >= params.n_threads) { + if (gid_i >= params.n_threads) { return; } @@ -156,7 +159,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) { let is_imrope = params.mode == 40; let is_vision = params.mode == 24; - var i = gid.x * 2; // start index for this thread + var i = gid_i * 2; // start index for this thread let i3 = i / (params.ne2 * params.ne1 * params.ne0); i = i % (params.ne2 * params.ne1 * params.ne0); let i2 = i / (params.ne1 * params.ne0); diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl index 7629bf5b4..a87bf897a 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl @@ -31,6 +31,7 @@ struct Params { ne0: u32, ne1: u32, ne2: u32, + ne3: u32, eps: f32 }; @@ -53,10 +54,14 @@ var<workgroup> scratch: array<f32, WG_SIZE * 2u>; @compute @workgroup_size(WG_SIZE) fn main(@builtin(workgroup_id) wid: vec3<u32>, + @builtin(num_workgroups) num_wg: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) { - // one thread per row - var i = wid.x; + // one workgroup per row + var i = wid.x + wid.y * num_wg.x; + if (i >= params.ne1 * params.ne2 * params.ne3) { + return; + } let i3 = i / (params.ne2 * params.ne1); i = i % (params.ne2 * params.ne1); let i2 = i / params.ne1; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl index 0a7ae9bdb..919db34c3 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl @@ -84,26 +84,29 @@ fn in_set_view(rel: u32, coords: vec4<u32>) -> bool { } @compute @workgroup_size(WG_SIZE) -fn main(@builtin(global_invocation_id) gid: vec3<u32>) { - if (gid.x >= params.ne) { +fn main(@builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(global_invocation_id) gid: vec3<u32>) { + + let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (gid_i >= params.ne) { return; } #ifdef INPLACE - let coords = decode_src1_coords(gid.x); + let coords = decode_src1_coords(gid_i); let src1_idx = params.offset_src1 + src1_idx_from_coords(coords); let dst_idx = params.offset_view + view_rel_from_coords(coords); dst[dst_idx] = src1[src1_idx]; #else - let rel = select(params.ne, gid.x - params.offset_view, gid.x >= params.offset_view); + let rel = select(params.ne, gid_i - params.offset_view, gid_i >= params.offset_view); let coords = decode_view_coords(rel); if (rel < params.stride_dst13 * params.src1_ne3 && in_set_view(rel, coords)) { - dst[gid.x] = src1[params.offset_src1 + src1_idx_from_coords(coords)]; + dst[gid_i] = src1[params.offset_src1 + src1_idx_from_coords(coords)]; } else { - dst[gid.x] = src0[params.offset_src0 + gid.x]; + dst[gid_i] = src0[params.offset_src0 + gid_i]; } #endif } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl index 91c3d9c74..2e1b0111e 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl @@ -70,13 +70,16 @@ struct Params { var<uniform> params: Params; @compute @workgroup_size(WG_SIZE) -fn main(@builtin(global_invocation_id) gid: vec3<u32>) { - if (gid.x >= (params.ne3 * params.ne2 * params.n_rows * params.ne0) / VEC_SIZE) { +fn main(@builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(global_invocation_id) gid: vec3<u32>) { + + let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (gid_i >= (params.ne3 * params.ne2 * params.n_rows * params.ne0) / VEC_SIZE) { return; } let elems_per_row = params.ne0 / VEC_SIZE; - var i = gid.x / elems_per_row; + var i = gid_i / elems_per_row; let i_src3 = i / (params.ne2 * params.n_rows); @@ -107,6 +110,6 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) { let i_dst_row = params.offset_dst + idx_val * params.stride_dst1 + i_src2 * params.stride_dst2 + i_src3 * params.stride_dst3; let i_src_row = params.offset_src + i_src1 * params.stride_src1 + i_src2 * params.stride_src2 + i_src3 * params.stride_src3; - let col_idx = gid.x % elems_per_row; + let col_idx = gid_i % elems_per_row; dst[i_dst_row / VEC_SIZE + col_idx] = DST_TYPE(src[i_src_row / VEC_SIZE + col_idx]); } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/set_tensor_n.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/set_tensor_n.wgsl new file mode 100644 index 000000000..f9ba26416 --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/set_tensor_n.wgsl @@ -0,0 +1,42 @@ +@group(0) @binding(0) +var<storage, read_write> output_buffer: array<atomic<u32>>; + +@group(0) @binding(1) +var<storage, read> src_buffer: array<u32>; + +@group(0) @binding(2) +var<storage, read> descs: array<vec4<u32>>; + +override wg_size: u32; + +@compute @workgroup_size(wg_size) +fn main(@builtin(workgroup_id) wid: vec3<u32>, + @builtin(num_workgroups) num_wg: vec3<u32>, + @builtin(local_invocation_id) lid: vec3<u32>) { + + let w = wid.y * num_wg.x + wid.x; + + if (w >= arrayLength(&descs)) { + return; + } + + let dst_i = descs[w].x / 4u; + let src_i = descs[w].y; + let lane = descs[w].x & 3u; + let end = lane + descs[w].z; + let n_words = (end + 3u) / 4u; + + for (var i = lid.x; i < n_words; i += wg_size) { + // [lo, hi) is the byte range of this word that belongs to the descs + let lo = select(0u, lane, i == 0u); + let hi = min(end - i * 4u, 4u); + let mask = select((1u << (hi * 8u)) - 1u, 0xffffffffu, hi == 4u) & (0xffffffffu << (lo * 8u)); + let value = src_buffer[src_i + i]; + if (mask == 0xffffffffu) { + atomicStore(&output_buffer[dst_i + i], value); + } else { + atomicAnd(&output_buffer[dst_i + i], ~mask); + atomicOr(&output_buffer[dst_i + i], value & mask); + } + } +} diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl index 1c29a9221..8f92a04c1 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl @@ -124,9 +124,10 @@ var<workgroup> scratch: array<f32, WG_SIZE>; @compute @workgroup_size(WG_SIZE) fn main(@builtin(workgroup_id) wid: vec3<u32>, + @builtin(num_workgroups) num_wg: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) { - var i = wid.x; + var i = wid.x + wid.y * num_wg.x; let i3 = i / (params.ne2 * params.ne1); i = i % (params.ne2 * params.ne1); let i2 = i / params.ne1; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl index 6ea2de9b7..e4f175262 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl @@ -15,7 +15,8 @@ struct Params { ne0: u32, ne1: u32, - ne2: u32 + ne2: u32, + nrows: u32 }; @group(0) @binding(2) @@ -25,9 +26,14 @@ var<workgroup> shared_sum: array<f32, WG_SIZE>; @compute @workgroup_size(WG_SIZE) fn main(@builtin(workgroup_id) wid: vec3<u32>, + @builtin(num_workgroups) num_wg: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) { - var i = wid.x; + let row = wid.x + wid.y * num_wg.x; + if (row >= params.nrows) { + return; + } + var i = row; let i3 = i / (params.ne2 * params.ne1); i = i % (params.ne2 * params.ne1); let i2 = i / params.ne1; @@ -50,6 +56,10 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>, } if (lid.x == 0) { - dst[params.offset_dst + wid.x] = shared_sum[0]; +#ifdef MEAN + dst[params.offset_dst + row] = shared_sum[0] / f32(params.ne0); +#else + dst[params.offset_dst + row] = shared_sum[0]; +#endif } } diff --git a/ggml/src/ggml-zdnn/common.hpp b/ggml/src/ggml-zdnn/common.hpp index 2462ded55..e28e7608b 100644 --- a/ggml/src/ggml-zdnn/common.hpp +++ b/ggml/src/ggml-zdnn/common.hpp @@ -6,11 +6,17 @@ #include "zdnn.h" -#include <vector> +#include "scratch.hpp" + +#include <algorithm> +#include <cstdlib> +#include <future> +#include <map> #include <memory> +#include <utility> +#include <vector> -#define GGML_ZDNN_NAME "zDNN" -#define GGML_ZDNN_VERSION ZDNN_VERNUM +#define GGML_ZDNN_NAME "zDNN" #define ZDNN_CHECK(stmt) \ do { \ @@ -27,12 +33,9 @@ struct ggml_backend_zdnn_device_context { size_t max_size; - char name[128]; -}; + int32_t n_threads; // CPU threads -struct ggml_backend_zdnn_context { - int device; - ggml_cgraph * gf; + char name[128]; }; struct ggml_backend_zdnn_buffer { @@ -44,6 +47,9 @@ struct ggml_backend_zdnn_buffer { zdnn_tensor_desc tfm_desc; zdnn_ztensor ztensor; + std::vector<float> scales; // per-row scales for quantized weights + std::vector<float> channel_scales; // per-input-channel scales for quantized weights, applied to the inputs + char name[GGML_MAX_NAME]; }; @@ -56,4 +62,131 @@ struct ggml_backend_zdnn_buffer_context { std::vector<std::unique_ptr<ggml_backend_zdnn_buffer>> buffers; }; +struct ggml_backend_zdnn_context { + int device; + int32_t n_threads; // for the CPU work around zDNN calls + ggml_cgraph * gf; + + // zero bias for stacked quantized matmuls, keyed by (n_stacks, ne0) + std::map<std::pair<int64_t, int64_t>, std::unique_ptr<ggml_backend_zdnn_buffer>> zero_bias; + + ggml_zdnn_pool pool; // scratch memory of the ops +}; + +// +// Scratch buffers +// + +// the ztensor uses the memory of scratch, so scratch must outlive it +static inline void ggml_zdnn_init_scratch_ztensor(ggml_backend_zdnn_context * ctx, + zdnn_tensor_desc * pre_tfm_desc, + zdnn_tensor_desc * tfm_desc, + zdnn_ztensor * ztensor, + ggml_zdnn_pool_alloc<uint8_t> & scratch) { + + const uint64_t size = zdnn_getsize_ztensor(tfm_desc); + + zdnn_init_quantized_ztensor(pre_tfm_desc, tfm_desc, 1.0f, 0.0f, ztensor); + ztensor->buffer = scratch.alloc(ctx->pool, size); + ztensor->buffer_size = size; +} + +// same as ggml_zdnn_init_scratch_ztensor, for a DLFLOAT16 ztensor +static inline void ggml_zdnn_init_scratch_ztensor_dlf16(ggml_backend_zdnn_context * ctx, + zdnn_tensor_desc * pre_tfm_desc, + zdnn_tensor_desc * tfm_desc, + zdnn_ztensor * ztensor, + ggml_zdnn_pool_alloc<uint8_t> & scratch) { + + const uint64_t size = zdnn_getsize_ztensor(tfm_desc); + + zdnn_init_ztensor(pre_tfm_desc, tfm_desc, ztensor); + ztensor->buffer = scratch.alloc(ctx->pool, size); + ztensor->buffer_size = size; +} + +// +// Stacked ztensors +// + +// view of the stacks [s0, s0 + n) of a stacked ztensor, the stacks are the outermost dim of its buffer +// the view descs must outlive the view +static inline zdnn_ztensor ggml_zdnn_stack_view(zdnn_tensor_desc * pre_tfm_desc, + zdnn_tensor_desc * tfm_desc, + const zdnn_ztensor & src, + int64_t n_stacks, + int64_t s0, + int64_t n) { + + GGML_ASSERT(src.buffer_size % n_stacks == 0); + const uint64_t stack_bytes = src.buffer_size / n_stacks; + + *pre_tfm_desc = *src.pre_transformed_desc; + *tfm_desc = *src.transformed_desc; + + switch (pre_tfm_desc->layout) { + case ZDNN_3DS: + { + pre_tfm_desc->dim3 = n; + } break; + case ZDNN_2DS: + { + pre_tfm_desc->dim2 = n; + } break; + default: + GGML_ABORT("%s: unsupported stacked layout", __func__); + } + + tfm_desc->dim4 = n; + GGML_ASSERT(zdnn_getsize_ztensor(tfm_desc) == n*stack_bytes); + + zdnn_ztensor view = src; + view.pre_transformed_desc = pre_tfm_desc; + view.transformed_desc = tfm_desc; + view.buffer = (char *) src.buffer + s0*stack_bytes; + view.buffer_size = n * stack_bytes; + return view; +} + +// zero bias of n_stacks stacks, created once per shape as it never changes +static inline const zdnn_ztensor & ggml_zdnn_zero_bias(ggml_backend_zdnn_context * ctx, int64_t n_stacks, int64_t ne0) { + std::unique_ptr<ggml_backend_zdnn_buffer> & bias = ctx->zero_bias[{ n_stacks, ne0 }]; + + if (!bias) { + bias = std::make_unique<ggml_backend_zdnn_buffer>(); + + std::vector<float> zeros(n_stacks * ne0, 0.0f); + + zdnn_init_pre_transformed_desc(ZDNN_2DS, FP32, &bias->pre_tfm_desc, n_stacks, ne0); + ZDNN_CHECK(zdnn_generate_transformed_desc(&bias->pre_tfm_desc, &bias->tfm_desc)); + ZDNN_CHECK(zdnn_init_ztensor_with_malloc(&bias->pre_tfm_desc, &bias->tfm_desc, &bias->ztensor)); + ZDNN_CHECK(zdnn_transform_ztensor(&bias->ztensor, zeros.data())); + } + + return bias->ztensor; +} + +// +// Threading +// + +// split [0, n) into n_threads ranges, the calling thread runs the first range +template <typename F> +static inline void ggml_zdnn_parallel_for(int32_t n_threads, int64_t n, const F & f) { + n_threads = (int32_t)std::max<int64_t>(1, std::min<int64_t>(n_threads, n)); + + std::vector<std::future<void>> tasks; + for (int32_t ith = 1; ith < n_threads; ith++) { + const int64_t start = (ith + 0) * n / n_threads; + const int64_t end = (ith + 1) * n / n_threads; + tasks.push_back(std::async(std::launch::async, [&f, start, end]() { f(start, end); })); + } + + f(0, n/n_threads); + + for (auto & task : tasks) { + task.get(); + } +} + #endif // GGML_ZDNN_COMMON_HPP diff --git a/ggml/src/ggml-zdnn/fattn.cpp b/ggml/src/ggml-zdnn/fattn.cpp new file mode 100644 index 000000000..e6a0c24ca --- /dev/null +++ b/ggml/src/ggml-zdnn/fattn.cpp @@ -0,0 +1,249 @@ +#include "ggml.h" +#include "fattn.hpp" + +#include <algorithm> +#include <cstring> + +// DLFLOAT16 has no infinity, so masked scores use the lowest FP16 value, exp of which is still 0 +static inline float ggml_zdnn_fattn_mask_value(ggml_fp16_t m) { + return std::max(GGML_FP16_TO_FP32(m), -65504.0f); +} + +bool ggml_zdnn_supports_flash_attn_ext(const ggml_backend_zdnn_device_context * ctx_dev, const ggml_tensor * op) { + const ggml_tensor * q = op->src[0]; + const ggml_tensor * k = op->src[1]; + const ggml_tensor * v = op->src[2]; + const ggml_tensor * mask = op->src[3]; + + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) op->op_params + 2, sizeof(float)); + + // sinks, alibi, softcap, sparse k/v sets and k/v row lists are not implemented + if (op->src[4] != nullptr || op->src[5] != nullptr || max_bias != 0.0f || logit_softcap != 0.0f || ggml_get_op_params_i32(op, 4) != 0) { + return false; + } + + if (op->type != GGML_TYPE_F32 || q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F16 || v->type != GGML_TYPE_F16) { + return false; + } + + if (mask != nullptr && (mask->type != GGML_TYPE_F16 || mask->ne[2] != 1 || mask->ne[3] != 1)) { + return false; + } + + if (q->ne[3] != 1 || k->ne[3] != 1 || v->ne[3] != 1 || k->ne[2] != v->ne[2] || q->ne[2] % k->ne[2] != 0) { + return false; + } + + // rows are gathered with their strides, but each row has to be contiguous + if (q->nb[0] != sizeof(float) || k->nb[0] != sizeof(ggml_fp16_t) || v->nb[0] != sizeof(ggml_fp16_t)) { + return false; + } + + const int64_t max_dim = ctx_dev->max_size; + const int64_t group = q->ne[2] / k->ne[2]; + + return q->ne[0] <= max_dim && v->ne[0] <= max_dim && k->ne[1] <= max_dim && + q->ne[2] <= max_dim && group * q->ne[1] <= max_dim; +} + +void ggml_zdnn_flash_attn_ext(ggml_backend_zdnn_context * ctx, ggml_tensor * dst) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + + const int64_t d_k = q->ne[0]; + const int64_t n_q = q->ne[1]; + const int64_t n_head = q->ne[2]; + const int64_t n_kv = k->ne[1]; + const int64_t n_head_kv = k->ne[2]; + const int64_t d_v = v->ne[0]; + const int64_t group = n_head / n_head_kv; + + float scale = 1.0f; + memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); + + const int32_t n_threads = n_q*n_kv >= 65536 ? ctx->n_threads : 1; + + // one query runs all heads in one call with the mask row as the K*Q bias, more queries run per head and add the mask + const bool single = n_q == 1; + + const int64_t n_mask = mask ? (single ? n_head_kv*n_kv : n_q*n_kv) : 0; + + ggml_zdnn_pool_alloc<float> q_host_alloc(ctx->pool, n_head*n_q*d_k); + ggml_zdnn_pool_alloc<float> o_host_alloc(ctx->pool, n_head*n_q*d_v); + ggml_zdnn_pool_alloc<float> mask_host_alloc(ctx->pool, n_mask); + ggml_zdnn_pool_alloc<ggml_fp16_t> k_host_alloc(ctx->pool, n_head_kv*n_kv*d_k); + ggml_zdnn_pool_alloc<ggml_fp16_t> v_host_alloc(ctx->pool, n_head_kv*n_kv*d_v); + + float * q_host = q_host_alloc.get(); + float * o_host = o_host_alloc.get(); + float * mask_host = mask_host_alloc.get(); + ggml_fp16_t * k_host = k_host_alloc.get(); + ggml_fp16_t * v_host = v_host_alloc.get(); + + // q is [n_head][n_q][d_k] with the scale folded in, k and v are [n_head_kv][n_kv][d] + ggml_zdnn_parallel_for(n_threads, n_head, [&](int64_t h_start, int64_t h_end) { + for (int64_t h = h_start; h < h_end; h++) { + for (int64_t iq = 0; iq < n_q; iq++) { + const float * src = (const float *)((const char *) q->data + iq*q->nb[1] + h*q->nb[2]); + float * row = q_host + (h*n_q + iq)*d_k; + for (int64_t i0 = 0; i0 < d_k; i0++) { + row[i0] = src[i0] * scale; + } + } + } + }); + + ggml_zdnn_parallel_for(n_threads, n_head_kv*n_kv, [&](int64_t r_start, int64_t r_end) { + for (int64_t r = r_start; r < r_end; r++) { + const int64_t hk = r / n_kv; + const int64_t ik = r % n_kv; + memcpy(k_host + r*d_k, (const char *) k->data + ik*k->nb[1] + hk*k->nb[2], d_k*sizeof(ggml_fp16_t)); + memcpy(v_host + r*d_v, (const char *) v->data + ik*v->nb[1] + hk*v->nb[2], d_v*sizeof(ggml_fp16_t)); + } + }); + + zdnn_tensor_desc q_pre_tfm_desc, q_tfm_desc; + zdnn_tensor_desc k_pre_tfm_desc, k_tfm_desc; + zdnn_tensor_desc v_pre_tfm_desc, v_tfm_desc; + zdnn_tensor_desc o_pre_tfm_desc, o_tfm_desc; + zdnn_ztensor q_ztensor, k_ztensor, v_ztensor, o_ztensor; + + ggml_zdnn_pool_alloc<uint8_t> q_ztensor_alloc, k_ztensor_alloc, v_ztensor_alloc, o_ztensor_alloc; + + // a single query keeps the heads of a kv head in one stack, which is the same memory order as one head per stack + if (single) { + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &q_pre_tfm_desc, n_head_kv, group, d_k); + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &o_pre_tfm_desc, n_head_kv, group, d_v); + } else { + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &q_pre_tfm_desc, n_head, n_q, d_k); + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &o_pre_tfm_desc, n_head, n_q, d_v); + } + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP16, &k_pre_tfm_desc, n_head_kv, n_kv, d_k); + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP16, &v_pre_tfm_desc, n_head_kv, n_kv, d_v); + + ZDNN_CHECK(zdnn_generate_transformed_desc(&q_pre_tfm_desc, &q_tfm_desc)); + ZDNN_CHECK(zdnn_generate_transformed_desc(&k_pre_tfm_desc, &k_tfm_desc)); + ZDNN_CHECK(zdnn_generate_transformed_desc(&v_pre_tfm_desc, &v_tfm_desc)); + ZDNN_CHECK(zdnn_generate_transformed_desc(&o_pre_tfm_desc, &o_tfm_desc)); + + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &q_pre_tfm_desc, &q_tfm_desc, &q_ztensor, q_ztensor_alloc); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &k_pre_tfm_desc, &k_tfm_desc, &k_ztensor, k_ztensor_alloc); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &v_pre_tfm_desc, &v_tfm_desc, &v_ztensor, v_ztensor_alloc); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &o_pre_tfm_desc, &o_tfm_desc, &o_ztensor, o_ztensor_alloc); + + ZDNN_CHECK(zdnn_transform_ztensor(&q_ztensor, q_host)); + ZDNN_CHECK(zdnn_transform_ztensor(&k_ztensor, k_host)); + ZDNN_CHECK(zdnn_transform_ztensor(&v_ztensor, v_host)); + + ggml_zdnn_pool_alloc<uint8_t> softmax_save_area(ctx->pool, ZDNN_SOFTMAX_SAVEAREA_SIZE); + + zdnn_tensor_desc s_pre_tfm_desc, s_tfm_desc; + zdnn_ztensor s_ztensor, s2_ztensor; + + ggml_zdnn_pool_alloc<uint8_t> s_ztensor_alloc, s2_ztensor_alloc; + + if (single) { + // the bias of K*Q is the mask row of the query, repeated for every kv head + zdnn_tensor_desc bias_pre_tfm_desc, bias_tfm_desc; + zdnn_ztensor bias_ztensor; + + ggml_zdnn_pool_alloc<uint8_t> bias_ztensor_alloc; + + const zdnn_ztensor * bias = &ggml_zdnn_zero_bias(ctx, n_head_kv, n_kv); + if (mask != nullptr) { + const ggml_fp16_t * mask_row = (const ggml_fp16_t *) mask->data; + for (int64_t hk = 0; hk < n_head_kv; hk++) { + for (int64_t ik = 0; ik < n_kv; ik++) { + mask_host[hk*n_kv + ik] = ggml_zdnn_fattn_mask_value(mask_row[ik]); + } + } + + zdnn_init_pre_transformed_desc(ZDNN_2DS, FP32, &bias_pre_tfm_desc, n_head_kv, n_kv); + ZDNN_CHECK(zdnn_generate_transformed_desc(&bias_pre_tfm_desc, &bias_tfm_desc)); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &bias_pre_tfm_desc, &bias_tfm_desc, &bias_ztensor, bias_ztensor_alloc); + ZDNN_CHECK(zdnn_transform_ztensor(&bias_ztensor, mask_host)); + bias = &bias_ztensor; + } + + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &s_pre_tfm_desc, n_head_kv, group, n_kv); + ZDNN_CHECK(zdnn_generate_transformed_desc(&s_pre_tfm_desc, &s_tfm_desc)); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &s_pre_tfm_desc, &s_tfm_desc, &s_ztensor, s_ztensor_alloc); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &s_pre_tfm_desc, &s_tfm_desc, &s2_ztensor, s2_ztensor_alloc); + + ZDNN_CHECK(zdnn_matmul_transpose_op(&q_ztensor, &k_ztensor, bias, false, true, MATMUL_OP_ADDITION, &s_ztensor)); + ZDNN_CHECK(zdnn_softmax(&s_ztensor, softmax_save_area.get(), SOFTMAX_ACT_NONE, &s2_ztensor)); + ZDNN_CHECK(zdnn_matmul_op(&s2_ztensor, &v_ztensor, &ggml_zdnn_zero_bias(ctx, n_head_kv, d_v), MATMUL_OP_ADDITION, &o_ztensor)); + } else { + zdnn_tensor_desc mask_pre_tfm_desc, mask_tfm_desc; + zdnn_ztensor mask_ztensor; + + ggml_zdnn_pool_alloc<uint8_t> mask_ztensor_alloc; + + if (mask != nullptr) { + ggml_zdnn_parallel_for(n_threads, n_q, [&](int64_t iq_start, int64_t iq_end) { + for (int64_t iq = iq_start; iq < iq_end; iq++) { + const ggml_fp16_t * mask_row = (const ggml_fp16_t *)((const char *) mask->data + iq*mask->nb[1]); + for (int64_t ik = 0; ik < n_kv; ik++) { + mask_host[iq*n_kv + ik] = ggml_zdnn_fattn_mask_value(mask_row[ik]); + } + } + }); + + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &mask_pre_tfm_desc, 1, n_q, n_kv); + ZDNN_CHECK(zdnn_generate_transformed_desc(&mask_pre_tfm_desc, &mask_tfm_desc)); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &mask_pre_tfm_desc, &mask_tfm_desc, &mask_ztensor, mask_ztensor_alloc); + ZDNN_CHECK(zdnn_transform_ztensor(&mask_ztensor, mask_host)); + } + + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &s_pre_tfm_desc, 1, n_q, n_kv); + ZDNN_CHECK(zdnn_generate_transformed_desc(&s_pre_tfm_desc, &s_tfm_desc)); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &s_pre_tfm_desc, &s_tfm_desc, &s_ztensor, s_ztensor_alloc); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &s_pre_tfm_desc, &s_tfm_desc, &s2_ztensor, s2_ztensor_alloc); + + const zdnn_ztensor & k_bias = ggml_zdnn_zero_bias(ctx, 1, n_kv); + const zdnn_ztensor & v_bias = ggml_zdnn_zero_bias(ctx, 1, d_v); + + zdnn_tensor_desc view_pre_tfm_desc[4]; + zdnn_tensor_desc view_tfm_desc[4]; + + for (int64_t h = 0; h < n_head; h++) { + const zdnn_ztensor q_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[0], &view_tfm_desc[0], q_ztensor, n_head, h, 1); + const zdnn_ztensor k_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[1], &view_tfm_desc[1], k_ztensor, n_head_kv, h / group, 1); + const zdnn_ztensor v_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[2], &view_tfm_desc[2], v_ztensor, n_head_kv, h / group, 1); + zdnn_ztensor o_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[3], &view_tfm_desc[3], o_ztensor, n_head, h, 1); + + s_ztensor.is_transformed = false; + s2_ztensor.is_transformed = false; + + ZDNN_CHECK(zdnn_matmul_transpose_op(&q_view, &k_view, &k_bias, false, true, MATMUL_OP_ADDITION, &s_ztensor)); + + zdnn_ztensor * scores = &s_ztensor; + zdnn_ztensor * probs = &s2_ztensor; + if (mask != nullptr) { + ZDNN_CHECK(zdnn_add(&s_ztensor, &mask_ztensor, &s2_ztensor)); + scores = &s2_ztensor; + probs = &s_ztensor; + probs->is_transformed = false; + } + + ZDNN_CHECK(zdnn_softmax(scores, softmax_save_area.get(), SOFTMAX_ACT_NONE, probs)); + ZDNN_CHECK(zdnn_matmul_op(probs, &v_view, &v_bias, MATMUL_OP_ADDITION, &o_view)); + } + + o_ztensor.is_transformed = true; + } + + ZDNN_CHECK(zdnn_transform_origtensor(&o_ztensor, o_host)); + + // the result is [d_v][n_head][n_q] + for (int64_t iq = 0; iq < n_q; iq++) { + for (int64_t h = 0; h < n_head; h++) { + memcpy((char *) dst->data + h*dst->nb[1] + iq*dst->nb[2], o_host + (h*n_q + iq)*d_v, d_v*sizeof(float)); + } + } +} diff --git a/ggml/src/ggml-zdnn/fattn.hpp b/ggml/src/ggml-zdnn/fattn.hpp new file mode 100644 index 000000000..a88e1eba7 --- /dev/null +++ b/ggml/src/ggml-zdnn/fattn.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_ZDNN_FATTN_HPP +#define GGML_ZDNN_FATTN_HPP + +#include "common.hpp" + +bool ggml_zdnn_supports_flash_attn_ext(const ggml_backend_zdnn_device_context * ctx_dev, const ggml_tensor * op); + +void ggml_zdnn_flash_attn_ext(ggml_backend_zdnn_context * ctx, ggml_tensor * dst); + +#endif // GGML_ZDNN_FATTN_HPP diff --git a/ggml/src/ggml-zdnn/ggml-zdnn.cpp b/ggml/src/ggml-zdnn/ggml-zdnn.cpp index 5d1209357..ccdd4f382 100644 --- a/ggml/src/ggml-zdnn/ggml-zdnn.cpp +++ b/ggml/src/ggml-zdnn/ggml-zdnn.cpp @@ -3,25 +3,30 @@ #include "ggml-backend-impl.h" #include "ggml-zdnn/common.hpp" +#include "ggml-zdnn/fattn.hpp" #include "ggml-zdnn/mmf.hpp" +#include "ggml-zdnn/mmq.hpp" #include "ggml-zdnn/utils.hpp" #include "ggml.h" #include <vector> #include <memory> #include <csignal> // raise(SIGTRAP) +#include <thread> #include <unistd.h> static void ggml_zdnn_compute_forward_mul_mat( - const ggml_backend_zdnn_context * ctx, + ggml_backend_zdnn_context * ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; // weights const ggml_tensor * src1 = dst->src[1]; // inputs - // TODO: implement support for quantized types - // we currently only support f32, f16, and bf16 - ggml_zdnn_mul_mat_f(ctx, src0, src1, dst); + if (ggml_is_quantized(src0->type)) { + ggml_zdnn_mul_mat_q(ctx, src0, src1, dst); + } else { + ggml_zdnn_mul_mat_f(ctx, src0, src1, dst); + } } static bool ggml_zdnn_compute_forward( @@ -34,6 +39,11 @@ static bool ggml_zdnn_compute_forward( ggml_zdnn_compute_forward_mul_mat(ctx, dst); } break; + case GGML_OP_FLASH_ATTN_EXT: + { + ggml_zdnn_flash_attn_ext(ctx, dst); + } break; + default: return false; } @@ -76,6 +86,19 @@ static enum ggml_status ggml_zdnn_graph_compute(ggml_backend_t backend, ggml_cgr GGML_UNUSED(ctx_dev); } +static int64_t ggml_zdnn_mul_mat_min_batch(ggml_type type, int32_t n_threads) { + switch (type) { + case GGML_TYPE_Q8_0: + return std::max<int64_t>(1, n_threads - 2); + default: + return 1; + } +} + +static bool ggml_zdnn_is_tensor_own(const ggml_tensor * tensor) { + return tensor->buffer != nullptr && tensor->buffer->buft == ggml_backend_zdnn_buffer_type(); +} + static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_dev, const ggml_tensor * op) { switch (op->op) { case GGML_OP_NONE: @@ -96,23 +119,49 @@ static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_d const int64_t max_batch = ctx_dev->max_size; + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; + } + if (!ggml_is_matrix(weights) || !ggml_is_matrix(inputs) || !ggml_is_contiguous(weights) || !ggml_is_contiguous(inputs) || - weights->view_src != nullptr || inputs->view_src != nullptr || + weights->view_src != nullptr || ne0 > max_batch || ne1 > max_batch || ne10 > max_batch) { return false; } + // offload smaller batches to the CPU + if (ne1 < ggml_zdnn_mul_mat_min_batch(weights->type, ctx_dev->n_threads)) { + return false; + } + + // tensors with no buffer yet get a zDNN buffer, computed or graph input weights and tensors in other buffers have no ztensor + if (weights->op != GGML_OP_NONE || (weights->flags & GGML_TENSOR_FLAG_INPUT) || + (weights->buffer != nullptr && !ggml_zdnn_is_tensor_own(weights)) || + (op->buffer != nullptr && !ggml_zdnn_is_tensor_own(op))) { + return false; + } + + // training writes allocated weights through ->data, test-backend-ops sets PARAM before it allocates and uses set_tensor + if ((weights->flags & GGML_TENSOR_FLAG_PARAM) && weights->buffer != nullptr) { + return false; + } + switch (weights->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_BF16: return true; + case GGML_TYPE_Q8_0: + return ctx_dev->has_parmblkformat_1 && inputs->type == GGML_TYPE_F32; default: return false; } } break; + case GGML_OP_FLASH_ATTN_EXT: + return ggml_zdnn_supports_flash_attn_ext(ctx_dev, op); + default: return false; } @@ -134,6 +183,7 @@ static ggml_backend_zdnn_device_context g_ggml_ctx_dev_main = { /* .has_parmblkformat_0 = */ false, /* .has_parmblkformat_1 = */ false, /* .max_size = */ 0, + /* .n_threads = */ 1, /* .name = */ "", }; @@ -148,6 +198,8 @@ static int ggml_backend_zdnn_device_acq(ggml_backend_zdnn_device_context * ctx) ctx->has_parmblkformat_0 = zdnn_is_nnpa_parmblk_fmt_installed(1, NNPA_PARMBLKFORMAT_0); ctx->has_parmblkformat_1 = zdnn_is_nnpa_parmblk_fmt_installed(1, NNPA_PARMBLKFORMAT_1); ctx->max_size = zdnn_get_nnpa_max_dim_idx_size(); + // llama.cpp uses every core by default, and splits the first graph before it sets the threads + ctx->n_threads = std::max<int32_t>(1, (int32_t)std::thread::hardware_concurrency()); strncpy(ctx->name, GGML_ZDNN_NAME, sizeof(ctx->name) - 1); } @@ -186,6 +238,7 @@ static ggml_backend_zdnn_context * ggml_zdnn_init(ggml_backend_dev_t dev) { GGML_LOG_INFO("%s: NNPA_PARMBLKFORMAT_0 = %s\n", __func__, ctx_dev->has_parmblkformat_0 ? "true" : "false"); GGML_LOG_INFO("%s: NNPA_PARMBLKFORMAT_1 = %s\n", __func__, ctx_dev->has_parmblkformat_1 ? "true" : "false"); + ctx->n_threads = GGML_DEFAULT_N_THREADS; ctx->gf = nullptr; return ctx; @@ -193,6 +246,11 @@ static ggml_backend_zdnn_context * ggml_zdnn_init(ggml_backend_dev_t dev) { static void ggml_zdnn_free(ggml_backend_zdnn_context * ctx) { GGML_LOG_INFO("%s: deallocating\n", __func__); + + for (const auto & it : ctx->zero_bias) { + ZDNN_CHECK(zdnn_free_ztensor_buffer(&it.second->ztensor)); + } + delete ctx; } @@ -234,6 +292,11 @@ static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer return GGML_STATUS_SUCCESS; } + // ops read other compute tensors through ->data + if (tensor->op != GGML_OP_NONE && tensor->op != GGML_OP_MUL_MAT) { + return GGML_STATUS_SUCCESS; + } + ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context; const int64_t tsize = ggml_nbytes(tensor); @@ -284,21 +347,45 @@ static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer GGML_UNUSED(buffer_idx); } +static void ggml_zdnn_reset_tensor(const ggml_tensor * tensor) { + const ggml_tensor * src = tensor->view_src != nullptr ? tensor->view_src : tensor; + + ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)src->extra; + if (extra != nullptr) { + zdnn_reset_ztensor(&extra->ztensor); + } +} + static void ggml_backend_zdnn_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { memset((char *)tensor->data + offset, value, size); + ggml_zdnn_reset_tensor(tensor); + GGML_UNUSED(buffer); } static void ggml_backend_zdnn_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { memcpy((char *)tensor->data + offset, data, size); + ggml_zdnn_reset_tensor(tensor); + ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)tensor->extra; + if (extra == nullptr) { + return; + } + + if (ggml_is_quantized(tensor->type)) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_zdnn_load_tensor_q(extra, tensor); + return; + } // Fixes the LLAMA_SET_ROWS bug // see: https://github.com/ggml-org/llama.cpp/issues/15414 - if (tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_COMPUTE && extra->ztensor.is_transformed) zdnn_reset_ztensor(&extra->ztensor); - if (extra->ztensor.is_transformed == false) ggml_zdnn_load_tensor(extra->ztensor, tensor->data); + // only complete writes to weights are transformed here, others on first use, as not every op reads the ztensor and some data (masks) is not finite + if (tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && size == ggml_nbytes(tensor)) { + ggml_zdnn_load_tensor(extra->ztensor, tensor->data); + } GGML_UNUSED(buffer); } @@ -313,6 +400,17 @@ static void ggml_backend_zdnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context; memset(ctx->all_data, value, ctx->all_size); + + for (const auto & buf_ptr : ctx->buffers) { + zdnn_reset_ztensor(&buf_ptr->ztensor); + } + + // the bias has its own data, and no op transforms it on first use + for (const auto & buf_ptr : ctx->buffers) { + if (buf_ptr->extra != nullptr) { + ggml_zdnn_load_tensor(buf_ptr->extra->ztensor, buf_ptr->extra->data); + } + } } static void ggml_backend_zdnn_buffer_reset(ggml_backend_buffer_t buffer) { @@ -349,6 +447,9 @@ static ggml_backend_buffer_i ggml_backend_zdnn_buffer_i = { /* .cpy_tensor = */ NULL, /* .clear = */ ggml_backend_zdnn_buffer_clear, /* .reset = */ ggml_backend_zdnn_buffer_reset, + /* .set_tensor_n = */ NULL, + /* .get_tensor_n = */ NULL, + /* .cpy_tensor_n = */ NULL, }; // @@ -405,8 +506,7 @@ static size_t ggml_backend_zdnn_buffer_type_get_alignment(ggml_backend_buffer_ty } static bool ggml_backend_zdnn_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - /* while it resides in host memory, additional transformation is needed */ - return false; + return true; GGML_UNUSED(buft); } @@ -556,8 +656,7 @@ static bool ggml_backend_zdnn_device_supports_op(ggml_backend_dev_t dev, const g } static bool ggml_backend_zdnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - return - buft->iface.get_name == ggml_backend_zdnn_buffer_type_get_name; + return ggml_backend_buft_is_host(buft); GGML_UNUSED(dev); } @@ -621,10 +720,23 @@ static ggml_backend_feature * ggml_backend_zdnn_get_features(ggml_backend_reg_t GGML_UNUSED(reg); } +static void ggml_backend_zdnn_set_n_threads(ggml_backend_t backend, int n_threads) { + GGML_ASSERT(ggml_backend_is_zdnn(backend)); + + ggml_backend_zdnn_context * ctx = (ggml_backend_zdnn_context *)backend->context; + ctx->n_threads = n_threads; + + ggml_backend_zdnn_device_context * ctx_dev = (ggml_backend_zdnn_device_context *)backend->device->context; + ctx_dev->n_threads = n_threads; +} + static void * ggml_backend_zdnn_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *) ggml_backend_zdnn_get_features; } + if (strcmp(name, "ggml_backend_set_n_threads") == 0) { + return (void *) ggml_backend_zdnn_set_n_threads; + } return NULL; @@ -651,7 +763,7 @@ ggml_backend_reg_t ggml_backend_zdnn_reg(void) { { g_ggml_backend_zdnn_reg = { - /* .api_version = */ GGML_ZDNN_VERSION, + /* .api_version = */ GGML_BACKEND_API_VERSION, /* .iface = */ ggml_backend_zdnn_reg_i, /* .context = */ NULL }; diff --git a/ggml/src/ggml-zdnn/mmf.cpp b/ggml/src/ggml-zdnn/mmf.cpp index 3ac9cf3c9..acd443e5b 100644 --- a/ggml/src/ggml-zdnn/mmf.cpp +++ b/ggml/src/ggml-zdnn/mmf.cpp @@ -1,8 +1,9 @@ #include "ggml.h" #include "mmf.hpp" +#include "utils.hpp" void ggml_zdnn_mul_mat_f( - const ggml_backend_zdnn_context * ctx, + ggml_backend_zdnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -30,10 +31,14 @@ void ggml_zdnn_mul_mat_f( ggml_tensor * output = dst; ggml_backend_zdnn_buffer * weights_extra = (ggml_backend_zdnn_buffer *)weights->extra; - ggml_backend_zdnn_buffer * inputs_extra = (ggml_backend_zdnn_buffer *)inputs->extra; ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra; ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra; + zdnn_tensor_desc inputs_pre_tfm_desc; + zdnn_tensor_desc inputs_tfm_desc; + zdnn_ztensor inputs_ztensor; + ggml_zdnn_pool_alloc<uint8_t> inputs_ztensor_alloc; + const int64_t weights_rows = ne01; const int64_t weights_cols = ne00; const int64_t inputs_rows = ne11; @@ -52,29 +57,29 @@ void ggml_zdnn_mul_mat_f( // weights_extra->pre_tfm_desc.dim3, // weights_extra->pre_tfm_desc.dim4); - // GGML_LOG_INFO("%s: tensor '%s' tensor dimensions: [%ld, %ld, %ld, %ld] pre_tfm_desc dimensions: [%ld, %ld, %ld, %ld]\n", - // __func__, inputs_extra->name, - // inputs->ne[3], inputs->ne[2], inputs->ne[1], inputs->ne[0], - // inputs_extra->pre_tfm_desc.dim1, - // inputs_extra->pre_tfm_desc.dim2, - // inputs_extra->pre_tfm_desc.dim3, - // inputs_extra->pre_tfm_desc.dim4); - GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[0] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[0]"); GGML_ASSERT(weights_extra->pre_tfm_desc.dim2 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim2 must match weights->ne[1]"); - GGML_ASSERT(inputs_extra->pre_tfm_desc.dim1 == inputs->ne[0] && "inputs_extra->pre_tfm_desc.dim1 must match inputs->ne[0]"); - GGML_ASSERT(inputs_extra->pre_tfm_desc.dim2 == inputs->ne[1] && "inputs_extra->pre_tfm_desc.dim2 must match inputs->ne[1]"); - ZDNN_CHECK(zdnn_matmul_transpose_op(&inputs_extra->ztensor, &weights_extra->ztensor, &bias_extra->ztensor, + // weights read straight into a host buffer skip set_tensor, so transform them on first use + if (!weights_extra->ztensor.is_transformed) { + ggml_zdnn_load_tensor(weights_extra->ztensor, weights->data); + } + + // the inputs can be in any host buffer or be a view, so transform them from their data on every call + { + zdnn_init_pre_transformed_desc(ZDNN_2D, ggml_zdnn_type_mapping(inputs->type), &inputs_pre_tfm_desc, inputs_rows, inputs_cols); + ZDNN_CHECK(zdnn_generate_transformed_desc(&inputs_pre_tfm_desc, &inputs_tfm_desc)); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &inputs_pre_tfm_desc, &inputs_tfm_desc, &inputs_ztensor, inputs_ztensor_alloc); + ggml_zdnn_load_tensor(inputs_ztensor, inputs->data); + } + + ZDNN_CHECK(zdnn_matmul_transpose_op(&inputs_ztensor, &weights_extra->ztensor, &bias_extra->ztensor, false, true, MATMUL_OP_ADDITION, &output_extra->ztensor)); // TODO: Remove in the future as we are currently DLF16 -> FP32 then in the next op, FP32 -> DLF16 again. Inefficient. ZDNN_CHECK(zdnn_transform_origtensor(&output_extra->ztensor, output->data)); - GGML_UNUSED(ctx); GGML_UNUSED(weights_rows); GGML_UNUSED(weights_cols); - GGML_UNUSED(inputs_rows); - GGML_UNUSED(inputs_cols); GGML_UNUSED(output_rows); GGML_UNUSED(output_cols); } diff --git a/ggml/src/ggml-zdnn/mmf.hpp b/ggml/src/ggml-zdnn/mmf.hpp index a12f1b8f8..afc02bc0e 100644 --- a/ggml/src/ggml-zdnn/mmf.hpp +++ b/ggml/src/ggml-zdnn/mmf.hpp @@ -4,7 +4,7 @@ #include "common.hpp" void ggml_zdnn_mul_mat_f( - const ggml_backend_zdnn_context * ctx, + ggml_backend_zdnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); diff --git a/ggml/src/ggml-zdnn/mmq.cpp b/ggml/src/ggml-zdnn/mmq.cpp new file mode 100644 index 000000000..285479b41 --- /dev/null +++ b/ggml/src/ggml-zdnn/mmq.cpp @@ -0,0 +1,282 @@ +#include "ggml.h" +#include "mmq.hpp" +#include "quantize.hpp" +#include "utils.hpp" + +#include <algorithm> +#include <future> +#include <vector> + +// the weights are stacked into groups along ne00, and each group of each row of the inputs and weights has its own scale +// zDNN multiplies the groups in stacked calls, then the partial results of each group are scaled and summed here +static void ggml_zdnn_mul_mat_q_grouped(ggml_backend_zdnn_context * ctx, + int32_t n_threads, + const ggml_tensor * inputs, + const ggml_backend_zdnn_buffer * weights_extra, + ggml_tensor * output) { + const int64_t ne10 = inputs->ne[0]; + const int64_t ne11 = inputs->ne[1]; + const int64_t ne0 = output->ne[0]; + + const int64_t n_groups = weights_extra->pre_tfm_desc.dim3; + const int64_t gs = weights_extra->pre_tfm_desc.dim2; + + // the last group is zero padded when gs does not divide ne10 + GGML_ASSERT(n_groups * gs >= ne10 && (n_groups - 1) * gs < ne10); + + // the rounding errors of the inputs are stacked after the inputs, see ggml_zdnn_quantize_inputs + ggml_zdnn_pool_alloc<int8_t> inputs_q_alloc(ctx->pool, 2 * n_groups * gs * ne11); + int8_t * inputs_q = inputs_q_alloc.get(); + std::vector<float> inputs_scales(n_groups * ne11); + ggml_zdnn_quantize_inputs(n_threads, inputs, gs, weights_extra->channel_scales.data(), + inputs_q, inputs_q + n_groups * gs * ne11, inputs_scales.data()); + + zdnn_tensor_desc inputs_pre_tfm_desc; + zdnn_tensor_desc inputs_tfm_desc; + zdnn_ztensor inputs_ztensor; + ggml_zdnn_pool_alloc<uint8_t> inputs_ztensor_alloc; + + zdnn_init_pre_transformed_desc(ZDNN_3DS, INT8, &inputs_pre_tfm_desc, 2 * n_groups, ne11, gs); + ZDNN_CHECK(zdnn_generate_quantized_transformed_desc(&inputs_pre_tfm_desc, QUANTIZED_INT8, &inputs_tfm_desc)); + ggml_zdnn_init_scratch_ztensor(ctx, &inputs_pre_tfm_desc, &inputs_tfm_desc, &inputs_ztensor, inputs_ztensor_alloc); + ZDNN_CHECK(zdnn_transform_quantized_ztensor(&inputs_ztensor, false, -127, 127, inputs_q)); + + // each group of the partial results is read back as its own 2D ztensor + zdnn_tensor_desc group_pre_tfm_desc; + zdnn_tensor_desc group_tfm_desc; + + zdnn_init_pre_transformed_desc(ZDNN_2D, FP32, &group_pre_tfm_desc, ne11, ne0); + ZDNN_CHECK(zdnn_generate_transformed_desc(&group_pre_tfm_desc, &group_tfm_desc)); + + const uint64_t group_bytes = zdnn_getsize_ztensor(&group_tfm_desc); + + // stacked calls with a large output are much slower per group, so run the groups in chunks + // 8 MiB per chunk was benchmarked to be the best on Llama-3.2-1B Q8_0 + // always re-measure with different models to see if different sizes need a dispatching logic + constexpr int64_t chunk_bytes = 8*1024*1024; + + const int64_t n_chunk = std::clamp<int64_t>(chunk_bytes / (int64_t)group_bytes, 1, n_groups); + + // all zeros, so it is already the pre-computed bias for every chunk + const zdnn_ztensor & bias_ztensor = ggml_zdnn_zero_bias(ctx, n_chunk, ne0); + + // two partial buffers, so the next chunk runs on the NNPA while this one is summed + // each holds the results of a chunk of inputs, followed by the results of their rounding errors + zdnn_tensor_desc partial_pre_tfm_desc; + zdnn_tensor_desc partial_tfm_desc; + zdnn_ztensor partial_ztensor[2]; + ggml_zdnn_pool_alloc<uint8_t> partial_ztensor_alloc[2]; + + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &partial_pre_tfm_desc, 2 * n_chunk, ne11, ne0); + ZDNN_CHECK(zdnn_generate_quantized_transformed_desc(&partial_pre_tfm_desc, QUANTIZED_DLFLOAT16, &partial_tfm_desc)); + GGML_ASSERT(zdnn_getsize_ztensor(&partial_tfm_desc) == (uint64_t)(2 * n_chunk) * group_bytes); + for (int32_t i = 0; i < 2; i++) { + ggml_zdnn_init_scratch_ztensor(ctx, &partial_pre_tfm_desc, &partial_tfm_desc, &partial_ztensor[i], partial_ztensor_alloc[i]); + } + + const float * weights_scales = weights_extra->scales.data(); + ggml_zdnn_pool_alloc<float> partial_alloc(ctx->pool, 2 * ne11 * ne0); + float * partial = partial_alloc.get(); + float * partial_res = partial + ne11 * ne0; + + const auto sum_chunk = [&](const zdnn_ztensor & chunk, int64_t g0, int64_t n) { + zdnn_ztensor group_ztensor; + zdnn_init_ztensor(&group_pre_tfm_desc, &group_tfm_desc, &group_ztensor); + + for (int64_t g = g0; g < g0 + n; g++) { + group_ztensor.buffer_size = group_bytes; + group_ztensor.is_transformed = true; + + group_ztensor.buffer = (char *)chunk.buffer + (g - g0)*group_bytes; + ZDNN_CHECK(zdnn_transform_origtensor(&group_ztensor, partial)); + + group_ztensor.buffer = (char *)chunk.buffer + (n_chunk + g - g0)*group_bytes; + ZDNN_CHECK(zdnn_transform_origtensor(&group_ztensor, partial_res)); + + const float * sw = weights_scales + g*ne0; + for (int64_t i1 = 0; i1 < ne11; i1++) { + const float sa = inputs_scales[g*ne11 + i1]; + const float * p = partial + i1*ne0; + const float * pr = partial_res + i1*ne0; + float * y = (float *)output->data + i1*ne0; + + if (g == 0) { + for (int64_t i0 = 0; i0 < ne0; i0++) { + y[i0] = sa * sw[i0] * (p[i0] + pr[i0] * (1.0f/254.0f)); + } + } else { + for (int64_t i0 = 0; i0 < ne0; i0++) { + y[i0] += sa * sw[i0] * (p[i0] + pr[i0] * (1.0f/254.0f)); + } + } + } + } + }; + + zdnn_tensor_desc view_pre_tfm_desc[6]; + zdnn_tensor_desc view_tfm_desc[6]; + std::future<void> sum_task; + + for (int64_t c = 0, g0 = 0; g0 < n_groups; c++, g0 += n_chunk) { + const int64_t n = std::min(n_chunk, n_groups - g0); + + const zdnn_ztensor inputs_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[0], &view_tfm_desc[0], inputs_ztensor, 2 * n_groups, g0, n); + const zdnn_ztensor inputs_res_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[1], &view_tfm_desc[1], inputs_ztensor, 2 * n_groups, n_groups + g0, n); + const zdnn_ztensor weights_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[2], &view_tfm_desc[2], weights_extra->ztensor, n_groups, g0, n); + const zdnn_ztensor bias_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[3], &view_tfm_desc[3], bias_ztensor, n_chunk, 0, n); + + // the partial buffer of chunk c - 2 is summed by now, as sum_task is waited on before starting the next sum + zdnn_ztensor & chunk = partial_ztensor[c % 2]; + chunk.pre_transformed_desc = &partial_pre_tfm_desc; + chunk.transformed_desc = &partial_tfm_desc; + chunk.is_transformed = false; + + zdnn_ztensor chunk_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[4], &view_tfm_desc[4], chunk, 2 * n_chunk, 0, n); + zdnn_ztensor chunk_res_view = ggml_zdnn_stack_view(&view_pre_tfm_desc[5], &view_tfm_desc[5], chunk, 2 * n_chunk, n_chunk, n); + ZDNN_CHECK(zdnn_quantized_matmul_op(&inputs_view, &weights_view, &bias_view, + MATMUL_OP_ADDITION, -127, 127, true, false, true, + nullptr, &chunk_view)); + ZDNN_CHECK(zdnn_quantized_matmul_op(&inputs_res_view, &weights_view, &bias_view, + MATMUL_OP_ADDITION, -127, 127, true, false, true, + nullptr, &chunk_res_view)); + + if (sum_task.valid()) { + sum_task.get(); + } + + if (n_threads > 1) { + sum_task = std::async(std::launch::async, sum_chunk, chunk, g0, n); + } else { + sum_chunk(chunk, g0, n); + } + } + + if (sum_task.valid()) { + sum_task.get(); + } + +} + +void ggml_zdnn_mul_mat_q( + ggml_backend_zdnn_context * ctx, + const ggml_tensor * src0, + const ggml_tensor * src1, + ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; + + const enum ggml_type type = src0->type; + + GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne1 == ne11); + GGML_ASSERT(ne2 == ne12); + GGML_ASSERT(ne3 == ne13); + + // we don't support permuted src0 or src1 + GGML_ASSERT(nb00 == ggml_type_size(type)); + GGML_ASSERT(nb10 == ggml_type_size(src1->type)); + + // dst cannot be transposed or permuted + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(nb0 <= nb1); + GGML_ASSERT(nb1 <= nb2); + GGML_ASSERT(nb2 <= nb3); + + const ggml_tensor * weights = src0; + const ggml_tensor * inputs = src1; + ggml_tensor * output = dst; + + GGML_ASSERT(inputs->type == GGML_TYPE_F32); + + ggml_backend_zdnn_buffer * weights_extra = (ggml_backend_zdnn_buffer *)weights->extra; + + // weights read straight into a host buffer skip set_tensor, so transform them on first use + if (!weights_extra->ztensor.is_transformed) { + ggml_zdnn_load_tensor_q(weights_extra, weights); + } + + // starting threads costs more than it saves for small inputs, e.g. during token generation + const int32_t n_threads = ne10*ne11 >= 65536 ? ctx->n_threads : 1; + + if (weights_extra->pre_tfm_desc.layout == ZDNN_3DS) { + GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[1]"); + + ggml_zdnn_mul_mat_q_grouped(ctx, n_threads, inputs, weights_extra, output); + return; + } + + ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra; + ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra; + + GGML_ASSERT(weights_extra->pre_tfm_desc.dim2 == weights->ne[0] && "weights_extra->pre_tfm_desc.dim2 must match weights->ne[0]"); + GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[1]"); + + // zDNN takes one scale per zTensor, so quantize each row of the inputs with its own scale + // the rounding errors of the inputs are stacked after the inputs, see ggml_zdnn_quantize_inputs + ggml_zdnn_pool_alloc<int8_t> inputs_q_alloc(ctx->pool, 2 * ne10 * ne11); + int8_t * inputs_q = inputs_q_alloc.get(); + std::vector<float> inputs_scales(ne11); + ggml_zdnn_quantize_inputs(n_threads, inputs, ne10, weights_extra->channel_scales.data(), + inputs_q, inputs_q + ne10 * ne11, inputs_scales.data()); + + zdnn_tensor_desc inputs_pre_tfm_desc; + zdnn_tensor_desc inputs_tfm_desc; + zdnn_ztensor inputs_ztensor; + ggml_zdnn_pool_alloc<uint8_t> inputs_ztensor_alloc; + + zdnn_init_pre_transformed_desc(ZDNN_3DS, INT8, &inputs_pre_tfm_desc, 2, ne11, ne10); + ZDNN_CHECK(zdnn_generate_quantized_transformed_desc(&inputs_pre_tfm_desc, QUANTIZED_INT8, &inputs_tfm_desc)); + ggml_zdnn_init_scratch_ztensor(ctx, &inputs_pre_tfm_desc, &inputs_tfm_desc, &inputs_ztensor, inputs_ztensor_alloc); + ZDNN_CHECK(zdnn_transform_quantized_ztensor(&inputs_ztensor, false, -127, 127, inputs_q)); + + zdnn_tensor_desc partial_pre_tfm_desc; + zdnn_tensor_desc partial_tfm_desc; + zdnn_ztensor partial_ztensor; + ggml_zdnn_pool_alloc<uint8_t> partial_ztensor_alloc; + + zdnn_init_pre_transformed_desc(ZDNN_3DS, FP32, &partial_pre_tfm_desc, 2, ne11, ne0); + ZDNN_CHECK(zdnn_generate_quantized_transformed_desc(&partial_pre_tfm_desc, QUANTIZED_DLFLOAT16, &partial_tfm_desc)); + ggml_zdnn_init_scratch_ztensor(ctx, &partial_pre_tfm_desc, &partial_tfm_desc, &partial_ztensor, partial_ztensor_alloc); + + // both stacks are multiplied by the same unstacked weights in one broadcast call + // the bias is all zeros, so it is already the pre-computed bias + ZDNN_CHECK(zdnn_quantized_matmul_op(&inputs_ztensor, &weights_extra->ztensor, &bias_extra->ztensor, + MATMUL_OP_ADDITION, -127, 127, true, false, true, + nullptr, &partial_ztensor)); + + // each stack of the partial results is read back as its own 2D ztensor + zdnn_tensor_desc stack_pre_tfm_desc; + zdnn_tensor_desc stack_tfm_desc; + + zdnn_init_pre_transformed_desc(ZDNN_2D, FP32, &stack_pre_tfm_desc, ne11, ne0); + ZDNN_CHECK(zdnn_generate_transformed_desc(&stack_pre_tfm_desc, &stack_tfm_desc)); + + const uint64_t stack_bytes = zdnn_getsize_ztensor(&stack_tfm_desc); + GGML_ASSERT(partial_ztensor.buffer_size == 2*stack_bytes); + + ggml_zdnn_pool_alloc<float> partial_res_alloc(ctx->pool, ne11 * ne0); + float * partial[2] = { (float *)output->data, partial_res_alloc.get() }; + + ggml_zdnn_parallel_for(n_threads, 2, [&](int64_t s_start, int64_t s_end) { + zdnn_ztensor stack_ztensor; + zdnn_init_ztensor(&stack_pre_tfm_desc, &stack_tfm_desc, &stack_ztensor); + + for (int64_t s = s_start; s < s_end; s++) { + stack_ztensor.buffer = (char *)partial_ztensor.buffer + s*stack_bytes; + stack_ztensor.buffer_size = stack_bytes; + stack_ztensor.is_transformed = true; + ZDNN_CHECK(zdnn_transform_origtensor(&stack_ztensor, partial[s])); + } + }); + + const float * weights_scales = weights_extra->scales.data(); + ggml_zdnn_parallel_for(n_threads, ne1, [&](int64_t i1_start, int64_t i1_end) { + for (int64_t i1 = i1_start; i1 < i1_end; i1++) { + const float sa = inputs_scales[i1]; + const float * pr = partial[1] + i1*ne0; + float * y = partial[0] + i1*ne0; + for (int64_t i0 = 0; i0 < ne0; i0++) { + y[i0] = sa * weights_scales[i0] * (y[i0] + pr[i0] * (1.0f/254.0f)); + } + } + }); +} diff --git a/ggml/src/ggml-zdnn/mmq.hpp b/ggml/src/ggml-zdnn/mmq.hpp new file mode 100644 index 000000000..3e49f0af8 --- /dev/null +++ b/ggml/src/ggml-zdnn/mmq.hpp @@ -0,0 +1,12 @@ +#ifndef GGML_ZDNN_MMQ_HPP +#define GGML_ZDNN_MMQ_HPP + +#include "common.hpp" + +void ggml_zdnn_mul_mat_q( + ggml_backend_zdnn_context * ctx, + const ggml_tensor * src0, + const ggml_tensor * src1, + ggml_tensor * dst); + +#endif // GGML_ZDNN_MMQ_HPP diff --git a/ggml/src/ggml-zdnn/quantize.cpp b/ggml/src/ggml-zdnn/quantize.cpp new file mode 100644 index 000000000..5012e92e9 --- /dev/null +++ b/ggml/src/ggml-zdnn/quantize.cpp @@ -0,0 +1,58 @@ +#include "ggml.h" +#include "quantize.hpp" + +#include <algorithm> +#include <cmath> + +// quantize each group of group_size values per row to int8, plus the rounding error into residuals for ~15 bits +void ggml_zdnn_quantize_inputs(int32_t n_threads, + const ggml_tensor * inputs, + int64_t group_size, + const float * channel_scales, + int8_t * quants, + int8_t * residuals, + float * scales) { + + const int64_t ne0 = inputs->ne[0]; + const int64_t ne1 = inputs->ne[1]; + const int64_t n_groups = (ne0 + group_size - 1) / group_size; + + ggml_zdnn_parallel_for(n_threads, ne1, [&](int64_t i1_start, int64_t i1_end) { + for (int64_t i1 = i1_start; i1 < i1_end; i1++) { + const float * xi = (const float *)((const char *)inputs->data + i1*inputs->nb[1]); + + for (int64_t g = 0; g < n_groups; g++) { + const float * xg = xi + g*group_size; + const float * cg = channel_scales + g*group_size; + const int64_t n = std::min(group_size, ne0 - g*group_size); + + float amax = 0.0f; + for (int64_t i0 = 0; i0 < n; i0++) { + amax = std::max(amax, fabsf(xg[i0] * cg[i0])); + } + + // the rounding error is at most d/2 + const float d = amax / 127.0f; + const float id = d ? 1.0f/d : 0.0f; + const float id_res = 254.0f * id; + scales[g*ne1 + i1] = d; + + int8_t * qg = quants + (g*ne1 + i1)*group_size; + int8_t * rg = residuals + (g*ne1 + i1)*group_size; + + for (int64_t i0 = 0; i0 < n; i0++) { + const float xc = xg[i0] * cg[i0]; + const float v = roundf(xc * id); + + qg[i0] = (int8_t)v; + rg[i0] = (int8_t)std::clamp(roundf((xc - v * d) * id_res), -127.0f, 127.0f); + } + + for (int64_t i0 = n; i0 < group_size; i0++) { + qg[i0] = 0; + rg[i0] = 0; + } + } + } + }); +} diff --git a/ggml/src/ggml-zdnn/quantize.hpp b/ggml/src/ggml-zdnn/quantize.hpp new file mode 100644 index 000000000..51acbe8ad --- /dev/null +++ b/ggml/src/ggml-zdnn/quantize.hpp @@ -0,0 +1,14 @@ +#ifndef GGML_ZDNN_QUANTIZE_HPP +#define GGML_ZDNN_QUANTIZE_HPP + +#include "common.hpp" + +void ggml_zdnn_quantize_inputs(int32_t n_threads, + const ggml_tensor * inputs, + int64_t group_size, + const float * channel_scales, + int8_t * quants, + int8_t * residuals, + float * scales); + +#endif // GGML_ZDNN_QUANTIZE_HPP diff --git a/ggml/src/ggml-zdnn/scratch.cpp b/ggml/src/ggml-zdnn/scratch.cpp new file mode 100644 index 000000000..f32541475 --- /dev/null +++ b/ggml/src/ggml-zdnn/scratch.cpp @@ -0,0 +1,45 @@ +#include "ggml.h" +#include "scratch.hpp" + +#include <algorithm> +#include <cstdint> +#include <cstdlib> + +ggml_zdnn_pool::~ggml_zdnn_pool() { + for (const buffer & b : cached) { + std::free(b.ptr); + } +} + +void * ggml_zdnn_pool::alloc(size_t size, size_t * actual_size) { + int64_t ibest = -1; + for (int64_t i = 0; i < (int64_t)cached.size(); i++) { + if (cached[i].size >= size && (ibest < 0 || cached[i].size < cached[ibest].size)) { + ibest = i; + } + } + + if (ibest >= 0) { + void * ptr = cached[ibest].ptr; + *actual_size = cached[ibest].size; + cached.erase(cached.begin() + ibest); + return ptr; + } + + // every cached buffer is too small, free the largest so the pool does not grow while sizes grow with the context + if (!cached.empty()) { + const auto largest = std::max_element(cached.begin(), cached.end(), [](const buffer & a, const buffer & b) { return a.size < b.size; }); + std::free(largest->ptr); + cached.erase(largest); + } + + // 5% more, so slowly growing sizes do not allocate every time + *actual_size = GGML_PAD(std::max<size_t>(1, (size_t)(1.05 * size)), 4096); + void * ptr = std::aligned_alloc(4096, *actual_size); + GGML_ASSERT(ptr != nullptr); + return ptr; +} + +void ggml_zdnn_pool::free(void * ptr, size_t size) { + cached.push_back({ ptr, size }); +} diff --git a/ggml/src/ggml-zdnn/scratch.hpp b/ggml/src/ggml-zdnn/scratch.hpp new file mode 100644 index 000000000..f2f2e21fc --- /dev/null +++ b/ggml/src/ggml-zdnn/scratch.hpp @@ -0,0 +1,61 @@ +#ifndef GGML_ZDNN_SCRATCH_HPP +#define GGML_ZDNN_SCRATCH_HPP + +#include "ggml.h" + +#include <cstddef> +#include <vector> + +// 4K aligned host buffers that are cached for reuse, see ggml_cuda_pool_leg +struct ggml_zdnn_pool { + struct buffer { + void * ptr = nullptr; + size_t size = 0; + }; + + std::vector<buffer> cached; + + ~ggml_zdnn_pool(); + + void * alloc(size_t size, size_t * actual_size); + void free(void * ptr, size_t size); +}; + +// a buffer of the pool that goes back to it at the end of its scope, see ggml_cuda_pool_alloc +template <typename T> +struct ggml_zdnn_pool_alloc { + ggml_zdnn_pool * pool = nullptr; + T * ptr = nullptr; + size_t actual_size = 0; + + ggml_zdnn_pool_alloc() = default; + + ggml_zdnn_pool_alloc(ggml_zdnn_pool & pool, size_t size) { + alloc(pool, size); + } + + ~ggml_zdnn_pool_alloc() { + if (ptr != nullptr) { + pool->free(ptr, actual_size); + } + } + + // size is in number of elements + T * alloc(ggml_zdnn_pool & pool, size_t size) { + GGML_ASSERT(ptr == nullptr); + this->pool = &pool; + ptr = (T *) pool.alloc(size * sizeof(T), &actual_size); + return ptr; + } + + T * get() { + return ptr; + } + + ggml_zdnn_pool_alloc(const ggml_zdnn_pool_alloc &) = delete; + ggml_zdnn_pool_alloc(ggml_zdnn_pool_alloc &&) = delete; + ggml_zdnn_pool_alloc & operator=(const ggml_zdnn_pool_alloc &) = delete; + ggml_zdnn_pool_alloc & operator=(ggml_zdnn_pool_alloc &&) = delete; +}; + +#endif // GGML_ZDNN_SCRATCH_HPP diff --git a/ggml/src/ggml-zdnn/utils.cpp b/ggml/src/ggml-zdnn/utils.cpp index 2977cb0fe..18e9879d0 100644 --- a/ggml/src/ggml-zdnn/utils.cpp +++ b/ggml/src/ggml-zdnn/utils.cpp @@ -1,6 +1,10 @@ #include "ggml.h" #include "utils.hpp" +#include <algorithm> +#include <cmath> +#include <cstdlib> + zdnn_data_types ggml_zdnn_type_mapping(ggml_type type) { switch (type) { case GGML_TYPE_F32: @@ -9,8 +13,6 @@ zdnn_data_types ggml_zdnn_type_mapping(ggml_type type) { return FP16; case GGML_TYPE_BF16: return BFLOAT; - case GGML_TYPE_Q8_0: - return INT8; case GGML_TYPE_I8: return INT8; case GGML_TYPE_I32: @@ -43,7 +45,141 @@ void ggml_zdnn_load_tensor(zdnn_ztensor & ztensor, void * buffer) { ZDNN_CHECK(zdnn_transform_ztensor(&ztensor, buffer)); } +// size of the groups along ne0 that get their own scale, or 0 for one scale per row +// the groups are spread evenly over ne0 and rounded up to 128 values, the last group is zero padded +int64_t ggml_zdnn_q_group_size(const ggml_tensor * tensor) { + // benchmarked on Llama-3.2-1B and Qwen2.5-1.5B Q8_0 with the channel scales: + // 2048 is 15-18% faster than 1024, and 1024 lowers the KL divergence to the CPU by 13-19% + // GGML_ZDNN_HIGH_PREC=1 selects 1024 for users that want the lower loss + // always re-measure with different models to see if different sizes need a dispatching logic + static const bool high_prec = std::getenv("GGML_ZDNN_HIGH_PREC") != nullptr && + std::atoi(std::getenv("GGML_ZDNN_HIGH_PREC")) != 0; + const int64_t group_size_max = high_prec ? 1024 : 2048; + + const int64_t ne0 = tensor->ne[0]; + if (ne0 <= group_size_max) { + return 0; + } + + const int64_t n_groups = (ne0 + group_size_max - 1) / group_size_max; + return GGML_PAD((ne0 + n_groups - 1) / n_groups, 128); +} + +void ggml_zdnn_load_tensor_q(ggml_backend_zdnn_buffer * buffer, const ggml_tensor * tensor) { + const int64_t ne0 = tensor->ne[0]; + const int64_t ne1 = tensor->ne[1]; + + const ggml_to_float_t to_float = ggml_get_type_traits(tensor->type)->to_float; + GGML_ASSERT(to_float != nullptr); + + // zDNN takes one scale per zTensor, so requantize each group of each row to int8 with its own scale + // a group is the whole row, unless the weights are stacked into groups along ne0 + // rows are written transposed as zDNN computes inputs (ne11, ne0) x weights (ne0, ne1) + const bool stacked = buffer->pre_tfm_desc.layout == ZDNN_3DS; + const int64_t n_groups = stacked ? buffer->pre_tfm_desc.dim3 : 1; + const int64_t gs = stacked ? buffer->pre_tfm_desc.dim2 : ne0; + + // benchmarked: 256 rows fill a 256-byte cache line per write, padding each row by a line avoids cache set conflicts + constexpr int64_t n_tile = 256; + const int64_t x_stride = ne0 + 64; + + // the rows past ne0 of the last group stay zero as padding + std::vector<int8_t> q(n_groups * gs * ne1); + std::vector<float> x(n_tile * x_stride); + std::vector<float> id(n_tile * n_groups); + + // a few input channels with larger weights than the rest would set the scale of every group, + // so divide each input channel by its RMS over the rows and multiply the inputs by it instead + // with groups of 1024, this lowered the KL divergence to the CPU by 10-18% on Llama-3.2-1B and Qwen2.5-1.5B Q8_0 at the same speed + std::vector<double> sum_sq(ne0, 0.0); + for (int64_t i1 = 0; i1 < ne1; i1++) { + to_float((const char *)tensor->data + i1*tensor->nb[1], x.data(), ne0); + for (int64_t i0 = 0; i0 < ne0; i0++) { + sum_sq[i0] += (double)x[i0] * x[i0]; + } + } + + std::vector<float> channel_id(ne0); + for (int64_t i0 = 0; i0 < ne0; i0++) { + const float c = (float)std::sqrt(sum_sq[i0] / ne1); + buffer->channel_scales[i0] = c ? c : 1.0f; + channel_id[i0] = 1.0f / buffer->channel_scales[i0]; + } + + for (int64_t i1 = 0; i1 < ne1; i1 += n_tile) { + const int64_t nt = std::min(n_tile, ne1 - i1); + + for (int64_t t = 0; t < nt; t++) { + float * xt = x.data() + t*x_stride; + to_float((const char *)tensor->data + (i1 + t)*tensor->nb[1], xt, ne0); + + for (int64_t i0 = 0; i0 < ne0; i0++) { + xt[i0] *= channel_id[i0]; + } + + for (int64_t g = 0; g < n_groups; g++) { + float amax = 0.0f; + for (int64_t i0 = g*gs; i0 < std::min((g + 1)*gs, ne0); i0++) { + amax = std::max(amax, fabsf(xt[i0])); + } + + const float d = amax / 127.0f; + buffer->scales[g*ne1 + i1 + t] = d; + id[t*n_groups + g] = d ? 1.0f/d : 0.0f; + } + } + + for (int64_t i0 = 0; i0 < ne0; i0++) { + const int64_t g = i0 / gs; + int8_t * qi = q.data() + i0*ne1 + i1; + for (int64_t t = 0; t < nt; t++) { + qi[t] = (int8_t)roundf(x[t*x_stride + i0] * id[t*n_groups + g]); + } + } + } + + if (buffer->ztensor.is_transformed) { + zdnn_reset_ztensor(&buffer->ztensor); + } + ZDNN_CHECK(zdnn_transform_quantized_ztensor(&buffer->ztensor, false, -127, 127, q.data())); +} + void ggml_zdnn_init_tensor(ggml_backend_zdnn_buffer * buffer, const ggml_tensor * tensor) { + if (ggml_is_quantized(tensor->type)) { + GGML_ASSERT(ggml_is_matrix(tensor)); + + const int64_t gs = ggml_zdnn_q_group_size(tensor); + + if (gs > 0) { + // stack the weights into groups along ne0, each group of each row gets its own scale + const int64_t n_groups = (tensor->ne[0] + gs - 1) / gs; + + zdnn_init_pre_transformed_desc( + ZDNN_3DS, + INT8, + &buffer->pre_tfm_desc, + n_groups, gs, tensor->ne[1] + ); + + buffer->scales.resize(n_groups * tensor->ne[1]); + } else { + zdnn_init_pre_transformed_desc( + ZDNN_2D, + INT8, + &buffer->pre_tfm_desc, + tensor->ne[0], tensor->ne[1] + ); + + buffer->scales.resize(tensor->ne[1]); + } + + buffer->channel_scales.resize(tensor->ne[0]); + + ZDNN_CHECK(zdnn_generate_quantized_transformed_desc(&buffer->pre_tfm_desc, QUANTIZED_WEIGHTS_INT8, &buffer->tfm_desc)); + ZDNN_CHECK(zdnn_init_quantized_ztensor_with_malloc(&buffer->pre_tfm_desc, &buffer->tfm_desc, 1.0f, 0.0f, &buffer->ztensor)); + return; + } + switch (tensor->op) { case GGML_OP_MUL_MAT: { diff --git a/ggml/src/ggml-zdnn/utils.hpp b/ggml/src/ggml-zdnn/utils.hpp index c1e2028ed..20f5be5b4 100644 --- a/ggml/src/ggml-zdnn/utils.hpp +++ b/ggml/src/ggml-zdnn/utils.hpp @@ -14,6 +14,10 @@ void ggml_zdnn_create_tensor(zdnn_tensor_desc & pre_tfm_desc, void ggml_zdnn_load_tensor(zdnn_ztensor & ztensor, void * buffer); +int64_t ggml_zdnn_q_group_size(const ggml_tensor * tensor); + +void ggml_zdnn_load_tensor_q(ggml_backend_zdnn_buffer * buffer, const ggml_tensor * tensor); + void ggml_zdnn_init_tensor(ggml_backend_zdnn_buffer * buffer, const ggml_tensor * tensor); #endif // GGML_ZDNN_UTILITIES_HPP diff --git a/ggml/src/ggml-zendnn/ggml-zendnn.cpp b/ggml/src/ggml-zendnn/ggml-zendnn.cpp index ec7ce2331..85f1263fb 100644 --- a/ggml/src/ggml-zendnn/ggml-zendnn.cpp +++ b/ggml/src/ggml-zendnn/ggml-zendnn.cpp @@ -375,7 +375,12 @@ static void ggml_zendnn_compute_forward_mul_mat_id( for (int id = 0; id < n_ids; ++id) { const int32_t i02 = *(const int32_t *)((const char *)ids->data + iid1*ids->nb[1] + id*ids->nb[0]); - GGML_ASSERT(i02 >= 0 && i02 < n_as); + GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as)); + + if (i02 == -1) { + memset((char *) dst->data + id*nb1 + iid1*nb2, 0, ne0*sizeof(float)); + continue; + } if (matrix_row_counts[i02] == 0) { n_active_experts++; @@ -715,6 +720,10 @@ static bool ggml_backend_zendnn_device_supports_op(ggml_backend_dev_t dev, const return false; } + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; + } + if (ggml_zendnn_adaptive_fallback_enabled()) { const int64_t K = inputs->ne[0]; const int64_t N = (inputs->ne[1]*inputs->ne[2]*inputs->ne[3]); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 9bcabdbd3..1fca258a5 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -71,12 +71,12 @@ uint64_t ggml_graph_next_uid(void) { // Needed for ggml_fp32_to_bf16_row() #if defined(__AVX512BF16__) -#if defined(_MSC_VER) +#if defined(_MSC_VER) && !defined(__clang__) #define m512i(p) p #else #include <immintrin.h> #define m512i(p) (__m512i)(p) -#endif // defined(_MSC_VER) +#endif // defined(_MSC_VER) && !defined(__clang__) #endif // defined(__AVX512BF16__) #if defined(__linux__) || \ @@ -329,6 +329,17 @@ void ggml_log_callback_default(enum ggml_log_level level, const char * text, voi #endif +// the thread sanitizer does not intercept vm_allocate and vm_deallocate, so it produces false positives +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define GGML_TSAN_ENABLED 1 +#endif +#else +#if defined(__SANITIZE_THREAD__) +#define GGML_TSAN_ENABLED 1 +#endif +#endif + void * ggml_aligned_malloc(size_t size) { #if defined(__s390x__) const int alignment = 256; @@ -346,7 +357,8 @@ void * ggml_aligned_malloc(size_t size) { void * aligned_memory = NULL; #ifdef GGML_USE_CPU_HBM int result = hbw_posix_memalign(&aligned_memory, alignment, size); - #elif TARGET_OS_OSX + #elif defined(__APPLE__) && !defined(GGML_TSAN_ENABLED) + // libmalloc can keep freed pages in the process footprint, vm_deallocate returns them at once GGML_UNUSED(alignment); kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE); int result = EFAULT; @@ -393,7 +405,7 @@ void ggml_aligned_free(void * ptr, size_t size) { if (ptr != NULL) { hbw_free(ptr); } -#elif TARGET_OS_OSX +#elif defined(__APPLE__) && !defined(GGML_TSAN_ENABLED) if (ptr != NULL) { vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size); } @@ -473,27 +485,25 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) { } void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) { - int i = 0; - for (; i < n; ++i) { + for (int64_t i = 0; i < n; i++) { y[i] = GGML_FP32_TO_FP16(x[i]); } } void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) { - int i = 0; - for (; i < n; ++i) { + for (int64_t i = 0; i < n; i++) { y[i] = GGML_BF16_TO_FP32(x[i]); } } void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) { - for (int i = 0; i < n; i++) { + for (int64_t i = 0; i < n; i++) { y[i] = ggml_compute_fp32_to_bf16(x[i]); } } void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) { - int i = 0; + int64_t i = 0; #if defined(__AVX512BF16__) // subnormals are flushed to zero on this platform for (; i + 32 <= n; i += 32) { @@ -535,9 +545,6 @@ static BOOL CALLBACK ggml_time_init_once(PINIT_ONCE once, PVOID param, PVOID *ct QueryPerformanceFrequency(&t); timer_freq = t.QuadPart; - // The multiplication by 1000 or 1000000 below can cause an overflow if timer_freq - // and the uptime is high enough. - // We subtract the program start time to reduce the likelihood of that happening. QueryPerformanceCounter(&t); timer_start = t.QuadPart; @@ -550,12 +557,12 @@ void ggml_time_init(void) { int64_t ggml_time_ms(void) { LARGE_INTEGER t; QueryPerformanceCounter(&t); - return ((t.QuadPart-timer_start) * 1000) / timer_freq; + return (int64_t) (((double) (t.QuadPart - timer_start) / timer_freq) * 1000); } int64_t ggml_time_us(void) { LARGE_INTEGER t; QueryPerformanceCounter(&t); - return ((t.QuadPart-timer_start) * 1000000) / timer_freq; + return (int64_t) (((double) (t.QuadPart - timer_start) / timer_freq) * 1000000); } #else void ggml_time_init(void) {} @@ -707,13 +714,13 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .from_float_ref = (ggml_from_float_t) quantize_row_q4_1_ref, }, [4] = { // GGML_TYPE_Q4_2 - .type_name = "DEPRECATED", + .type_name = "REMOVED", .blck_size = 0, .type_size = 0, .is_quantized = false, }, [5] = { // GGML_TYPE_Q4_3 - .type_name = "DEPRECATED", + .type_name = "REMOVED", .blck_size = 0, .type_size = 0, .is_quantized = false, @@ -765,6 +772,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_nvfp4, .from_float_ref = (ggml_from_float_t)quantize_row_nvfp4_ref, }, + [GGML_TYPE_F8_E4M3] = { + .type_name = "f8_e4m3", + .blck_size = 1, + .type_size = sizeof(ggml_fp8_e4m3_t), + .is_quantized = false, + .to_float = (ggml_to_float_t) dequantize_row_f8_e4m3, + .from_float_ref = (ggml_from_float_t) quantize_row_f8_e4m3_ref, + }, [GGML_TYPE_Q2_K] = { .type_name = "q2_K", .blck_size = QK_K, @@ -994,7 +1009,6 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "DUP", "ADD", "ADD_ID", - "ADD1", "ACC", "SUB", "MUL", @@ -1036,8 +1050,6 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "GET_ROWS_BACK", "SET_ROWS", "DIAG", - "DIAG_MASK_INF", - "DIAG_MASK_ZERO", "SOFT_MAX", "SOFT_MAX_BACK", "ROPE", @@ -1084,6 +1096,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "DSV4_HC_COMB", "DSV4_HC_PRE", "DSV4_HC_POST", + "TOP_K_MASK", "UNARY", @@ -1101,7 +1114,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "GLU", }; -static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101"); +static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1109,7 +1122,6 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "x", "x+y", "x[i]+y", - "x+y", "view(x,nb,offset)+=y->x", "x-y", "x*y", @@ -1151,8 +1163,6 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "get_rows_back(x)", "set_rows(x)", "diag(x)", - "diag_mask_inf(x)", - "diag_mask_zero(x)", "soft_max(x)", "soft_max_back(x)", "rope(x)", @@ -1199,6 +1209,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "dsv4_hc_comb(mixes, scale, base)", "dsv4_hc_pre(x, weights)", "dsv4_hc_post(x, residual, post, comb)", + "top_k_mask(mask, top_k, score, blocks, extra)", "unary(x)", @@ -1216,7 +1227,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "glu(x)", }; -static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101"); +static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -1343,12 +1354,6 @@ size_t ggml_row_size(enum ggml_type type, int64_t ne) { return ggml_type_size(type)*ne/ggml_blck_size(type); } -double ggml_type_sizef(enum ggml_type type) { - assert(type >= 0); - assert(type < GGML_TYPE_COUNT); - return ((double)(type_traits[type].type_size))/type_traits[type].blck_size; -} - const char * ggml_type_name(enum ggml_type type) { assert(type >= 0); assert(type < GGML_TYPE_COUNT); @@ -1440,6 +1445,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break; case GGML_FTYPE_MOSTLY_MXFP4: wtype = GGML_TYPE_MXFP4; break; case GGML_FTYPE_MOSTLY_NVFP4: wtype = GGML_TYPE_NVFP4; break; + case GGML_FTYPE_MOSTLY_F8_E4M3: wtype = GGML_TYPE_F8_E4M3; break; case GGML_FTYPE_MOSTLY_Q2_K: wtype = GGML_TYPE_Q2_K; break; case GGML_FTYPE_MOSTLY_Q3_K: wtype = GGML_TYPE_Q3_K; break; case GGML_FTYPE_MOSTLY_Q4_K: wtype = GGML_TYPE_Q4_K; break; @@ -1768,6 +1774,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( enum ggml_type type, int n_dims, const int64_t * ne, + const size_t * nb, struct ggml_tensor * view_src, size_t view_offs) { @@ -1794,8 +1801,6 @@ static struct ggml_tensor * ggml_new_tensor_impl( data_size *= ne[i]; } - GGML_ASSERT(view_src == NULL || data_size == 0 || data_size + view_offs <= ggml_nbytes(view_src)); - void * data = view_src != NULL ? view_src->data : NULL; if (data != NULL) { data = (char *) data + view_offs; @@ -1839,12 +1844,23 @@ static struct ggml_tensor * ggml_new_tensor_impl( result->ne[i] = ne[i]; } - result->nb[0] = ggml_type_size(type); - result->nb[1] = result->nb[0]*(result->ne[0]/ggml_blck_size(type)); - for (int i = 2; i < GGML_MAX_DIMS; i++) { - result->nb[i] = result->nb[i - 1]*result->ne[i - 1]; + // a view built with strides takes them from the caller, any other tensor is contiguous + if (nb != NULL) { + for (int i = 0; i < GGML_MAX_DIMS; i++) { + result->nb[i] = nb[i]; + } + } else { + result->nb[0] = ggml_type_size(type); + result->nb[1] = result->nb[0]*(result->ne[0]/ggml_blck_size(type)); + for (int i = 2; i < GGML_MAX_DIMS; i++) { + result->nb[i] = result->nb[i - 1]*result->ne[i - 1]; + } } + // a view stays inside its source, over the bytes its strides actually reach + const size_t view_size = ggml_nbytes(result); + GGML_ASSERT(view_src == NULL || view_size == 0 || view_size + view_offs <= ggml_nbytes(view_src)); + ctx->n_objects++; return result; @@ -1855,7 +1871,7 @@ struct ggml_tensor * ggml_new_tensor( enum ggml_type type, int n_dims, const int64_t * ne) { - return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL, 0); + return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL, NULL, 0); } struct ggml_tensor * ggml_new_tensor_1d( @@ -1972,13 +1988,9 @@ struct ggml_tensor * ggml_format_name(struct ggml_tensor * tensor, const char * struct ggml_tensor * ggml_view_tensor( struct ggml_context * ctx, struct ggml_tensor * src) { - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS, src->ne, src, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS, src->ne, src->nb, src, 0); ggml_format_name(result, "%s (view)", src->name); - for (int i = 0; i < GGML_MAX_DIMS; i++) { - result->nb[i] = src->nb[i]; - } - return result; } @@ -2148,39 +2160,6 @@ struct ggml_tensor * ggml_add_id( return result; } -// ggml_add1 - -static struct ggml_tensor * ggml_add1_impl( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - bool inplace) { - GGML_ASSERT(ggml_is_scalar(b)); - GGML_ASSERT(ggml_is_padded_1d(a)); - - struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - - result->op = GGML_OP_ADD1; - result->src[0] = a; - result->src[1] = b; - - return result; -} - -struct ggml_tensor * ggml_add1( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b) { - return ggml_add1_impl(ctx, a, b, false); -} - -struct ggml_tensor * ggml_add1_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b) { - return ggml_add1_impl(ctx, a, b, true); -} - // ggml_acc static struct ggml_tensor * ggml_acc_impl( @@ -2199,6 +2178,7 @@ static struct ggml_tensor * ggml_acc_impl( struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); + GGML_ASSERT(offset < (size_t)(1 << 30)); int32_t params[] = { nb1, nb2, nb3, offset, inplace ? 1 : 0 }; ggml_set_op_params(result, params, sizeof(params)); @@ -2927,7 +2907,7 @@ static struct ggml_tensor * ggml_glu_impl( } int64_t ne[GGML_MAX_DIMS] = { a->ne[0] / 2 }; for (int i = 1; i < GGML_MAX_DIMS; i++) ne[i] = a->ne[i]; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b ? a->ne : ne, NULL, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b ? a->ne : ne, NULL, NULL, 0); ggml_set_op_params_i32(result, 0, (int32_t) op); ggml_set_op_params_i32(result, 1, (int32_t) swapped); @@ -3337,6 +3317,36 @@ bool ggml_prec_set_src( return true; } +enum ggml_prec ggml_prec_get_acc( + const struct ggml_tensor * a) { + switch (a->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + return (enum ggml_prec) ggml_get_op_params_i32(a, 0); + case GGML_OP_FLASH_ATTN_EXT: + return (enum ggml_prec) ggml_get_op_params_i32(a, 3); + default: + return GGML_PREC_UNDEFINED; + }; +} + +enum ggml_prec ggml_prec_get_src( + const struct ggml_tensor * a, + int idx) { + GGML_ASSERT(idx >= 0 && idx < GGML_MAX_SRC); + + switch (a->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + if (idx != 1) { + return GGML_PREC_UNDEFINED; + } + return (enum ggml_prec) ggml_get_op_params_i32(a, 2 + idx); + default: + return GGML_PREC_UNDEFINED; + }; +} + // ggml_mul_mat static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { @@ -3364,16 +3374,6 @@ struct ggml_tensor * ggml_mul_mat( return result; } -void ggml_mul_mat_set_prec( - struct ggml_tensor * a, - enum ggml_prec prec) { - GGML_ASSERT(a->op == GGML_OP_MUL_MAT); - - const int32_t prec_i32 = (int32_t) prec; - - ggml_set_op_params_i32(a, 0, prec_i32); -} - void ggml_mul_mat_set_hint( struct ggml_tensor * a, enum ggml_op_hint hint) { @@ -3397,6 +3397,8 @@ void ggml_mul_mat_set_hint( in b, n_expert_used can be broadcasted to match the n_expert_used of ids c ~= as[:,:,i] @ b[:,i%r,t], i = ids[e,t] for all e,t in ids + + an id of -1 skips the slot: no matrix is selected, c[:,e,t] is set to zero */ struct ggml_tensor * ggml_mul_mat_id( struct ggml_context * ctx, @@ -3460,7 +3462,7 @@ static struct ggml_tensor * ggml_scale_impl( float s, float b, bool inplace) { - GGML_ASSERT(ggml_is_padded_1d(a)); + GGML_ASSERT(ggml_is_contiguous_rows(a)); struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); @@ -3704,7 +3706,7 @@ struct ggml_tensor * ggml_reshape( // as only the shape of b is relevant, and not its memory layout, b is allowed to be non contiguous. GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b)); - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b->ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b->ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3721,7 +3723,7 @@ struct ggml_tensor * ggml_reshape_1d( GGML_ASSERT(ggml_nelements(a) == ne0); const int64_t ne[1] = { ne0 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3739,7 +3741,7 @@ struct ggml_tensor * ggml_reshape_2d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1); const int64_t ne[2] = { ne0, ne1 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3758,7 +3760,7 @@ struct ggml_tensor * ggml_reshape_3d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2); const int64_t ne[3] = { ne0, ne1, ne2 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3778,7 +3780,7 @@ struct ggml_tensor * ggml_reshape_4d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2*ne3); const int64_t ne[4] = { ne0, ne1, ne2, ne3 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 4, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 4, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3792,8 +3794,12 @@ static struct ggml_tensor * ggml_view_impl( struct ggml_tensor * a, int n_dims, const int64_t * ne, + const size_t * nb, size_t offset) { - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, n_dims, ne, a, offset); + // a strided view counts its strides from elements that are contiguous along dim 0 of its source + GGML_ASSERT(nb == NULL || a->nb[0] == ggml_type_size(a->type)); + + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, n_dims, ne, nb, a, offset); ggml_format_name(result, "%s (view)", a->name); ggml_set_op_params(result, &offset, sizeof(offset)); @@ -3811,7 +3817,7 @@ struct ggml_tensor * ggml_view_1d( struct ggml_tensor * a, int64_t ne0, size_t offset) { - struct ggml_tensor * result = ggml_view_impl(ctx, a, 1, &ne0, offset); + struct ggml_tensor * result = ggml_view_impl(ctx, a, 1, &ne0, NULL, offset); return result; } @@ -3826,12 +3832,9 @@ struct ggml_tensor * ggml_view_2d( size_t nb1, size_t offset) { const int64_t ne[2] = { ne0, ne1 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb1*ne1, nb1*ne1 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 2, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = result->nb[1]*ne1; - result->nb[3] = result->nb[2]; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 2, ne, nb, offset); return result; } @@ -3848,12 +3851,9 @@ struct ggml_tensor * ggml_view_3d( size_t nb2, size_t offset) { const int64_t ne[3] = { ne0, ne1, ne2 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb2, nb2*ne2 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 3, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = nb2; - result->nb[3] = result->nb[2]*ne2; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 3, ne, nb, offset); return result; } @@ -3872,12 +3872,9 @@ struct ggml_tensor * ggml_view_4d( size_t nb3, size_t offset) { const int64_t ne[4] = { ne0, ne1, ne2, ne3 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb2, nb3 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 4, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = nb2; - result->nb[3] = nb3; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 4, ne, nb, offset); return result; } @@ -4050,70 +4047,6 @@ struct ggml_tensor * ggml_diag( return result; } -// ggml_diag_mask_inf - -static struct ggml_tensor * ggml_diag_mask_inf_impl( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past, - bool inplace) { - struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - - int32_t params[] = { n_past }; - ggml_set_op_params(result, params, sizeof(params)); - - result->op = GGML_OP_DIAG_MASK_INF; - result->src[0] = a; - - return result; -} - -struct ggml_tensor * ggml_diag_mask_inf( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past) { - return ggml_diag_mask_inf_impl(ctx, a, n_past, false); -} - -struct ggml_tensor * ggml_diag_mask_inf_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past) { - return ggml_diag_mask_inf_impl(ctx, a, n_past, true); -} - -// ggml_diag_mask_zero - -static struct ggml_tensor * ggml_diag_mask_zero_impl( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past, - bool inplace) { - struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - - int32_t params[] = { n_past }; - ggml_set_op_params(result, params, sizeof(params)); - - result->op = GGML_OP_DIAG_MASK_ZERO; - result->src[0] = a; - - return result; -} - -struct ggml_tensor * ggml_diag_mask_zero( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past) { - return ggml_diag_mask_zero_impl(ctx, a, n_past, false); -} - -struct ggml_tensor * ggml_diag_mask_zero_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past) { - return ggml_diag_mask_zero_impl(ctx, a, n_past, true); -} - // ggml_clamp static struct ggml_tensor * ggml_clamp_impl( @@ -4307,7 +4240,7 @@ static struct ggml_tensor * ggml_rope_impl( struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - int32_t params[16] = { /*n_past*/ 0, n_dims, mode, /*n_ctx*/ 0, n_ctx_orig }; + int32_t params[16] = { /*n_past*/ 0, n_dims, mode, /*no_trunc*/ 0, n_ctx_orig }; memcpy(params + 5, &freq_base, sizeof(float)); memcpy(params + 6, &freq_scale, sizeof(float)); memcpy(params + 7, &ext_factor, sizeof(float)); @@ -4435,44 +4368,6 @@ struct ggml_tensor * ggml_rope_ext_inplace( ); } -struct ggml_tensor * ggml_rope_custom( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - int mode, - int n_ctx_orig, - float freq_base, - float freq_scale, - float ext_factor, - float attn_factor, - float beta_fast, - float beta_slow) { - return ggml_rope_impl( - ctx, a, b, NULL, n_dims, NULL, mode, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow, false - ); -} - -struct ggml_tensor * ggml_rope_custom_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - int mode, - int n_ctx_orig, - float freq_base, - float freq_scale, - float ext_factor, - float attn_factor, - float beta_fast, - float beta_slow) { - return ggml_rope_impl( - ctx, a, b, NULL, n_dims, NULL, mode, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow, true - ); -} - // Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get // `corr_dim(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))` static float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) { @@ -4480,11 +4375,15 @@ static float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, fl } void ggml_rope_yarn_corr_dims( - int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2] + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, bool truncate, float dims[2] ) { // start and end correction dims - float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base)); - float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base)); + float start = ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base); + float end = ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base); + if (truncate) { + start = floorf(start); + end = ceilf(end); + } dims[0] = MAX(0, start); dims[1] = MIN(n_dims - 1, end); } @@ -4545,6 +4444,19 @@ struct ggml_tensor * ggml_rope_set_offset( return a; } +struct ggml_tensor * ggml_rope_set_truncate( + struct ggml_tensor * a, + bool truncate) { + GGML_ASSERT(a->op == GGML_OP_ROPE || a->op == GGML_OP_ROPE_BACK); + ggml_set_op_params_i32(a, 3, !truncate); + return a; +} + +bool ggml_rope_get_truncate(const struct ggml_tensor * a) { + GGML_ASSERT(a->op == GGML_OP_ROPE || a->op == GGML_OP_ROPE_BACK); + return ggml_get_op_params_i32(a, 3) == 0; +} + static int64_t ggml_calc_conv_output_size(int64_t ins, int64_t ks, int s, int p, int d) { return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; } @@ -4636,7 +4548,8 @@ struct ggml_tensor * ggml_conv_1d( ggml_reshape_2d(ctx, im2col, im2col->ne[0], (im2col->ne[2] * im2col->ne[1])), // [N, OL, IC * K] => [N*OL, IC * K] ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1]), a->ne[2])); // [OC,IC, K] => [OC, IC * K] - result = ggml_reshape_3d(ctx, result, im2col->ne[1], a->ne[2], im2col->ne[2]); // [N, OC, OL] + result = ggml_reshape_3d(ctx, result, im2col->ne[1], im2col->ne[2], a->ne[2]); // [OC, N, OL] + result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 2, 1, 3)); // [N, OC, OL] return result; } @@ -5194,17 +5107,6 @@ struct ggml_tensor * ggml_upscale( return ggml_interpolate_impl(ctx, a, a->ne[0] * scale_factor, a->ne[1] * scale_factor, a->ne[2], a->ne[3], mode); } -struct ggml_tensor * ggml_upscale_ext( - struct ggml_context * ctx, - struct ggml_tensor * a, - int ne0, - int ne1, - int ne2, - int ne3, - enum ggml_scale_mode mode) { - return ggml_interpolate_impl(ctx, a, ne0, ne1, ne2, ne3, mode); -} - struct ggml_tensor * ggml_interpolate( struct ggml_context * ctx, struct ggml_tensor * a, @@ -5335,10 +5237,10 @@ struct ggml_tensor * ggml_roll( int shift2, int shift3) { GGML_ASSERT(a->nb[0] == ggml_type_size(a->type)); - GGML_ASSERT(abs(shift0) < a->ne[0]); - GGML_ASSERT(abs(shift1) < a->ne[1]); - GGML_ASSERT(abs(shift2) < a->ne[2]); - GGML_ASSERT(abs(shift3) < a->ne[3]); + GGML_ASSERT(shift0 > -a->ne[0] && shift0 < a->ne[0]); + GGML_ASSERT(shift1 > -a->ne[1] && shift1 < a->ne[1]); + GGML_ASSERT(shift2 > -a->ne[2] && shift2 < a->ne[2]); + GGML_ASSERT(shift3 > -a->ne[3] && shift3 < a->ne[3]); struct ggml_tensor * result = ggml_dup_tensor(ctx, a); @@ -5503,20 +5405,30 @@ struct ggml_tensor * ggml_arange( // ggml_flash_attn_ext -struct ggml_tensor * ggml_flash_attn_ext( +static struct ggml_tensor * ggml_flash_attn_ext_impl( struct ggml_context * ctx, struct ggml_tensor * q, struct ggml_tensor * k, struct ggml_tensor * v, struct ggml_tensor * mask, + struct ggml_tensor * kv_rows, float scale, float max_bias, float logit_softcap) { GGML_ASSERT(ggml_can_mul_mat(k, q)); // TODO: check if vT can be multiplied by (k*qT) - GGML_ASSERT(q->ne[3] == k->ne[3]); - GGML_ASSERT(q->ne[3] == v->ne[3]); + if (kv_rows) { + // k and v are shared by all q slices, kv_rows picks the rows of each slice + GGML_ASSERT(mask && mask->ne[0] == kv_rows->ne[0]); + GGML_ASSERT(kv_rows->type == GGML_TYPE_I32 && ggml_is_contiguous(kv_rows)); + GGML_ASSERT(kv_rows->ne[1] == q->ne[3] && kv_rows->ne[2] == 1 && kv_rows->ne[3] == 1); + GGML_ASSERT(k->ne[3] == 1); + GGML_ASSERT(v->ne[3] == 1); + } else { + GGML_ASSERT(q->ne[3] == k->ne[3]); + GGML_ASSERT(q->ne[3] == v->ne[3]); + } if (mask) { GGML_ASSERT(mask->type == GGML_TYPE_F16); @@ -5543,21 +5455,37 @@ struct ggml_tensor * ggml_flash_attn_ext( result->src[1] = k; result->src[2] = v; result->src[3] = mask; + result->src[5] = kv_rows; return result; } +struct ggml_tensor * ggml_flash_attn_ext( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + float scale, + float max_bias, + float logit_softcap) { + return ggml_flash_attn_ext_impl(ctx, q, k, v, mask, NULL, scale, max_bias, logit_softcap); +} -void ggml_flash_attn_ext_set_prec( - struct ggml_tensor * a, - enum ggml_prec prec) { - GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT); - - const int32_t prec_i32 = (int32_t) prec; - - ggml_set_op_params_i32(a, 3, prec_i32); // scale is on first pos, max_bias on second +struct ggml_tensor * ggml_flash_attn_ext_rows( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + struct ggml_tensor * kv_rows, + float scale, + float max_bias, + float logit_softcap) { + return ggml_flash_attn_ext_impl(ctx, q, k, v, mask, kv_rows, scale, max_bias, logit_softcap); } + enum ggml_prec ggml_flash_attn_ext_get_prec( const struct ggml_tensor * a) { GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT); @@ -6617,6 +6545,48 @@ struct ggml_tensor * ggml_dsv4_hc_post( return result; } +// ggml_top_k_mask + +struct ggml_tensor * ggml_top_k_mask( + struct ggml_context * ctx, + struct ggml_tensor * mask, + struct ggml_tensor * top_k, + struct ggml_tensor * score, + struct ggml_tensor * blocks, + struct ggml_tensor * extra) { + GGML_ASSERT(mask->type == GGML_TYPE_F16 || mask->type == GGML_TYPE_F32); + GGML_ASSERT(mask->nb[0] == ggml_type_size(mask->type)); + GGML_ASSERT(top_k->type == GGML_TYPE_I32); + GGML_ASSERT(top_k->ne[1] % mask->ne[1] == 0 && top_k->ne[2] % mask->ne[2] == 0 && top_k->ne[3] % mask->ne[3] == 0); + + if (score) { + GGML_ASSERT(score->type == GGML_TYPE_F32); + GGML_ASSERT(score->ne[1] == top_k->ne[1] && score->ne[2] == top_k->ne[2] && score->ne[3] == top_k->ne[3]); + } + + if (blocks) { + GGML_ASSERT(blocks->type == GGML_TYPE_I32); + GGML_ASSERT(blocks->ne[2] == 1 && blocks->ne[3] == 1); + GGML_ASSERT(!score || blocks->ne[1] == score->ne[0]); + } + + if (extra) { + GGML_ASSERT(extra->type == GGML_TYPE_I32); + GGML_ASSERT(extra->ne[1] == top_k->ne[1] && extra->ne[2] == top_k->ne[2] && extra->ne[3] == top_k->ne[3]); + } + + struct ggml_tensor * result = ggml_new_tensor_4d(ctx, mask->type, mask->ne[0], top_k->ne[1], top_k->ne[2], top_k->ne[3]); + + result->op = GGML_OP_TOP_K_MASK; + result->src[0] = mask; + result->src[1] = top_k; + result->src[2] = score; + result->src[3] = blocks; + result->src[4] = extra; + + return result; +} + //////////////////////////////////////////////////////////////////////////////// struct ggml_hash_set ggml_hash_set_new(size_t size) { @@ -6725,22 +6695,6 @@ static void ggml_acc_or_set( ggml_build_forward_expand(cgraph, cgraph->grads[isrc]); } -static void ggml_add1_or_set( - struct ggml_context * ctx, - struct ggml_cgraph * cgraph, - size_t isrc, - struct ggml_tensor * tensor) { - struct ggml_tensor * src = cgraph->visited_hash_set.keys[isrc]; - GGML_ASSERT(src); - if (cgraph->grads[isrc]) { - cgraph->grads[isrc] = ggml_add1_impl(ctx, cgraph->grads[isrc], tensor, cgraph->grad_accs[isrc]); - } else { - cgraph->grads[isrc] = ggml_repeat(ctx, tensor, src); - } - ggml_format_name(cgraph->grads[isrc], "grad for %s", src->name); - ggml_build_forward_expand(cgraph, cgraph->grads[isrc]); -} - static void ggml_sub_or_set( struct ggml_context * ctx, struct ggml_cgraph * cgraph, @@ -6795,14 +6749,6 @@ static void ggml_compute_backward( ggml_add_or_set(ctx, cgraph, isrc1, tmp); } } break; - case GGML_OP_ADD1: { - if (src0_needs_grads) { - ggml_add_or_set(ctx, cgraph, isrc0, grad); - } - if (src1_needs_grads) { - ggml_add_or_set(ctx, cgraph, isrc1, ggml_mean(ctx, grad)); // TODO: should probably be sum instead of mean - } - } break; case GGML_OP_ACC: { if (src0_needs_grads) { ggml_add_or_set(ctx, cgraph, isrc0, grad); @@ -6875,7 +6821,7 @@ static void ggml_compute_backward( } break; case GGML_OP_SUM: { if (src0_needs_grads) { - ggml_add1_or_set(ctx, cgraph, isrc0, grad); + ggml_add_or_set(ctx, cgraph, isrc0, ggml_repeat(ctx, grad, src0)); } } break; case GGML_OP_SUM_ROWS: { @@ -6885,7 +6831,7 @@ static void ggml_compute_backward( } break; case GGML_OP_MEAN: { if (src0_needs_grads) { - ggml_add1_or_set(ctx, cgraph, isrc0, ggml_scale_impl(ctx, grad, 1.0f/src0->ne[0], 0.0, false)); + ggml_add_or_set(ctx, cgraph, isrc0, ggml_repeat(ctx, ggml_scale_impl(ctx, grad, 1.0f/src0->ne[0], 0.0, false), src0)); } } break; case GGML_OP_REPEAT: { @@ -7075,18 +7021,9 @@ static void ggml_compute_backward( // noop } } break; - case GGML_OP_DIAG_MASK_INF: { - if (src0_needs_grads) { - /* ggml_diag_mask_inf_impl() shouldn't be here */ - /* ref: https://github.com/ggml-org/llama.cpp/pull/4203#discussion_r1412377992 */ - const int n_past = ((const int32_t *) tensor->op_params)[0]; - ggml_add_or_set(ctx, cgraph, isrc0, ggml_diag_mask_zero_impl(ctx, grad, n_past, false)); - } - } break; - case GGML_OP_DIAG_MASK_ZERO: { + case GGML_OP_TRI: { if (src0_needs_grads) { - const int n_past = ((const int32_t *) tensor->op_params)[0]; - ggml_add_or_set(ctx, cgraph, isrc0, ggml_diag_mask_zero_impl(ctx, grad, n_past, false)); + ggml_add_or_set(ctx, cgraph, isrc0, ggml_tri(ctx, grad, (enum ggml_tri_type) ggml_get_op_params_i32(tensor, 0))); } } break; case GGML_OP_SOFT_MAX: { @@ -7106,7 +7043,6 @@ static void ggml_compute_backward( //const int n_past = ((int32_t *) tensor->op_params)[0]; const int n_dims = ((const int32_t *) tensor->op_params)[1]; const int mode = ((const int32_t *) tensor->op_params)[2]; - //const int n_ctx = ((int32_t *) tensor->op_params)[3]; const int n_ctx_orig = ((const int32_t *) tensor->op_params)[4]; float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow; int sections[4] = {0, 0, 0, 0}; @@ -7124,6 +7060,7 @@ static void ggml_compute_backward( mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow) : ggml_rope_multi_back(ctx, grad, src1, src2, n_dims, sections, mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + ggml_rope_set_truncate(rope_back, ggml_rope_get_truncate(tensor)); ggml_add_or_set(ctx, cgraph, isrc0, rope_back); } GGML_ASSERT((!src2 || !src2_needs_grads) && "gradients for freq factors not implemented"); @@ -8098,6 +8035,7 @@ size_t ggml_quantize_chunk( case GGML_TYPE_Q8_0: result = quantize_q8_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_MXFP4: result = quantize_mxfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_NVFP4: result = quantize_nvfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_F8_E4M3: result = quantize_f8_e4m3(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q2_K: result = quantize_q2_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q3_K: result = quantize_q3_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q4_K: result = quantize_q4_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 75ceade7d..c80b68218 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -1463,6 +1463,7 @@ struct gguf_writer_base { // we bet on devirtualization virtual void write(int8_t val) = 0; virtual void write(const std::vector<int8_t> & val) = 0; + virtual void write(const void * data, size_t size) = 0; virtual void write_tensor_data(const struct gguf_tensor_info & info, size_t offset_data, size_t alignment) = 0; template <typename T> @@ -1579,21 +1580,31 @@ struct gguf_writer_buf final : public gguf_writer_base { written_bytes += val.size(); } + void write(const void * data, const size_t size) override { + buf.insert(buf.end(), (const int8_t *) data, (const int8_t *) data + size); + written_bytes += size; + } + void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override { GGML_ASSERT(buf.size() - offset_data == info.offset); GGML_ASSERT(ggml_is_contiguous(&info.t)); - const size_t offset = buf.size(); const size_t nbytes = ggml_nbytes(&info.t); - buf.resize(offset + nbytes); - if (info.t.buffer) { - ggml_backend_tensor_get(&info.t, buf.data() + offset, 0, nbytes); + if (info.t.buffer && ggml_backend_buffer_is_host(info.t.buffer)) { + GGML_ASSERT(info.t.data != NULL && "tensor not allocated"); + write(info.t.data, nbytes); } else { - GGML_ASSERT(info.t.data); - memcpy(buf.data() + offset, info.t.data, nbytes); + const size_t offset = buf.size(); + buf.resize(offset + nbytes); + if (info.t.buffer) { + ggml_backend_tensor_get(&info.t, buf.data() + offset, 0, nbytes); + } else { + GGML_ASSERT(info.t.data); + memcpy(buf.data() + offset, info.t.data, nbytes); + } + written_bytes += nbytes; } - written_bytes += nbytes; pad(alignment); } @@ -1624,20 +1635,33 @@ struct gguf_writer_file final : public gguf_writer_base { } } + void write(const void * data, const size_t size) override { + const auto ret = fwrite(data, 1, size, file); + written_bytes += size; + if (ret != size) { + throw std::runtime_error("unexpected fwrite number of bytes written, '" + std::to_string(ret) + "' instead of '" + std::to_string(size) + "'"); + } + } + void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override { GGML_ASSERT(written_bytes - offset_data == info.offset); GGML_ASSERT(ggml_is_contiguous(&info.t)); const size_t nbytes = ggml_nbytes(&info.t); - std::vector<int8_t> buf(nbytes); - if (info.t.buffer) { - ggml_backend_tensor_get(&info.t, buf.data(), 0, nbytes); + if (info.t.buffer && ggml_backend_buffer_is_host(info.t.buffer)) { + GGML_ASSERT(info.t.data != NULL && "tensor not allocated"); + write(info.t.data, nbytes); } else { - GGML_ASSERT(info.t.data); - memcpy(buf.data(), info.t.data, nbytes); + std::vector<int8_t> buf(nbytes); + if (info.t.buffer) { + ggml_backend_tensor_get(&info.t, buf.data(), 0, nbytes); + } else { + GGML_ASSERT(info.t.data); + memcpy(buf.data(), info.t.data, nbytes); + } + write(buf); } - write(buf); pad(alignment); } diff --git a/ggml/tests/test-backend-ops.cpp b/ggml/tests/test-backend-ops.cpp index d8893025e..3abdff5fd 100644 --- a/ggml/tests/test-backend-ops.cpp +++ b/ggml/tests/test-backend-ops.cpp @@ -48,9 +48,33 @@ #ifdef __EMSCRIPTEN__ # define N_THREADS 1 #else -# define N_THREADS std::thread::hardware_concurrency() +static unsigned test_hardware_concurrency() { + const char * env = std::getenv("TEST_ARG_HARDWARE_CONCURRENCY"); + if (env != nullptr) { + const long n = std::strtol(env, nullptr, 10); + if (n > 0) { + return (unsigned) n; + } + } + + return std::thread::hardware_concurrency(); +} + +# define N_THREADS test_hardware_concurrency() #endif +static int test_parallel_workers(int default_workers) { + const char * env = std::getenv("TEST_ARG_PARALLEL_WORKERS"); + if (env != nullptr) { + const long n = std::strtol(env, nullptr, 10); + if (n > 0) { + return (int) n; + } + } + + return default_workers; +} + static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { if (ggml_is_empty(tensor)) { return; @@ -88,7 +112,7 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) { ggml_backend_tensor_set(tensor, data.data(), 0, nels * sizeof(float)); - } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { + } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F8_E4M3 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0); // dummy importance matrix @@ -262,7 +286,7 @@ static std::vector<float> tensor_to_float(const ggml_tensor * t) { const auto * tt = ggml_get_type_traits(t->type); size_t bs = ggml_blck_size(t->type); std::vector<float> vq(ggml_blck_size(t->type)); - bool quantized = ggml_is_quantized(t->type); + bool uses_type_conversion = ggml_is_quantized(t->type) || t->type == GGML_TYPE_F8_E4M3; // access elements by index to avoid gaps in views for (int64_t i3 = 0; i3 < t->ne[3]; i3++) { @@ -284,7 +308,7 @@ static std::vector<float> tensor_to_float(const ggml_tensor * t) { tv.push_back((float)*(int16_t *) &buf[i]); } else if (t->type == GGML_TYPE_I8) { tv.push_back((float)*(int8_t *) &buf[i]); - } else if (quantized) { + } else if (uses_type_conversion) { tt->to_float(&buf[i], vq.data(), bs); tv.insert(tv.end(), vq.begin(), vq.end()); } else { @@ -418,7 +442,11 @@ static std::string var_to_str(ggml_type type) { } static std::string var_to_str(ggml_prec prec) { - return prec == GGML_PREC_F32 ? "f32" : "def"; + switch (prec) { + case GGML_PREC_F32: return "f32"; + case GGML_PREC_BF16: return "bf16"; + default: return "def"; + } } static std::string var_to_str(ggml_op_pool pool) { @@ -465,6 +493,7 @@ static std::string var_to_str(ggml_scale_mode mode) { #define VARS_TO_STR15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VAR_TO_STR(a) + "," + VARS_TO_STR14(b, c, d, e, f, g, h, i, j, k, l, m, n, o) #define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) #define VARS_TO_STR17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) VAR_TO_STR(a) + "," + VARS_TO_STR16(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) +#define VARS_TO_STR18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) VAR_TO_STR(a) + "," + VARS_TO_STR17(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) // accept FLT_MAX as infinity static bool isinf_or_max(float f) { @@ -1194,6 +1223,29 @@ static bool op_filter_entry_matches(std::string_view entry, std::string_view op_ return std::regex_search(op_name.data(), op_name.data() + op_name.size(), re); } +// TODO: [TAG_MUL_MAT_ID_SKIP] see https://github.com/ggml-org/llama.cpp/pull/26631 +static bool backend_lacks_skipped_ids(ggml_backend_t backend, ggml_cgraph * gf) { + const char * name = ggml_backend_reg_name(ggml_backend_dev_backend_reg(ggml_backend_get_device(backend))); + if (strcmp(name, "CUDA") != 0 && strcmp(name, "ROCm") != 0 && strcmp(name, "MUSA") != 0) { + return false; + } + for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { + const ggml_tensor * node = ggml_graph_node(gf, i); + if (node->op != GGML_OP_MUL_MAT_ID && node->op != GGML_OP_ADD_ID) { + continue; + } + const ggml_tensor * ids = node->src[2]; + std::vector<int32_t> row(ids->ne[0]); + for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) { + ggml_backend_tensor_get(ids, row.data(), i1*ids->nb[1], ids->ne[0]*sizeof(int32_t)); + if (std::find(row.begin(), row.end(), -1) != row.end()) { + return true; + } + } + } + return false; +} + struct test_case { virtual ~test_case() {} @@ -1290,6 +1342,18 @@ struct test_case { } virtual bool run_whole_graph() { return false; } + // the output must be the same in each run on the same backend + virtual bool deterministic(ggml_backend_t backend) { + GGML_UNUSED(backend); + return false; + } + virtual bool same_output(ggml_tensor * t1, ggml_tensor * t2) { + std::vector<uint8_t> t1_data(ggml_nbytes(t1)); + std::vector<uint8_t> t2_data(ggml_nbytes(t2)); + ggml_backend_tensor_get(t1, t1_data.data(), 0, ggml_nbytes(t1)); + ggml_backend_tensor_get(t2, t2_data.data(), 0, ggml_nbytes(t2)); + return t1_data == t2_data; + } virtual std::vector<ggml_tensor *> fusion_test_nodes() { return {}; } virtual bool use_weight_context() { return false; } @@ -1475,6 +1539,13 @@ struct test_case { initialize_tensors(ctx_weights.get()); } + // TODO: [TAG_MUL_MAT_ID_SKIP] @ngxson : remove this once CUDA implements mul_mat_id with -1 = skipped + if (backend_lacks_skipped_ids(backend1, gf)) { + test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", false, false, "not supported"); + print_test_result_locked(output_printer, result); + return test_status_t::NOT_SUPPORTED; + } + // compare struct callback_userdata { bool ok; @@ -1558,6 +1629,24 @@ struct test_case { run_whole_graph() ? fused_nodes_to_verify.data() : nullptr, fused_nodes_to_verify.size()); + // compare backend1 with itself: a second copy of the graph must give the same bytes + if (deterministic(backend1)) { + auto callback_same = [](int index, ggml_tensor * t1, ggml_tensor * t2, void * user_data) -> bool { + callback_userdata * ud = (callback_userdata *) user_data; + if (!ud->tc->same_output(t1, t2)) { + printf("[%s] not deterministic ", ggml_op_desc(t1)); + ud->ok = false; + } + return true; + + GGML_UNUSED(index); + }; + + for (int i = 0; i < 4 && ud.ok; i++) { + ggml_backend_compare_graph_backend(backend1, backend1, gf, callback_same, &ud, &out, 1); + } + } + // Create test result bool test_passed = ud.ok && cmp_ok; std::string error_msg = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); @@ -2069,6 +2158,15 @@ struct test_unary : public test_case { return VARS_TO_STR3(type, ne_a, v); } + double max_nmse_err(ggml_backend_t backend) override { + // Metal computes f32 gelu_quick in f32, so an fp16 table in the CPU reference (NMSE about 5e-8) fails this + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); + if (op == GGML_UNARY_OP_GELU_QUICK && type == GGML_TYPE_F32 && strcmp(ggml_backend_reg_name(reg), "MTL") == 0) { + return 1e-10; + } + return test_case::max_nmse_err(backend); + } + test_unary(ggml_unary_op op, ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne_a = {128, 2, 2, 2}, @@ -2157,6 +2255,15 @@ struct test_glu : public test_case { return VARS_TO_STR4(type, ne_a, v, swapped); } + double max_nmse_err(ggml_backend_t backend) override { + // Metal computes f32 geglu_quick in f32, so an fp16 table in the CPU reference (NMSE about 5e-8) fails this + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); + if (op == GGML_GLU_OP_GEGLU_QUICK && type == GGML_TYPE_F32 && strcmp(ggml_backend_reg_name(reg), "MTL") == 0) { + return 1e-10; + } + return test_case::max_nmse_err(backend); + } + test_glu(ggml_glu_op op, ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne_a = {128, 2, 2, 2}, @@ -2202,6 +2309,15 @@ struct test_glu_split : public test_case { return VARS_TO_STR3(type, ne_a, v) + ",split"; } + double max_nmse_err(ggml_backend_t backend) override { + // Metal computes f32 geglu_quick in f32, so an fp16 table in the CPU reference (NMSE about 5e-8) fails this + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); + if (op == GGML_GLU_OP_GEGLU_QUICK && type == GGML_TYPE_F32 && strcmp(ggml_backend_reg_name(reg), "MTL") == 0) { + return 1e-10; + } + return test_case::max_nmse_err(backend); + } + test_glu_split(ggml_glu_op op, ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne_a = {128, 2, 2, 2}, @@ -2308,6 +2424,12 @@ struct test_swiglu_oai : public test_case { init_tensor_uniform(t, -150.f, 150.f); } } + + double max_nmse_err() override { + // F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the + // natural roundoff drift from the exp()/clamp() chain computed in F16. + return type == GGML_TYPE_F16 ? 5e-6 : 1e-7; + } }; struct test_swiglu_clamp : public test_case { @@ -2365,6 +2487,12 @@ struct test_swiglu_clamp : public test_case { init_tensor_uniform(t, -150.f, 150.f); } } + + double max_nmse_err() override { + // F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the + // natural roundoff drift from the sigmoid()/clamp() chain computed in F16. + return type == GGML_TYPE_F16 ? 5e-6 : 1e-7; + } }; // GGML_OP_GET_ROWS @@ -2441,6 +2569,107 @@ struct test_get_rows : public test_case { } }; +// GGML_OP_GET_ROWS on a weight also used by GGML_OP_MUL_MAT (tied embeddings) +// that the preceding mul_mat may have put in the SYCL reorder layout. The +// mul_mat runs first, so a backend that changes the weight layout in place +// must still return the right rows from get_rows. +struct test_get_rows_after_mul_mat : public test_case { + const ggml_type type; + const int n; // cols + const int m; // rows + const int r; // rows to get + + std::string vars() override { + return VARS_TO_STR4(type, n, m, r); + } + + test_get_rows_after_mul_mat(ggml_type type = GGML_TYPE_Q8_0, int n = 256, int m = 16, int r = 4) + : type(type), n(n), m(m), r(r) {} + + double max_nmse_err() override { + return 5e-4; // same as test_mul_mat + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * w = ggml_new_tensor_2d(ctx, type, n, m); + ggml_set_name(w, "w"); + + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n, 1); + ggml_set_name(x, "x"); + + ggml_tensor * rows = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, r); + ggml_set_name(rows, "rows"); + + ggml_tensor * mm = ggml_mul_mat(ctx, w, x); + ggml_set_name(mm, "mm"); + + ggml_tensor * gr = ggml_get_rows(ctx, w, rows); + ggml_set_name(gr, "gr"); + + // repeat(mm) is the first source, so mul_mat is computed before get_rows + ggml_tensor * out = ggml_add(ctx, ggml_repeat(ctx, mm, gr), gr); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + std::vector<int> data(ggml_nelements(t)); + for (size_t i = 0; i < data.size(); i++) { + data[i] = rand() % m; + } + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(int)); + } else { + init_tensor_uniform(t); + } + } + } +}; + +// The persistent SYCL weight reorder installs on the first small-batch mul_mat; a +// later >8-column mul_mat on the same weight goes through the fused-dequant GEMM / +// conversion routes, which must decode the reordered layout (regression test for the +// #29107 crash: the plain fused-dequant GEMM decoded standard layout unconditionally). +struct test_mul_mat_reorder_then_big_batch : public test_case { + const ggml_type type; + const int n; // cols of the big batch + const int m; // rows + + std::string vars() override { + return VARS_TO_STR3(type, n, m); + } + + test_mul_mat_reorder_then_big_batch(ggml_type type = GGML_TYPE_IQ3_S, int n = 64, int m = 64) + : type(type), n(n), m(m) {} + + double max_nmse_err() override { + return 5e-4; // same as test_mul_mat + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * w = ggml_new_tensor_2d(ctx, type, 1*256, m); + ggml_set_name(w, "w"); + + ggml_tensor * x1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1*256, 1); + ggml_set_name(x1, "x1"); + + ggml_tensor * x2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1*256, n); + ggml_set_name(x2, "x2"); + + ggml_tensor * mm1 = ggml_mul_mat(ctx, w, x1); + ggml_set_name(mm1, "mm1"); + + ggml_tensor * mm2 = ggml_mul_mat(ctx, w, x2); + ggml_set_name(mm2, "mm2"); + + // mm1 is the first source of the final ADD, so the reorder-installing + // small-batch mul_mat is computed before the big-batch one + return ggml_add(ctx, ggml_repeat(ctx, mm1, mm2), mm2); + } +}; + // GGML_OP_GET_ROWS_BACK struct test_get_rows_back : public test_case { const ggml_type type; @@ -2628,11 +2857,13 @@ struct test_rope_set_rows : public test_case { const ggml_type type_idx; const std::array<int64_t, 4> ne_a; int mode; + const int n_dims; + const bool yarn; + const bool truncate; const int n_ctx{512}; - const int n_dims{128}; std::string vars() override { - return VARS_TO_STR4(type, type_idx, ne_a, mode); + return VARS_TO_STR7(type, type_idx, ne_a, mode, n_dims, yarn, truncate); } std::string op_desc(ggml_tensor * t) override { @@ -2645,8 +2876,9 @@ struct test_rope_set_rows : public test_case { test_rope_set_rows(ggml_type type, ggml_type type_idx, std::array<int64_t, 4> ne_a, - int mode) - : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode) {} + int mode, + int n_dims = 128, bool yarn = false, bool truncate = true) + : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims), yarn(yarn), truncate(truncate) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne_a[0], ne_a[1], ne_a[2], 1); @@ -2663,9 +2895,12 @@ struct test_rope_set_rows : public test_case { } ggml_set_name(pos, "pos"); - float fs = 1.4245f; - float ef = 0.7465f; - float af = 1.4245f; + const float fs = yarn ? 1.0f/256 : 1.4245f; + const float ef = yarn ? 1.0f : 0.7465f; + const float af = yarn ? 1.0f : 1.4245f; + const int n_ctx_orig = yarn ? 4096 : 0; + const float freq_base = yarn ? 1000000.0f : 10000.0f; + const float beta_fast = yarn ? 32.0f : 1.0f; ggml_tensor * freq = nullptr; ggml_tensor * rope = nullptr; @@ -2673,15 +2908,18 @@ struct test_rope_set_rows : public test_case { if (is_vision) { GGML_ASSERT(n_dims/4 > 0); int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate - rope = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + rope = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { GGML_ASSERT(n_dims/3 > 0); int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; - rope = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + rope = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } + } else if (yarn) { + rope = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { - rope = ggml_rope(ctx, a, pos, ne_a[0], mode); + rope = ggml_rope(ctx, a, pos, n_dims, mode); } + ggml_rope_set_truncate(rope, truncate); ggml_tensor * view = ggml_view_2d(ctx, rope, ne_a[0] * ne_a[1], ne_a[2], rope->nb[2], 0); @@ -2709,7 +2947,7 @@ struct test_rope_set_rows : public test_case { const int num_pos_ids = (mode & GGML_ROPE_TYPE_MROPE) ? ne_a[2] * 4 : ne_a[2]; std::vector<int> data(num_pos_ids); for (int i = 0; i < num_pos_ids; i++) { - data[i] = rand() % n_ctx; + data[i] = yarn ? 4096 + i * 67 : rand() % n_ctx; } ggml_backend_tensor_set(t, data.data(), 0, num_pos_ids * sizeof(int)); } else { @@ -2735,6 +2973,8 @@ struct test_rms_norm_mul_rope : public test_case { const bool broadcast; // multiply by a 1D [ne0] weight, as model norm weights are const ggml_type set_rows_type; int mode; + const bool yarn; + const bool truncate; std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -2744,14 +2984,15 @@ struct test_rms_norm_mul_rope : public test_case { bool run_whole_graph() override { return true; } std::string vars() override { - return VARS_TO_STR9(ne, eps, multi_add, mul, rope, set_rows, broadcast, mode, set_rows_type); + return VARS_TO_STR11(ne, eps, multi_add, mul, rope, set_rows, broadcast, mode, set_rows_type, yarn, truncate); } test_rms_norm_mul_rope(std::array<int64_t, 4> ne, float eps = 1e-6f, bool multi_add = false, bool set_rows = false, bool broadcast = false, int mode = GGML_ROPE_TYPE_NORMAL, - bool mul = true, bool rope = true, ggml_type set_rows_type = GGML_TYPE_F16) + bool mul = true, bool rope = true, ggml_type set_rows_type = GGML_TYPE_F16, + bool yarn = false, bool truncate = true) : ne(ne), eps(eps), multi_add(multi_add), mul(mul), rope(rope), set_rows(set_rows), broadcast(broadcast), - set_rows_type(set_rows_type), mode(mode) {} + set_rows_type(set_rows_type), mode(mode), yarn(yarn), truncate(truncate) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], ne[3]); @@ -2783,14 +3024,21 @@ struct test_rms_norm_mul_rope : public test_case { if (rope) { const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne[2] * (is_mrope ? 4 : 1)); + const int n_ctx_orig = yarn ? 4096 : 0; + const float freq_base = yarn ? 1000000.0f : 10000.0f; + const float freq_scale = yarn ? 1.0f/256 : 1.0f; + const float ext_factor = yarn ? 1.0f : 0.0f; if (is_mrope) { const int n_dims = ne[0]; int sections[4] = { n_dims/3, n_dims/3, n_dims/3, 0 }; - a = ggml_rope_multi(ctx, a, pos, nullptr, n_dims, sections, mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); + a = ggml_rope_multi(ctx, a, pos, nullptr, n_dims, sections, mode, n_ctx_orig, freq_base, freq_scale, ext_factor, 1.0f, 32.0f, 1.0f); + } else if (yarn) { + a = ggml_rope_ext(ctx, a, pos, nullptr, ne[0], mode, n_ctx_orig, freq_base, freq_scale, ext_factor, 1.0f, 32.0f, 1.0f); } else { a = ggml_rope(ctx, a, pos, ne[0], mode); } + ggml_rope_set_truncate(a, truncate); } if (set_rows) { @@ -2815,8 +3063,8 @@ struct test_rms_norm_mul_rope : public test_case { init_set_rows_row_ids(t, ne[2] * 2); } else if (t->type == GGML_TYPE_I32) { std::vector<int32_t> data(ggml_nelements(t)); - for (int32_t & value : data) { - value = rand() % 512; + for (size_t i = 0; i < data.size(); i++) { + data[i] = yarn ? 127 + i * 67 : rand() % 512; } ggml_backend_tensor_set(t, data.data(), 0, ggml_nbytes(t)); } else { @@ -2826,7 +3074,14 @@ struct test_rms_norm_mul_rope : public test_case { } double max_nmse_err() override { - return ne[0] == 8192 ? 5e-6 : test_case::max_nmse_err(); + if (ne[0] == 8192) { + return 5e-6; + } + // large positions amplify the difference between the GPU and CPU trig functions + if (ne[2] > 8192) { + return 1e-5; + } + return test_case::max_nmse_err(); } }; @@ -2882,14 +3137,15 @@ struct test_argmax : public test_case { struct test_count_equal : public test_case { const ggml_type type; const std::array<int64_t, 4> ne; + const bool same_src; std::string vars() override { - return VARS_TO_STR2(type, ne); + return VARS_TO_STR3(type, ne, same_src); } test_count_equal(ggml_type type = GGML_TYPE_F32, - std::array<int64_t, 4> ne = {4, 500, 1, 1}) - : type(type), ne(ne) {} + std::array<int64_t, 4> ne = {4, 500, 1, 1}, bool same_src = false) + : type(type), ne(ne), same_src(same_src) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); @@ -2904,7 +3160,7 @@ struct test_count_equal : public test_case { ggml_tensor * b_argmax = ggml_argmax(ctx, b); ggml_set_name(b_argmax, "b_argmax"); - ggml_tensor * out = ggml_count_equal(ctx, a_argmax, b_argmax); + ggml_tensor * out = ggml_count_equal(ctx, a_argmax, same_src ? a_argmax : b_argmax); ggml_set_name(out, "out"); return out; @@ -3296,6 +3552,7 @@ struct test_bin_bcast : public test_case { bool perm1; // permute src1? bool src_overlap; // src0 and src1 are overlapping views of the same buffer const ggml_type type_b; // type of src1, GGML_TYPE_COUNT -> same as src0 + bool weights; bool run_whole_graph() override { return nf > 1; } @@ -3304,6 +3561,9 @@ struct test_bin_bcast : public test_case { if (type_b != GGML_TYPE_COUNT) { s += "," + VAR_TO_STR(type_b); } + if (weights) { + s += ",weights=1"; + } return s; } @@ -3316,28 +3576,38 @@ struct test_bin_bcast : public test_case { std::array<int, 4> nr = {1, 2, 1, 1}, int nf = 1, bool perm1 = false, bool src_overlap = false, - ggml_type type_b = GGML_TYPE_COUNT) - : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap), type_b(type_b) {} + ggml_type type_b = GGML_TYPE_COUNT, bool weights = false) + : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap), type_b(type_b), weights(weights) {} + + bool use_weight_context() override { + return weights; + } ggml_tensor * build_graph(ggml_context * ctx) override { + return build_graph(ctx, ctx); + } + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { GGML_ASSERT(nf <= 16); ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]); ggml_set_name(a, "a"); const ggml_type tb = type_b == GGML_TYPE_COUNT ? type : type_b; + ggml_context * ctx_b = weights ? ctx_weights : ctx; + GGML_ASSERT(ctx_b); ggml_tensor * b[16]; for (int i = 0; i < nf; ++i) { if (perm1) { const int p[4] = { 1, 2, 0, 3 }; // hardcoded for now - b[i] = ggml_new_tensor_4d(ctx, tb, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]); - b[i] = ggml_permute(ctx, b[i], p[0], p[1], p[2], p[3]); + b[i] = ggml_new_tensor_4d(ctx_b, tb, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]); + b[i] = ggml_permute(ctx_b, b[i], p[0], p[1], p[2], p[3]); } else if (src_overlap) { b[i] = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], 2 * (ne[3] / 3), a->nb[1], a->nb[2], a->nb[3], (ne[3] / 3) * a->nb[3]); } else { - b[i] = ggml_new_tensor(ctx, tb, 4, ne.data()); + b[i] = ggml_new_tensor(ctx_b, tb, 4, ne.data()); } ggml_set_name(b[i], (std::string("b") + std::to_string(i)).c_str()); } @@ -3403,6 +3673,16 @@ struct test_bin_bcast : public test_case { } }; +// set some ids to -1, so that MUL_MAT_ID and ADD_ID skip those slots +// every 4th row skips all of its slots, the other rows skip only some of them +static void mask_skipped_ids(std::vector<int32_t> & ids, int64_t row) { + for (size_t i = 0; i < ids.size(); i++) { + if (row % 4 == 1 || (row + (int64_t) i) % 3 == 0) { + ids[i] = -1; + } + } +} + // GGML_OP_ADD_ID struct test_add_id : public test_case { const ggml_type type_a; @@ -3411,9 +3691,10 @@ struct test_add_id : public test_case { const int64_t n_experts; const int64_t n_experts_used; const int64_t n_token; + const bool skip; // set some ids to a negative value std::string vars() override { - return VARS_TO_STR6(type_a, type_b, n_embd, n_experts, n_experts_used, n_token); + return VARS_TO_STR7(type_a, type_b, n_embd, n_experts, n_experts_used, n_token, skip); } size_t op_size(ggml_tensor * t) override { @@ -3425,9 +3706,10 @@ struct test_add_id : public test_case { int64_t n_embd = 128, int64_t n_experts = 16, int64_t n_experts_used = 8, - int64_t n_token = 10) + int64_t n_token = 10, + bool skip = false) : type_a(type_a), type_b(type_b), n_embd(n_embd), - n_experts(n_experts), n_experts_used(n_experts_used), n_token(n_token) {} + n_experts(n_experts), n_experts_used(n_experts_used), n_token(n_token), skip(skip) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor_3d(ctx, type_a, n_embd, n_experts_used, n_token); @@ -3456,6 +3738,9 @@ struct test_add_id : public test_case { data[i] = i % n_experts; } std::shuffle(data.begin(), data.end(), rng); + if (skip) { + mask_skipped_ids(data, r); + } ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); } } else { @@ -3472,23 +3757,37 @@ struct test_scale : public test_case { float scale; float bias; bool inplace; + int v; // view (1 : non-contiguous a) std::string vars() override { - return VARS_TO_STR5(type, ne, scale, bias, inplace); + return VARS_TO_STR6(type, ne, scale, bias, inplace, v); } test_scale(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne = {10, 10, 10, 10}, float scale = 2.0f, float bias = 0.0f, - bool inplace = false) - : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace) {} + bool inplace = false, + int v = 0) + : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace), v(v) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + auto ne_a = ne; + if (v & 1) { + ne_a[0] *= 3; + ne_a[1] *= 2; + ne_a[2] *= 5; + ne_a[3] *= 4; + } + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); ggml_set_param(a); ggml_set_name(a, "a"); + if (v & 1) { + a = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], ne[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + } + ggml_tensor * out; if (inplace) { out = ggml_scale_bias_inplace(ctx, a, scale, bias); @@ -4159,6 +4458,93 @@ struct test_unary_mul : public test_case { } }; +// GGML_OP_ADD + GGML_OP_UNARY(SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL with the ADD's bias and +// the MUL's scale broadcast over dim 0: the delta-net alpha gate, softplus(alpha + dt) * a_coeff. +struct test_add_unary_mul : public test_case { + const ggml_unary_op op; + const ggml_type type; + const std::array<int64_t, 4> ne; + const bool swap; // unary result is the second MUL operand + const std::string layout; // bias/scale layout, see build_graph() + const std::string tail; // extra consumer past the MUL, see build_graph() + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "ADD_" + std::string(ggml_unary_op_name(op)) + "_MUL"; + } + bool run_whole_graph() override { return true; } + + double max_nmse_err() override { + switch (type) { + // f16 never fuses (the kernel is f32-only), so this bound is the unfused + // chain's own f16 rounding drift, as in test_unary_mul + case GGML_TYPE_F16: return 5e-5; + // gelu never fuses either, and the backends' exp form drifts from the CPU's tanhf + default: return op == GGML_UNARY_OP_GELU ? 5e-7 : 1e-7; + } + } + + std::string vars() override { + return VARS_TO_STR5(type, ne, swap, layout, tail); + } + + test_add_unary_mul(ggml_unary_op op, + ggml_type type = GGML_TYPE_F32, + std::array<int64_t, 4> ne = {32, 7, 1, 1}, + bool swap = false, + std::string layout = "bcast", + std::string tail = "") + : op(op), type(type), ne(ne), swap(swap), layout(std::move(layout)), tail(std::move(tail)) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + std::array<int64_t, 4> ne_v = { ne[0], 1, 1, 1 }; + if (layout == "bcast") { + // one ne0 row each, broadcast over the outer dims, which is the alpha-gate form + } else if (layout == "same_shape") { + // no broadcast at all; fuses only while the activation is a single row + ne_v = ne; + } else if (layout == "rep_ne0") { + // repeat on dim 0, which bias[col] cannot address, so this must not fuse + ne_v[0] = ne[0] / 4; + } else { + GGML_ABORT("unknown layout %s", layout.c_str()); + } + + ggml_tensor * bias = ggml_new_tensor(ctx, type, 4, ne_v.data()); + ggml_set_name(bias, "bias"); + + ggml_tensor * scale = ggml_new_tensor(ctx, type, 4, ne_v.data()); + ggml_set_name(scale, "scale"); + + ggml_tensor * s = ggml_add(ctx, a, bias); + ggml_set_name(s, "add"); + + ggml_tensor * u = ggml_unary(ctx, s, op); + ggml_set_name(u, "unary"); + + // a broadcasting operand can only be the second one, so swap needs same-shape operands + ggml_tensor * out = swap ? ggml_mul(ctx, scale, u) : ggml_mul(ctx, u, scale); + + if (tail == "reuse") { + // a second read of the add result must block the fusion + ggml_set_name(out, "mul"); + out = ggml_add(ctx, out, s); + } else if (tail == "consumer") { + // fusion still applies; catches a dispatcher that skips one node too many + ggml_set_name(out, "mul"); + out = ggml_add(ctx, out, scale); + } else if (!tail.empty()) { + GGML_ABORT("unknown tail %s", tail.c_str()); + } + ggml_set_name(out, "out"); + + return out; + } +}; + // SNAKE activation fusion: y = x + sin(a*x)^2 * inv_b // CUDA backend matches the naive 5-op chain (mul, sin, sqr, mul, add) // and dispatches a single fused kernel. @@ -4674,55 +5060,153 @@ struct test_ssm_scan_rollback : public test_case { } }; -// GGML_OP_RWKV_WKV6 -struct test_rwkv_wkv6 : public test_case { +// GGML_OP_SSM_SCAN + GGML_OP_CPY (recurrent cache fusion) +struct test_ssm_scan_cache_fusion : public test_case { const ggml_type type; - const int64_t head_count; - const int64_t head_size; + const int64_t d_state; + const int64_t head_dim; + const int64_t n_head; + const int64_t n_group; const int64_t n_seq_tokens; const int64_t n_seqs; + const int64_t K; // snapshot slot count (1 = final state only) + + ggml_tensor * cpy_node = nullptr; std::string vars() override { - return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); + return VARS_TO_STR8(type, d_state, head_dim, n_head, n_group, n_seq_tokens, n_seqs, K); } - test_rwkv_wkv6(ggml_type type = GGML_TYPE_F32, - int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) - : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} + test_ssm_scan_cache_fusion(ggml_type type = GGML_TYPE_F32, + int64_t d_state = 128, int64_t head_dim = 64, int64_t n_head = 16, int64_t n_group = 2, + int64_t n_seq_tokens = 4, int64_t n_seqs = 1, int64_t K = 4) + : type(type), d_state(d_state), head_dim(head_dim), n_head(n_head), n_group(n_group), + n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), K(K) {} ggml_tensor * build_graph(ggml_context * ctx) override { - const int64_t n_tokens = n_seq_tokens * n_seqs; - ggml_tensor * r = ggml_new_tensor(ctx, type, 3, std::vector<int64_t>{ head_size, head_count, n_tokens }.data()); - ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector<int64_t>{ head_size, head_count, n_tokens }.data()); - ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector<int64_t>{ head_size, head_count, n_tokens }.data()); - ggml_tensor * tf = ggml_new_tensor(ctx, type, 2, std::vector<int64_t>{ head_size, head_count }.data()); - ggml_tensor * td = ggml_new_tensor(ctx, type, 3, std::vector<int64_t>{ head_size, head_count, n_tokens }.data()); - ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector<int64_t>{ head_size * head_size * head_count, n_seqs }.data()); - ggml_tensor * out = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, s); - return out; - } -}; - -// GGML_OP_GATED_DELTA_NET -struct test_gated_delta_net : public test_case { - const ggml_type type; - - const int64_t head_count; - const int64_t head_size; - const int64_t n_seq_tokens; - const int64_t n_seqs; - const int v_repeat; - const bool permuted; - const bool kda; - const int64_t K; // snapshot slot count: 1 = final-only, >1 = last K states + const int64_t D = d_state * head_dim * n_head; + const int64_t n_written = std::min<int64_t>(n_seq_tokens, K); - std::string vars() override { - return VARS_TO_STR9(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K); - } + // more cache rows per slot than seqs and a non-zero first row, so a wrong slot stride or offset shows up + const int64_t mem_size = n_seqs + 2; + const int64_t kv_head = 1; - test_gated_delta_net(ggml_type type = GGML_TYPE_F32, - int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1, + ggml_tensor * s = ggml_new_tensor_4d(ctx, type, d_state, head_dim, n_head, n_seqs); + ggml_tensor * x = ggml_new_tensor_4d(ctx, type, head_dim, n_head, n_seq_tokens, n_seqs); + ggml_tensor * dt = ggml_new_tensor_3d(ctx, type, n_head, n_seq_tokens, n_seqs); + ggml_tensor * A = ggml_new_tensor_2d(ctx, type, 1, n_head); + ggml_tensor * B = ggml_new_tensor_4d(ctx, type, d_state, n_group, n_seq_tokens, n_seqs); + ggml_tensor * C = ggml_new_tensor_4d(ctx, type, d_state, n_group, n_seq_tokens, n_seqs); + ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs); + ggml_set_name(A, "A"); + ggml_set_name(ids, "ids"); + + ggml_tensor * out = ggml_ssm_scan(ctx, s, x, dt, A, B, C, ids, K); + ggml_set_name(out, "ssm_out"); + + // snapshot tail view [D, n_seqs, n_written] + ggml_tensor * src = ggml_view_3d(ctx, out, + D, n_seqs, n_written, + ggml_row_size(out->type, D), + ggml_row_size(out->type, D * n_seqs), + ggml_row_size(out->type, ggml_nelements(x))); + + // recurrent cache view [D, n_seqs, n_written] + ggml_tensor * cache = ggml_new_tensor_2d(ctx, type, D, mem_size * n_written); + ggml_set_name(cache, "cache"); + ggml_tensor * dst = ggml_view_3d(ctx, cache, + D, n_seqs, n_written, + cache->nb[1], + mem_size * cache->nb[1], + kv_head * cache->nb[1]); + + ggml_tensor * cpy = ggml_cpy(ctx, src, dst); + ggml_set_name(cpy, "ssm_cache_cpy"); + cpy_node = cpy; + + // read the cpy output so that neither the scan nor the cpy is the graph output (cont, since the cache view is strided) + ggml_tensor * res = ggml_sum(ctx, ggml_cont(ctx, cpy)); + return res; + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "SSM_SCAN_CACHE_FUSION"; + } + + bool run_whole_graph() override { return true; } + std::vector<ggml_tensor *> fusion_test_nodes() override { return { cpy_node }; } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_is_view_op(t->op)) { continue; } + if (strcmp(t->name, "ids") == 0) { + std::vector<int32_t> data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i; + } + ggml_backend_tensor_set(t, data.data(), 0, t->ne[0] * sizeof(int32_t)); + } else if (strcmp(t->name, "A") == 0) { + init_tensor_uniform(t, -1.0f, -0.5f); + } else if (strcmp(t->name, "cache") == 0) { + init_tensor_uniform(t, 0.0f, 0.0f); + } else { + init_tensor_uniform(t); + } + } + } +}; + +// GGML_OP_RWKV_WKV6 +struct test_rwkv_wkv6 : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + + std::string vars() override { + return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); + } + + test_rwkv_wkv6(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t n_tokens = n_seq_tokens * n_seqs; + ggml_tensor * r = ggml_new_tensor(ctx, type, 3, std::vector<int64_t>{ head_size, head_count, n_tokens }.data()); + ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector<int64_t>{ head_size, head_count, n_tokens }.data()); + ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector<int64_t>{ head_size, head_count, n_tokens }.data()); + ggml_tensor * tf = ggml_new_tensor(ctx, type, 2, std::vector<int64_t>{ head_size, head_count }.data()); + ggml_tensor * td = ggml_new_tensor(ctx, type, 3, std::vector<int64_t>{ head_size, head_count, n_tokens }.data()); + ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector<int64_t>{ head_size * head_size * head_count, n_seqs }.data()); + ggml_tensor * out = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, s); + return out; + } +}; + +// GGML_OP_GATED_DELTA_NET +struct test_gated_delta_net : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + const int v_repeat; + const bool permuted; + const bool kda; + const int64_t K; // snapshot slot count: 1 = final-only, >1 = last K states + + std::string vars() override { + return VARS_TO_STR9(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K); + } + + test_gated_delta_net(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1, int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1) : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), v_repeat(v_repeat), permuted(permuted), kda(kda), K(K) {} @@ -4989,9 +5473,15 @@ struct test_mul_mat : public test_case { const bool src_overlap; // a and b are overlapping views of the same tensor const int64_t m_v; // rows of a in memory, the batches of a are strided for m_v > m, no view for m_v == 0 const int64_t pad; // bytes after the m_v rows of each batch of a, so nb[2] of a is not a multiple of nb[1] + const size_t offset_a; // byte offset of the a view when k_v > k or m_v > m + const bool weights; // a in a weights buffer (CUDA with PDL reads weights early) std::string vars() override { - return VARS_TO_STR13(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad); + std::string s = VARS_TO_STR14(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad, offset_a); + if (weights) { + s += ",weights=1"; + } + return s; } double max_nmse_err() override { @@ -5005,6 +5495,9 @@ struct test_mul_mat : public test_case { backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { return 2e-2; } + if (type_a == GGML_TYPE_F8_E4M3 && backend_has_feature(backend, "NATIVE_FP8")) { + return 5e-3; + } return max_nmse_err(); } @@ -5022,10 +5515,21 @@ struct test_mul_mat : public test_case { std::array<int64_t, 2> bs = {10, 10}, std::array<int64_t, 2> nr = {2, 2}, std::array<int64_t, 4> per = {0, 1, 2, 3}, - int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0) - : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad) {} + int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0, size_t offset_a = 0, bool weights = false) + : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad), offset_a(offset_a), weights(weights) {} + + bool use_weight_context() override { + return weights; + } ggml_tensor * build_graph(ggml_context * ctx) override { + return build_graph(ctx, ctx); + } + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_context * ctx_a = weights ? ctx_weights : ctx; + GGML_ASSERT(ctx_a); + // C^T = A * B^T: (k, m) * (k, n) => (m, n) ggml_tensor * a; ggml_tensor * b; @@ -5042,7 +5546,7 @@ struct test_mul_mat : public test_case { const int64_t ne_a[4] = {k, m, bs[0], bs[1]}; const int64_t ne_b[4] = {k, n, bs[0]*nr[0], bs[1]*nr[1]}; - a = ggml_new_tensor_4d(ctx, type_a, ne_a[per[0]], ne_a[per[1]], ne_a[per[2]], ne_a[per[3]]); + a = ggml_new_tensor_4d(ctx_a, type_a, ne_a[per[0]], ne_a[per[1]], ne_a[per[2]], ne_a[per[3]]); b = ggml_new_tensor_4d(ctx, type_b, ne_b[per[0]], ne_b[per[1]], ne_b[per[2]], ne_b[per[3]]); if (!ggml_is_quantized(type_a)) { if (bs[1] == 1 && nr[1] == 1) { @@ -5061,6 +5565,7 @@ struct test_mul_mat : public test_case { GGML_ASSERT(type_a == type_b); GGML_ASSERT(k_v == 0); GGML_ASSERT(m_v == 0); + GGML_ASSERT(ctx_a == ctx); // not handled: b is a view of the same tensor // a and b are interleaved views of the same tensor: (e.g. fused QKV in MiniMax-01) ggml_tensor * base = ggml_new_tensor_4d(ctx, type_a, 2*k, std::max(m, n), bs[0]*nr[0], bs[1]*nr[1]); @@ -5073,7 +5578,7 @@ struct test_mul_mat : public test_case { } else { const int64_t k_physical = k_v == 0 ? k : k_v; const int64_t m_physical = m_v == 0 ? m : m_v + (pad != 0); - a = ggml_new_tensor_4d(ctx, type_a, k_physical, m_physical, bs[0], bs[1]); + a = ggml_new_tensor_4d(ctx_a, type_a, k_physical, m_physical, bs[0], bs[1]); b = ggml_new_tensor_4d(ctx, type_b, k_physical, n, bs[0]*nr[0], bs[1]*nr[1]); if (!ggml_is_quantized(type_a)) { @@ -5085,13 +5590,15 @@ struct test_mul_mat : public test_case { if (k_v != 0) { GGML_ASSERT(k_v > k); - a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], 0); + GGML_ASSERT(offset_a <= ggml_row_size(type_a, k_v) - ggml_row_size(type_a, k)); + a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], offset_a); b = ggml_view_4d(ctx, b, k, n, bs[0]*nr[0], bs[1]*nr[1], b->nb[1], b->nb[2], b->nb[3], 0); } if (m_v != 0) { GGML_ASSERT(m_v > m); GGML_ASSERT(pad < (int64_t) a->nb[1]); - a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], m_v*a->nb[1] + pad, a->nb[3], 0); + GGML_ASSERT(k_v != 0 || offset_a <= (m_v - m)*a->nb[1]); + a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], m_v*a->nb[1] + pad, a->nb[3], k_v == 0 ? offset_a : 0); } ggml_set_name(a, "a"); ggml_set_name(b, "b"); @@ -5126,8 +5633,8 @@ struct test_mul_mat_hadamard : public test_mul_mat { GGML_ASSERT(type_a == GGML_TYPE_F32); } - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); // Find the mul_mat op in the graph and set the hint for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->op == GGML_OP_MUL_MAT) { @@ -5176,8 +5683,8 @@ struct test_mul_mat_w4a8 : public test_mul_mat { std::array<int64_t, 2> bs = {1, 1}, std::array<int64_t, 2> nr = {1, 1}) : test_mul_mat(type_a, type_b, m, n, k, bs, nr) {} - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->op == GGML_OP_MUL_MAT) { ggml_prec_set_src(t, GGML_PREC_Q8, 1); @@ -5191,6 +5698,91 @@ struct test_mul_mat_w4a8 : public test_mul_mat { } }; +// src1 GGML_PREC_BF16: the implementation may keep src1 in F32 or BF16, but must not convert it down to F16, +// which cannot hold the src1 values that the BF16 range allows +struct test_mul_mat_prec_src1_bf16 : public test_mul_mat { + test_mul_mat_prec_src1_bf16(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256, + std::array<int64_t, 2> bs = {1, 1}, + std::array<int64_t, 2> nr = {1, 1}) + : test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {} + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op == GGML_OP_MUL_MAT) { + ggml_prec_set_src(t, GGML_PREC_BF16, 1); + } + } + return out; + } + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "b") == 0) { + init_tensor_uniform(t, -1e6f, 1e6f); + } else { + init_tensor_uniform(t); + } + } + } + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_PREC_SRC1_BF16"; + } +}; + +// src1 GGML_PREC_F32: the implementation can not convert src1 or the F32 src0 tiles down to F16 or TF32 +struct test_mul_mat_prec_src1_f32 : public test_mul_mat { + test_mul_mat_prec_src1_f32(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256, + std::array<int64_t, 2> bs = {1, 1}, + std::array<int64_t, 2> nr = {1, 1}) + : test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {} + double max_nmse_err() override { + return 1e-10; + } + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op == GGML_OP_MUL_MAT) { + ggml_prec_set_src(t, GGML_PREC_F32, 1); + } + } + return out; + } + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "b") == 0) { + init_tensor_uniform(t, -1e6f, 1e6f); + } else { + init_tensor_uniform(t); + } + } + } + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_PREC_SRC1_F32"; + } +}; + +// src1 GGML_PREC_BF16 with src0 values outside the F16 range: the F32 src0 tiles must not be downcast to F16 +struct test_mul_mat_prec_src1_bf16_src0 : public test_mul_mat_prec_src1_bf16 { + test_mul_mat_prec_src1_bf16_src0(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256, + std::array<int64_t, 2> bs = {1, 1}, + std::array<int64_t, 2> nr = {1, 1}) + : test_mul_mat_prec_src1_bf16(type_a, m, n, k, bs, nr) {} + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "a") == 0) { + init_tensor_uniform(t, -1e6f, 1e6f); + } else { + init_tensor_uniform(t); + } + } + } + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_PREC_SRC1_BF16_SRC0"; + } +}; + // FP4 native W4A4 path (default precision) struct test_mul_mat_w4a4 : public test_mul_mat { test_mul_mat_w4a4(ggml_type type_a = GGML_TYPE_NVFP4, ggml_type type_b = GGML_TYPE_F32, @@ -5204,7 +5796,8 @@ struct test_mul_mat_w4a4 : public test_mul_mat { } }; -static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats) { +static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats, bool skip = false) { + std::random_device rd; std::default_random_engine rng(rd()); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { @@ -5217,23 +5810,21 @@ static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats) { data[i] = i % n_mats; } std::shuffle(data.begin(), data.end(), rng); + if (skip) { + mask_skipped_ids(data, r); + } ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); } } } -static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats, float amax = 1.0f) { +static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats, bool skip = false) { for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - continue; - } - if (amax != 1.0f && t->type == GGML_TYPE_F32) { - init_tensor_uniform(t, -amax, amax); - } else { + if (t->type != GGML_TYPE_I32) { init_tensor_uniform(t); } } - init_mul_mat_id_ids(ctx, n_mats); + init_mul_mat_id_ids(ctx, n_mats, skip); } // GGML_OP_MUL_MAT_ID @@ -5246,11 +5837,12 @@ struct test_mul_mat_id : public test_case { const int64_t m; const int64_t n; const int64_t k; - const float amax; // magnitude of src1 const int64_t m_v; // rows of as in memory, the experts of as are strided for m_v > m, no view for m_v == 0 + const bool ids_offset; + const bool skip; // set some ids to a negative value std::string vars() override { - return VARS_TO_STR10(type_a, type_b, n_mats, n_used, b, m, n, k, amax, m_v); + return VARS_TO_STR11(type_a, type_b, n_mats, n_used, b, m, n, k, m_v, ids_offset, skip); } double max_nmse_err() override { @@ -5264,6 +5856,9 @@ struct test_mul_mat_id : public test_case { backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { return 2e-2; } + if (type_a == GGML_TYPE_F8_E4M3 && backend_has_feature(backend, "NATIVE_FP8")) { + return 5e-3; + } return max_nmse_err(); } @@ -5275,9 +5870,9 @@ struct test_mul_mat_id : public test_case { test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, int n_mats = 8, int n_used = 2, bool b = false, int64_t m = 32, int64_t n = 32, int64_t k = 32, - float amax = 1.0f, int64_t m_v = 0) + int64_t m_v = 0, bool ids_offset = false, bool skip = false) : type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b), - m(m), n(n), k(k), amax(amax), m_v(m_v) { + m(m), n(n), k(k), m_v(m_v), ids_offset(ids_offset), skip(skip) { GGML_ASSERT(n_used <= n_mats); GGML_ASSERT(m_v == 0 || m_v > m); } @@ -5290,10 +5885,10 @@ struct test_mul_mat_id : public test_case { } ggml_set_name(as, "as"); - ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n); + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats + int(ids_offset), n); ggml_set_name(ids, "ids"); - if (n_used != n_mats) { - ids = ggml_view_2d(ctx, ids, n_used, n, ids->nb[1], 0); + if (n_used != n_mats || ids_offset) { + ids = ggml_view_2d(ctx, ids, n_used, n, ids->nb[1], ids_offset ? sizeof(int32_t) : 0); ggml_set_name(ids, "view_of_ids"); } @@ -5303,20 +5898,41 @@ struct test_mul_mat_id : public test_case { ggml_tensor * out = ggml_mul_mat_id(ctx, as, b, ids); ggml_set_name(out, "out"); - if (amax > 65504.0f) { - // src1 exceeds F16 range - ggml_prec_set_src(out, GGML_PREC_F32, 1); - } - return out; } void initialize_tensors(ggml_context * ctx) override { - init_mul_mat_id_tensors(ctx, n_mats, amax); + init_mul_mat_id_tensors(ctx, n_mats, skip); } void reinit_perf_iter(ggml_context * ctx) override { - init_mul_mat_id_ids(ctx, n_mats); + init_mul_mat_id_ids(ctx, n_mats, skip); + } +}; + +// MUL_MAT_ID with expert ids repeated within a token's row +struct test_mul_mat_id_dup : public test_mul_mat_id { + using test_mul_mat_id::test_mul_mat_id; + std::string vars() override { return test_mul_mat_id::vars() + ",dup=1"; } + void initialize_tensors(ggml_context * ctx) override { + std::default_random_engine rng(1234); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + if (ggml_is_view_op(t->op)) { continue; } + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector<int32_t> data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = (rng() % 4 == 0) ? (int32_t) (rng() % n_mats) : 0; // repeated ids, mostly expert 0 + } + if (r == ggml_nrows(t) - 1) { + data[ids_offset ? 1 : 0] = n_mats - 1; + } + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } else { + init_tensor_uniform(t); + } + } } }; @@ -5365,9 +5981,10 @@ struct test_mul_mat_id_fusion : public test_case { const int64_t k; const uint32_t o; // number of outputs const bool mul; + const bool skip; // set some ids to a negative value std::string vars() override { - return VARS_TO_STR10(type_a, type_b, n_mats, n_used, b, m, n, k, o, mul); + return VARS_TO_STR11(type_a, type_b, n_mats, n_used, b, m, n, k, o, mul, skip); } double max_nmse_err() override { @@ -5381,9 +5998,9 @@ struct test_mul_mat_id_fusion : public test_case { test_mul_mat_id_fusion(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, int n_mats = 8, int n_used = 2, bool b = false, - int64_t m = 32, int64_t n = 32, int64_t k = 32, uint32_t o = 1, bool mul = false) + int64_t m = 32, int64_t n = 32, int64_t k = 32, uint32_t o = 1, bool mul = false, bool skip = false) : type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b), - m(m), n(n), k(k), o(o), mul(mul) { + m(m), n(n), k(k), o(o), mul(mul), skip(skip) { GGML_ASSERT(n_used <= n_mats); } @@ -5423,7 +6040,7 @@ struct test_mul_mat_id_fusion : public test_case { } void initialize_tensors(ggml_context * ctx) override { - init_mul_mat_id_tensors(ctx, n_mats); + init_mul_mat_id_tensors(ctx, n_mats, skip); } bool run_whole_graph() override { return true; } @@ -5710,19 +6327,34 @@ struct test_clamp : public test_case { const std::array<int64_t, 4> ne; float min; float max; + int v; // view (1 : non-contiguous a) std::string vars() override { - return VARS_TO_STR4(type, ne, min, max); + return VARS_TO_STR5(type, ne, min, max, v); } test_clamp(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne = {10, 5, 4, 3}, - float min = -0.5f, float max = 0.5f) - : type(type), ne(ne), min(min), max(max) {} + float min = -0.5f, float max = 0.5f, int v = 0) + : type(type), ne(ne), min(min), max(max), v(v) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); + ggml_tensor * a; + if (v & 1) { + auto ne_a = ne; + ne_a[0] *= 3; + ne_a[1] *= 2; + ne_a[2] *= 5; + ne_a[3] *= 4; + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + + a = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], ne[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + } else { + a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + } ggml_tensor * out = ggml_clamp(ctx, a, min, max); ggml_set_name(out, "out"); @@ -5863,33 +6495,6 @@ struct test_trunc : public test_case { } }; -// GGML_OP_DIAG_MASK_INF -struct test_diag_mask_inf : public test_case { - const ggml_type type; - const std::array<int64_t, 4> ne; - const int n_past; - - std::string vars() override { - return VARS_TO_STR3(type, ne, n_past); - } - - test_diag_mask_inf(ggml_type type = GGML_TYPE_F32, - std::array<int64_t, 4> ne = {10, 10, 3, 2}, - int n_past = 5) - : type(type), ne(ne), n_past(n_past) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_diag_mask_inf(ctx, a, n_past); - ggml_set_name(out, "out"); - - return out; - } -}; - // GGML_OP_SOFT_MAX struct test_soft_max : public test_case { const ggml_type type; @@ -6003,18 +6608,23 @@ struct test_rope : public test_case { bool forward; bool inplace; int n_offs; // offset of the rotated dims window, set via ggml_rope_set_offset() + int n_ctx_orig; + float freq_base; + float beta_fast; + bool truncate; std::string vars() override { // forward can be inferred from the op, does not need to be printed - return VARS_TO_STR12(type, ne_a, n_dims, mode, n_ctx, fs, ef, af, ff, v, inplace, n_offs); + return VARS_TO_STR16(type, ne_a, n_dims, mode, n_ctx, fs, ef, af, ff, v, inplace, n_offs, n_ctx_orig, freq_base, beta_fast, truncate); } test_rope(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne_a = {10, 5, 3, 1}, int n_dims = 10, int mode = GGML_ROPE_TYPE_NORMAL, int n_ctx = 512, float fs = 1.0f, float ef = 0.0f, float af = 0.0f, bool ff = false, int v = 0, bool forward = true, bool inplace = false, - int n_offs = 0) - : type(type), ne_a(ne_a), n_dims(n_dims), mode(mode), n_ctx(n_ctx), fs(fs), ef(ef), af(af), ff(ff), v(v), forward(forward), inplace(inplace), n_offs(n_offs) {} + int n_offs = 0, int n_ctx_orig = 0, float freq_base = 10000.0f, float beta_fast = 1.0f, bool truncate = true) + : type(type), ne_a(ne_a), n_dims(n_dims), mode(mode), n_ctx(n_ctx), fs(fs), ef(ef), af(af), ff(ff), v(v), forward(forward), inplace(inplace), n_offs(n_offs), + n_ctx_orig(n_ctx_orig), freq_base(freq_base), beta_fast(beta_fast), truncate(truncate) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a; @@ -6030,7 +6640,7 @@ struct test_rope : public test_case { a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); ggml_set_name(a, "view_of_a"); } else if (v == 2) { - // second-half slice along dim 0 (mimics build_rope_2d in clip.cpp). + // second-half slice along dim 0 (mimics build_rope_2d in tools/mtmd/mtmd-graph.cpp). // The non-zero view offset (ne_a[0] * elem_size) often produces a // non-aligned buffer offset, which exercises backends' alignment paths. auto ne = ne_a; ne[0] *= 2; @@ -6078,37 +6688,38 @@ struct test_rope : public test_case { int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate if (forward) { if (inplace) { - out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims/2, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { - out = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } else { - out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims/2, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } else { GGML_ASSERT(n_dims/3 > 0); int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; if (forward) { if (inplace) { - out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { - out = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } else { - out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } } else { if (forward) { if (inplace) { - out = ggml_rope_ext_inplace(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_ext_inplace(ctx, a, pos, freq, n_dims, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { - out = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } else { - out = ggml_rope_ext_back(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_ext_back(ctx, a, pos, freq, n_dims, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } + ggml_rope_set_truncate(out, truncate); if (n_offs != 0) { out = ggml_rope_set_offset(out, n_offs); } @@ -6138,6 +6749,13 @@ struct test_rope : public test_case { } } + double max_nmse_err() override { + // the f32 rotation angle of a position below n_ctx carries a rounding error of up to n_ctx*FLT_EPSILON radians + // backends compute the angle with different rounding sequences, so their agreement is bounded by that error + const double eps_theta = n_ctx * FLT_EPSILON; + return std::max(test_case::max_nmse_err(), eps_theta * eps_theta); + } + double max_maa_err() override { return 1e-3; } @@ -6697,7 +7315,7 @@ struct test_concat : public test_case { const std::array<int64_t, 4> ne_a; const int64_t ne_b_d; const int dim; - const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim)) + const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim), 1 << 4: transposed b) std::string vars() override { return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v); @@ -6746,6 +7364,13 @@ struct test_concat : public test_case { b = ggml_view_4d(ctx, b, ne_b[0], ne_b[1], ne_b[2], ne_b[3], b->nb[1], b->nb[2], b->nb[3], 0); ggml_set_name(b, "view_of_b"); + } else if (v & 16) { + auto ne = ne_b; std::swap(ne[0], ne[1]); + b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(b, "b"); + + b = ggml_transpose(ctx, b); + ggml_set_name(b, "transpose_of_b"); } else { b = ggml_new_tensor(ctx, type, 4, ne_b.data()); ggml_set_name(b, "b"); @@ -6837,6 +7462,34 @@ struct test_top_k : public test_case { // The logic in err can't handle the sentinel tensors. bool run_whole_graph() override { return ties; } + // the backends may break ties in different ways, but each backend must break them in the same way in each run + bool deterministic(ggml_backend_t backend) override { + // TODO: Vulkan is not deterministic yet, SYCL gathers the ties with an atomic counter as well + const char * name = ggml_backend_reg_name(ggml_backend_dev_backend_reg(ggml_backend_get_device(backend))); + // ROCm has no cub::DeviceTopK, every row goes through the radix or bitonic path + if (strcmp(name, "ROCm") == 0) { + return ties; + } + if (strcmp(name, "CUDA") != 0 && strcmp(name, "MTL") != 0) { + return false; + } + // CUDA uses cub::DeviceTopK for 1 or 2 rows, which does not guarantee deterministic ties (CCCL <= 3.5) + return ties && ne[1]*ne[2]*ne[3] > 2; + } + + // the order in a row is not specified, so compare the sets + bool same_output(ggml_tensor * t1, ggml_tensor * t2) override { + std::vector<int32_t> i1(ggml_nelements(t1)); + std::vector<int32_t> i2(ggml_nelements(t2)); + ggml_backend_tensor_get(t1, i1.data(), 0, ggml_nbytes(t1)); + ggml_backend_tensor_get(t2, i2.data(), 0, ggml_nbytes(t2)); + for (size_t r = 0; r < i1.size(); r += k) { + std::sort(i1.begin() + r, i1.begin() + r + k); + std::sort(i2.begin() + r, i2.begin() + r + k); + } + return i1 == i2; + } + double err(const float * a, const float * b, size_t n) override { // When there are no ties, we expect the exact same set of indices, // but possibly in a different order. When there are ties, the indices @@ -6933,6 +7586,46 @@ struct test_top_k : public test_case { } }; +// top_k over rows like log-probabilities: distinct negative values, fewer +// than k +inf (none for k = 1, so the expected indices are unique) and many +// -inf (masked tokens) +struct test_top_k_inf : public test_top_k { + test_top_k_inf(std::array<int64_t, 4> ne, int k) + : test_top_k(GGML_TYPE_F32, ne, k, false) {} + + std::string vars() override { + return test_top_k::vars() + ",inf=1"; + } + + // compare only the output: the input holds infinities, which err() would + // read as indices + bool run_whole_graph() override { return true; } + + void initialize_tensors(ggml_context * ctx) override { + std::random_device rd; + std::default_random_engine rng(rd()); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector<float> data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = -1.0f - i; + } + std::shuffle(data.begin(), data.end(), rng); + const int n_pinf = k / 2; + for (int i = 0; i < t->ne[0]; i++) { + if (i < n_pinf) { + data[i] = INFINITY; + } else if (i % 3 == 0) { + data[i] = -INFINITY; + } + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); + } + } + } +}; + // qwen4exp QSA indexer top-k fusion: expand per-block scores to cells, add the f16 mask, top-k. struct test_topk_qsa : public test_case { const int64_t n_blocks; @@ -7028,6 +7721,10 @@ struct test_topk_moe : public test_case { const bool bias_probs; const MoeGatingFunc gating_func; const float scale_w; + const int n_expert_groups; + ggml_tensor * logits {}; + ggml_tensor * exp_probs_b {}; + const bool norm_bias; ggml_tensor * weights {}; ggml_tensor * selected_experts {}; @@ -7036,17 +7733,24 @@ struct test_topk_moe : public test_case { bool with_norm = false, bool bias_probs = false, MoeGatingFunc gating_func = GATING_FUNC_SOFTMAX, - float scale_w = 0.0f) : + float scale_w = 0.0f, + int n_expert_groups = 1, + bool norm_bias = false) : ne(ne), n_expert_used(n_expert_used), with_norm(with_norm), bias_probs(bias_probs), gating_func(gating_func), - scale_w(scale_w) { + scale_w(scale_w), + n_expert_groups(n_expert_groups), + norm_bias(norm_bias) { GGML_ASSERT(n_expert_used <= ne[0]); + GGML_ASSERT(ne[0] % n_expert_groups == 0); + GGML_ASSERT(n_expert_used <= ne[0] / n_expert_groups); + GGML_ASSERT(!norm_bias || with_norm); } - std::string vars() override { return VARS_TO_STR6(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w); } + std::string vars() override { return VARS_TO_STR8(ne, n_expert_used, with_norm, norm_bias, bias_probs, gating_func, scale_w, n_expert_groups); } std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -7055,11 +7759,106 @@ struct test_topk_moe : public test_case { bool run_whole_graph() override { return true; } + // the selection scores (probs plus bias) of a row are a random permutation of a grid, and the experts of + // the n_group_used best groups move up by half a step, so every expert and group decision is one step + // apart; the logits are the inverse of the gating function at these scores + void initialize_tensors(ggml_context * ctx) override { + const int n_expert = ne[0]; + const int n_exp_per_group = n_expert / n_expert_groups; + const int n_group_used = n_expert_groups / 2; + const bool softmax = gating_func == GATING_FUNC_SOFTMAX; + + // softmax probs sum to one, so its grid step and bias scale with 1/n_expert + const double step = softmax ? 1.0 / ((double) n_expert * n_expert) : 0.5 / n_expert; + const double bmax = softmax ? 0.125 / n_expert : 0.2; + + std::default_random_engine rng(std::random_device{}()); + std::uniform_real_distribution<double> dist(-bmax, bmax); + + std::vector<float> bias(n_expert, 0.0f); + if (exp_probs_b) { + for (float & v : bias) { + v = dist(rng); + } + ggml_backend_tensor_set(exp_probs_b, bias.data(), 0, ggml_nbytes(exp_probs_b)); + } + + std::vector<int> perm(n_expert); + for (int i = 0; i < n_expert; ++i) { + perm[i] = i; + } + + std::vector<double> rank(n_expert); + std::vector<int> score(n_expert_groups); + std::vector<int> order(n_expert_groups); + std::vector<float> x(ggml_nelements(logits)); + for (int64_t r = 0; r < ggml_nrows(logits); ++r) { + std::shuffle(perm.begin(), perm.end(), rng); + for (int i = 0; i < n_expert; ++i) { + rank[i] = perm[i]; + } + + if (n_expert_groups > 1) { + for (int g = 0; g < n_expert_groups; ++g) { + const int * grp = perm.data() + g*n_exp_per_group; + int r0 = -1; + int r1 = -1; + for (int i = 0; i < n_exp_per_group; ++i) { + if (grp[i] > r0) { + r1 = r0; + r0 = grp[i]; + } else if (grp[i] > r1) { + r1 = grp[i]; + } + } + score[g] = r0 + r1; + order[g] = g; + } + std::sort(order.begin(), order.end(), [&](int a, int b) { return score[a] > score[b]; }); + for (int k = 0; k < n_group_used; ++k) { + for (int i = 0; i < n_exp_per_group; ++i) { + rank[order[k]*n_exp_per_group + i] += 0.5; + } + } + } + + // probs = scores - bias, with softmax probs shifted to sum to one + double c = 0.25; + if (softmax) { + double sum = 0.0; + for (int i = 0; i < n_expert; ++i) { + sum += rank[i]*step - bias[i]; + } + c = (1.0 - sum) / n_expert; + } + + float * row = x.data() + r*n_expert; + for (int i = 0; i < n_expert; ++i) { + const double p = c + rank[i]*step - bias[i]; + row[i] = gating_func == GATING_FUNC_SOFTMAX ? std::log(p) : + gating_func == GATING_FUNC_SIGMOID ? std::log(p / (1.0 - p)) : + gating_func == GATING_FUNC_SQRT_SOFTPLUS ? std::log(std::expm1(p*p)) : + p; + } + } + ggml_backend_tensor_set(logits, x.data(), 0, ggml_nbytes(logits)); + + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + const ggml_tensor * src = t->view_src ? t->view_src : t; + if (src != logits && src != exp_probs_b) { + init_tensor_uniform(t); + } + } + } + ggml_tensor * build_graph(ggml_context * ctx) override { const int n_expert = ne[0]; const int n_tokens = ne[1]; + const int n_exp_per_group = n_expert / n_expert_groups; + const int n_group_used = n_expert_groups / 2; - ggml_tensor * logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + exp_probs_b = nullptr; ggml_tensor * probs = (gating_func == GATING_FUNC_SOFTMAX) ? ggml_soft_max(ctx, logits) : (gating_func == GATING_FUNC_SIGMOID) ? ggml_sigmoid(ctx, logits) : @@ -7068,12 +7867,30 @@ struct test_topk_moe : public test_case { ggml_tensor * selection_probs = probs; if (bias_probs) { - ggml_tensor * exp_probs_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]); + exp_probs_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]); ggml_set_name(exp_probs_b, "exp_probs_b"); selection_probs = ggml_add(ctx, probs, exp_probs_b); ggml_set_name(selection_probs, "selection_probs"); } + if (n_expert_groups > 1) { + ggml_tensor * selection_groups = ggml_reshape_3d(ctx, selection_probs, n_exp_per_group, n_expert_groups, n_tokens); // [n_exp_per_group, n_expert_groups, n_tokens] + + ggml_tensor * group_scores = ggml_argsort_top_k(ctx, selection_groups, 2); // [2, n_expert_groups, n_tokens] + group_scores = ggml_get_rows(ctx, ggml_reshape_4d(ctx, selection_groups, 1, selection_groups->ne[0], selection_groups->ne[1], selection_groups->ne[2]), group_scores); // [1, 2, n_expert_groups, n_tokens] + + group_scores = ggml_sum_rows(ctx, ggml_reshape_3d(ctx, group_scores, group_scores->ne[1], group_scores->ne[2], group_scores->ne[3])); // [1, n_expert_groups, n_tokens] + group_scores = ggml_reshape_2d(ctx, group_scores, group_scores->ne[1], group_scores->ne[2]); // [n_expert_groups, n_tokens] + + ggml_tensor * expert_groups = ggml_argsort_top_k(ctx, group_scores, n_group_used); // [n_group_used, n_tokens] + ggml_set_name(expert_groups, "ffn_moe_group_topk"); + + selection_probs = ggml_get_rows(ctx, selection_groups, expert_groups); // [n_exp_per_group, n_group_used, n_tokens] + selection_probs = ggml_set_rows(ctx, ggml_fill(ctx, selection_groups, -INFINITY), selection_probs, expert_groups); // [n_exp_per_group, n_expert_groups, n_tokens] + selection_probs = ggml_reshape_2d(ctx, selection_probs, n_expert, n_tokens); // [n_expert, n_tokens] + ggml_set_name(selection_probs, "ffn_moe_probs_masked"); + } + selected_experts = ggml_argsort_top_k(ctx, selection_probs, n_expert_used); // [n_expert_used, n_tokens] ggml_set_name(selected_experts, "selected_experts"); @@ -7091,7 +7908,11 @@ struct test_topk_moe : public test_case { ggml_tensor * weights_sum = ggml_sum_rows(ctx, weights); // [1, n_tokens] ggml_set_name(weights_sum, "weights_sum"); - weights_sum = ggml_clamp(ctx, weights_sum, 6.103515625e-5, INFINITY); + if (norm_bias) { + weights_sum = ggml_scale_bias(ctx, weights_sum, 1.0f, 1e-20f); + } else { + weights_sum = ggml_clamp(ctx, weights_sum, 6.103515625e-5, INFINITY); + } weights = ggml_div(ctx, weights, weights_sum); // [n_expert_used, n_tokens] weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens); } @@ -7218,6 +8039,7 @@ enum mul_mat_add_mode { MUL_MAT_ADD_ROW, // mm + a one-row res, which a same-shape fusion must leave alone MUL_MAT_ADD_RES_INPLACE, // res += mm MUL_MAT_ADD_B_INPLACE, // b += mm: the sum overwrites the mat-mul input (m == k) + MUL_MAT_ADD_MM_MM, // mm2 + mm: the residual is itself a mat-mul output, so both operands are MUL_MAT }; static std::string var_to_str(mul_mat_add_mode mode) { @@ -7227,6 +8049,7 @@ static std::string var_to_str(mul_mat_add_mode mode) { case MUL_MAT_ADD_ROW: return "mm+row"; case MUL_MAT_ADD_RES_INPLACE: return "res+=mm"; case MUL_MAT_ADD_B_INPLACE: return "b+=mm"; + case MUL_MAT_ADD_MM_MM: return "mm2+mm"; } return "unknown"; } @@ -7272,6 +8095,12 @@ struct test_mul_mat_add : public test_case { case MUL_MAT_ADD_RES_MM: out = ggml_add(ctx, res, mm); break; case MUL_MAT_ADD_RES_INPLACE: out = ggml_add_inplace(ctx, res, mm); break; case MUL_MAT_ADD_B_INPLACE: out = ggml_add_inplace(ctx, b, mm); break; + case MUL_MAT_ADD_MM_MM: + { + ggml_tensor * a2 = ggml_new_tensor_2d(ctx, type_a, k, m); + ggml_tensor * mm2 = ggml_mul_mat(ctx, a2, b); + out = ggml_add(ctx, mm2, mm); + } break; } ggml_set_name(out, "out"); @@ -7279,7 +8108,7 @@ struct test_mul_mat_add : public test_case { } }; -struct test_mul_mat_vec_fusion : public test_case { +struct test_mul_mat_fusion : public test_case { const ggml_type type; const ggml_glu_op glu_op; const int64_t m; @@ -7293,24 +8122,25 @@ struct test_mul_mat_vec_fusion : public test_case { const bool with_gate; const bool with_lane_scale; std::array<int64_t, 2> batch_dims; + const bool skip; // set some ids to a negative value (only for use_id) - test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k, + test_mul_mat_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k, bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true, - bool with_lane_scale = false, std::array<int64_t, 2> batch_dims = {4, 2}) + bool with_lane_scale = false, std::array<int64_t, 2> batch_dims = {4, 2}, bool skip = false) : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), - with_gate(with_gate), with_lane_scale(with_lane_scale), batch_dims(batch_dims) { + with_gate(with_gate), with_lane_scale(with_lane_scale), batch_dims(batch_dims), skip(skip) { if (use_id) { GGML_ASSERT(n_used <= n_mats); } } std::string vars() override { - return VARS_TO_STR13(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_lane_scale, batch_dims); + return VARS_TO_STR14(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_lane_scale, batch_dims, skip); } std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); - return "MUL_MAT_VEC_FUSION"; + return "MUL_MAT_FUSION"; } bool run_whole_graph() override { return true; } @@ -7456,13 +8286,43 @@ struct test_mul_mat_vec_fusion : public test_case { init_tensor_uniform(t); } } else { - init_mul_mat_id_tensors(ctx, n_mats); + init_mul_mat_id_tensors(ctx, n_mats, skip); } } double max_nmse_err() override { return 5e-3; } + + double max_nmse_err(ggml_backend_t backend) override { + // 4-bit activations on Blackwell, see test_mul_mat + if (type == GGML_TYPE_NVFP4 && !graph_mul_mat_hi_prec_act(gf, use_id ? GGML_OP_MUL_MAT_ID : GGML_OP_MUL_MAT) && + backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { + return 2e-2; + } + return max_nmse_err(); + } +}; + +// Fused gate/up/GLU with GGML_PREC_Q8 on src1, e.g. a W4A16 NVFP4 checkpoint (W4A8, see test_mul_mat_w4a8) +struct test_mul_mat_fusion_w4a8 : public test_mul_mat_fusion { + using test_mul_mat_fusion::test_mul_mat_fusion; + using test_mul_mat_fusion::build_graph; + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat_fusion::build_graph(ctx, ctx_weights); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op == GGML_OP_MUL_MAT || t->op == GGML_OP_MUL_MAT_ID) { + ggml_prec_set_src(t, GGML_PREC_Q8, 1); + } + } + return out; + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_FUSION_W4A8"; + } }; // GGML_OP_SUM @@ -7471,24 +8331,43 @@ struct test_sum : public test_case { const std::array<int64_t, 4> ne; const std::array<int64_t, 4> permute; bool _use_permute; + const bool slice; std::string vars() override { std::string v = VARS_TO_STR2(type, ne); if (_use_permute) v += "," + VAR_TO_STR(permute); + if (slice) v += "," + VAR_TO_STR(slice); return v; } test_sum(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne = {10, 5, 4, 3}, - std::array<int64_t, 4> permute = {0, 0, 0, 0}) + std::array<int64_t, 4> permute = {0, 0, 0, 0}, + bool slice = false) : type(type), ne(ne), permute(permute), - _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {} + _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0), + slice(slice) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); + + // the backward pass of a sliced view scatters the gradient via GGML_OP_ACC, + // whose CPU implementation requires contiguous rows; a row-breaking permute + // (permute[0] != 0) on top of a slice violates that, so skip grad there + const bool grad_supported = !(slice && _use_permute && permute[0] != 0); + if (grad_supported) { + ggml_set_param(a); + } + ggml_set_name(a, "a"); + if (slice) { + a = ggml_view_4d(ctx, a, + ne[0], ne[1], ne[2] / 2, ne[3] - 1, + a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]); + ggml_set_name(a, "a_sliced"); + } + if (_use_permute) { a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]); ggml_set_name(a, "a_permuted"); @@ -7636,21 +8515,32 @@ struct test_interpolate : public test_case { const std::array<int64_t, 4> ne; const std::array<int64_t, 4> ne_tgt; const ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST; + const std::array<int64_t, 4> permute; + bool _use_permute; std::string vars() override { - return VARS_TO_STR4(type, ne, ne_tgt, mode); + std::string v = VARS_TO_STR4(type, ne, ne_tgt, mode); + if (_use_permute) v += "," + VAR_TO_STR(permute); + return v; } test_interpolate(ggml_type type = GGML_TYPE_F32, - std::array<int64_t, 4> ne = {2, 5, 7, 11}, - std::array<int64_t, 4> ne_tgt = {5, 7, 11, 13}, - ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST) - : type(type), ne(ne), ne_tgt(ne_tgt), mode(mode) {} + std::array<int64_t, 4> ne = {2, 5, 7, 11}, + std::array<int64_t, 4> ne_tgt = {5, 7, 11, 13}, + ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST, + std::array<int64_t, 4> permute = {0, 0, 0, 0}) + : type(type), ne(ne), ne_tgt(ne_tgt), mode(mode), permute(permute), + _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); ggml_set_name(a, "a"); + if (_use_permute) { + a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]); + ggml_set_name(a, "a_permuted"); + } + ggml_tensor * out = ggml_interpolate(ctx, a, ne_tgt[0], ne_tgt[1],ne_tgt[2], ne_tgt[3], mode); ggml_set_name(out, "out"); @@ -7943,18 +8833,28 @@ struct test_pad_reflect_1d : public test_case { const std::array<int64_t, 4> ne_a; const int pad_0; const int pad_1; + const bool view; std::string vars() override { - return VARS_TO_STR4(type, ne_a, pad_0, pad_1); + return VARS_TO_STR5(type, ne_a, pad_0, pad_1, view); } test_pad_reflect_1d(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne_a = {512, 34, 2, 1}, - int pad_0 = 10, int pad_1 = 9) - : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1) {} + int pad_0 = 10, int pad_1 = 9, bool view = false) + : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1), view(view) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 2, ne_a.data()); + ggml_tensor * a; + if (view) { + const size_t nb1 = ggml_row_size(type, ne_a[0]); + const size_t nb2 = nb1 * ne_a[1]; + const size_t nb3 = nb2 * ne_a[2]; + ggml_tensor * base = ggml_new_tensor_1d(ctx, type, ne_a[0] * ne_a[1] * ne_a[2] * ne_a[3] + 1); + a = ggml_view_4d(ctx, base, ne_a[0], ne_a[1], ne_a[2], ne_a[3], nb1, nb2, nb3, ggml_type_size(type)); + } else { + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + } ggml_set_name(a, "a"); ggml_tensor * out = ggml_pad_reflect_1d(ctx, a, pad_0, pad_1); @@ -8095,9 +8995,10 @@ struct test_flash_attn_ext : public test_case { const bool kv_view; // create K/V as views of a larger buffer (like a KV cache) const bool v_is_view_of_k; const int64_t n_kv_max; + const int64_t q_head_padding; std::string vars() override { - return VARS_TO_STR17(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k, n_kv_max); + return VARS_TO_STR18(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k, n_kv_max, q_head_padding); } double max_nmse_err() override { @@ -8114,9 +9015,9 @@ struct test_flash_attn_ext : public test_case { test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array<int64_t, 2> nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8, bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32, ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array<int32_t, 4> permute = {0, 1, 2, 3}, - bool kv_view = true, bool v_is_view_of_k = false, int64_t n_kv_max = 0) + bool kv_view = true, bool v_is_view_of_k = false, int64_t n_kv_max = 0, int64_t q_head_padding = 0) : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), - type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k), n_kv_max(n_kv_max) {} + type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k), n_kv_max(n_kv_max), q_head_padding(q_head_padding) {} ggml_tensor * build_graph(ggml_context * ctx) override { const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K)); @@ -8141,7 +9042,11 @@ struct test_flash_attn_ext : public test_case { return t; }; - ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded, nb, nh*nr23[0], nr23[1], false); + ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded + q_head_padding, nb, nh*nr23[0], nr23[1], false); + if (q_head_padding > 0) { + ggml_set_name(q, "q_padded"); + q = ggml_view_4d(ctx, q, hsk_padded, q->ne[1], q->ne[2], q->ne[3], q->nb[1], q->nb[2], q->nb[3], 0); + } ggml_set_name(q, "q"); ggml_tensor * k = create_permuted(type_K, hsk_padded, kv, nh, nr23[1], kv_view); // the K tensor is usually a view of the K cache @@ -8221,6 +9126,102 @@ struct test_flash_attn_ext_large_logits : public test_flash_attn_ext { } }; +// GGML_OP_FLASH_ATTN_EXT with kv_rows: each q slice attends to its own list of K/V cache rows +struct test_flash_attn_ext_kv_rows : public test_case { + const int64_t hs; + const int64_t nh; // K/V heads + const int64_t gqa; + const int64_t n_cells; // rows of the K/V cache + const int64_t kv; // list length per slice + const int64_t nb; // queries per slice + const int64_t n_slices; + const ggml_type type_KV; + + std::vector<int32_t> rows; + + std::string vars() override { + return VARS_TO_STR8(hs, nh, gqa, n_cells, kv, nb, n_slices, type_KV); + } + + double max_nmse_err() override { + return 5e-4; + } + + test_flash_attn_ext_kv_rows(int64_t hs = 256, int64_t nh = 2, int64_t gqa = 8, int64_t n_cells = 2048, int64_t kv = 512, + int64_t nb = 8, int64_t n_slices = 4, ggml_type type_KV = GGML_TYPE_F16) + : hs(hs), nh(nh), gqa(gqa), n_cells(n_cells), kv(kv), nb(nb), n_slices(n_slices), type_KV(type_KV) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, hs, nb, nh*gqa, n_slices); + ggml_set_name(q, "q"); + + // cache layout [hs, nh, n_cells], shared by all slices + ggml_tensor * k0 = ggml_new_tensor_3d(ctx, type_KV, hs, nh, n_cells); + ggml_tensor * v0 = ggml_new_tensor_3d(ctx, type_KV, hs, nh, n_cells); + ggml_set_name(k0, "k0"); + ggml_set_name(v0, "v0"); + + ggml_tensor * k = ggml_permute(ctx, k0, 0, 2, 1, 3); + ggml_tensor * v = ggml_permute(ctx, v0, 0, 2, 1, 3); + + ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, nb, 1, n_slices); + ggml_set_name(m, "m"); + + ggml_tensor * r = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, kv, n_slices); + ggml_set_name(r, "r"); + + ggml_tensor * out = ggml_flash_attn_ext_rows(ctx, q, k, v, m, r, 1.0f/sqrtf(hs), 0.0f, 0.0f); + ggml_prec_set_acc(out, GGML_PREC_F32); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + std::mt19937 gen(0x6B76); + + // per slice: distinct random cells, the tail of the list is padding + rows.assign(kv*n_slices, -1); + std::vector<int32_t> order(n_cells); + for (int64_t s = 0; s < n_slices; ++s) { + for (int64_t i = 0; i < n_cells; ++i) { + order[i] = i; + } + std::shuffle(order.begin(), order.end(), gen); + const int64_t n_used = std::min<int64_t>(n_cells, kv - (s*37) % std::max<int64_t>(1, kv/2)); + for (int64_t i = 0; i < n_used; ++i) { + rows[s*kv + i] = order[i]; + } + } + + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->view_src) { + continue; + } + if (strcmp(t->name, "r") == 0) { + ggml_backend_tensor_set(t, rows.data(), 0, rows.size()*sizeof(int32_t)); + } else if (strcmp(t->name, "m") == 0) { + std::vector<float> f(ggml_nelements(t)); + std::uniform_real_distribution<float> dis(-1.0f, 1.0f); + for (int64_t s = 0; s < n_slices; ++s) { + for (int64_t j = 0; j < nb; ++j) { + for (int64_t i = 0; i < kv; ++i) { + const bool pad = rows[s*kv + i] < 0; + const bool drop = (i + 3*j + 5*s) % 11 == 0; + f[(s*nb + j)*kv + i] = pad || drop ? -INFINITY : dis(gen); + } + } + } + std::vector<ggml_fp16_t> h(f.size()); + ggml_fp32_to_fp16_row(f.data(), h.data(), f.size()); + ggml_backend_tensor_set(t, h.data(), 0, h.size()*sizeof(ggml_fp16_t)); + } else { + init_tensor_uniform(t); + } + } + } +}; + // GGML_OP_CROSS_ENTROPY_LOSS struct test_cross_entropy_loss : public test_case { const ggml_type type; @@ -8488,6 +9489,69 @@ struct test_tri : public test_case { } }; +// ggml_tri where the values that have to be discarded are not finite: they must come back as a plain +// zero. A backend that scales by the comparison instead of selecting turns them into NaN. +struct test_tri_special : public test_case { + const ggml_type type; + const std::array<int64_t, 4> ne; + const ggml_tri_type tri_type; + + std::string vars() override { return VARS_TO_STR3(type, ne, tri_type); } + + test_tri_special(ggml_tri_type tri_type, ggml_type type = GGML_TYPE_F32, + std::array<int64_t, 4> ne = { 16, 16, 2, 2 }) + : type(type), ne(ne), tri_type(tri_type) { + GGML_ASSERT(type == GGML_TYPE_F32); + GGML_ASSERT(ne[0] == ne[1]); + } + + bool keeps(int64_t i, int64_t r) const { + switch (tri_type) { + case GGML_TRI_TYPE_LOWER: return i < r; + case GGML_TRI_TYPE_LOWER_DIAG: return i <= r; + case GGML_TRI_TYPE_UPPER: return i > r; + case GGML_TRI_TYPE_UPPER_DIAG: return i >= r; + } + return false; + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_tri(ctx, a, tri_type); + + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + static const float discarded[] = { 0.0f, -0.0f, 1.0f, -1.0f, INFINITY, -INFINITY }; + + int64_t k = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type != type || t->ne[0] != ne[0] || t->ne[1] != ne[1]) { + init_tensor_uniform(t, -1.0f, 1.0f); + continue; + } + std::vector<float> data(ggml_nelements(t)); + for (int64_t i3 = 0; i3 < t->ne[3]; ++i3) { + for (int64_t i2 = 0; i2 < t->ne[2]; ++i2) { + for (int64_t i1 = 0; i1 < t->ne[1]; ++i1) { + for (int64_t i0 = 0; i0 < t->ne[0]; ++i0) { + const int64_t off = ((i3*t->ne[2] + i2)*t->ne[1] + i1)*t->ne[0] + i0; + data[off] = keeps(i0, i1) ? 1.0f : discarded[k++ % (int64_t) (sizeof(discarded)/sizeof(discarded[0]))]; + } + } + } + } + ggml_backend_tensor_set(t, data.data(), 0, data.size()*sizeof(float)); + } + } +}; + // GGML_OP_FILL struct test_fill : public test_case { const ggml_type type; @@ -8518,8 +9582,9 @@ struct test_solve_tri : public test_case { const ggml_type type; const std::array<int64_t, 4> ne_lhs; const std::array<int64_t, 4> ne_rhs; + const bool nan_out; // fill the output with NaN before the op runs - std::string vars() override { return VARS_TO_STR3(type, ne_lhs, ne_rhs); } + std::string vars() override { return VARS_TO_STR4(type, ne_lhs, ne_rhs, nan_out); } uint64_t op_flops(ggml_tensor * t) override { GGML_UNUSED(t); @@ -8532,9 +9597,10 @@ struct test_solve_tri : public test_case { test_solve_tri(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne_lhs = { 10, 10, 4, 3 }, - std::array<int64_t, 4> ne_rhs = { 3, 10, 4, 3 } + std::array<int64_t, 4> ne_rhs = { 3, 10, 4, 3 }, + bool nan_out = false ) - : type(type), ne_lhs(ne_lhs), ne_rhs(ne_rhs) {} + : type(type), ne_lhs(ne_lhs), ne_rhs(ne_rhs), nan_out(nan_out) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne_lhs[0], ne_lhs[1], ne_lhs[2], ne_lhs[3]); @@ -8556,6 +9622,9 @@ struct test_solve_tri : public test_case { if (strcmp(t->name, "a") == 0) { // note: avoid zeros in the diagonal init_tensor_tril(t, 0.1, 1.0f); + } else if (nan_out && strcmp(t->name, "out") == 0) { + std::vector<float> data(ggml_nelements(t), NAN); + ggml_backend_tensor_set(t, data.data(), 0, ggml_nbytes(t)); } else { init_tensor_uniform(t, -1.0f, 1.0f); } @@ -8648,6 +9717,150 @@ struct test_lightning_indexer : public test_case { } }; +// GGML_OP_TOP_K_MASK +struct test_top_k_mask : public test_case { + const ggml_type type; + const int64_t n_kv; + const int64_t n_blk; // 0: no blocks, a pick is a cell + const int64_t n_pick; // number of blocks, ignored without blocks + const int64_t n_top; + const int64_t n_extra; // 0: no extra cells + const bool score; // false: every pick is live + const int64_t nb; // rows per stream + const int64_t nh; // rows of top_k per mask row, broadcast over ne2 + const int64_t ns; // streams + const bool real; // blocks of consecutive cells, live picks, as in a real cache + const bool bcast; // one mask row for all rows and streams + + std::string vars() override { + return VARS_TO_STR12(type, n_kv, n_blk, n_pick, n_top, n_extra, score, nb, nh, ns, real, bcast); + } + + // the result is a copy of the mask or -inf, the harness checks the -inf pattern, the finite values must match exactly + double max_err() override { + return 0.0; + } + + double err(const float * a, const float * b, size_t n) override { + double result = 0.0; + for (size_t i = 0; i < n; ++i) { + if (std::isfinite(a[i]) && std::isfinite(b[i])) { + result = std::max(result, (double) fabsf(a[i] - b[i])); + } + } + return result; + } + + size_t op_size(ggml_tensor * t) override { + const size_t ts = ggml_type_size(type); + const size_t nc = std::max<int64_t>(n_blk, 1); + return ggml_nbytes(t) + nb*nh*ns*(n_top*(sizeof(int32_t) + (score ? sizeof(float) : 0) + nc*(sizeof(int32_t) + 2*ts)) + n_extra*(sizeof(int32_t) + 2*ts)); + } + + test_top_k_mask(ggml_type type = GGML_TYPE_F16, int64_t n_kv = 256, int64_t n_blk = 4, int64_t n_pick = 64, + int64_t n_top = 16, int64_t n_extra = 3, bool score = true, int64_t nb = 32, int64_t nh = 1, int64_t ns = 1, bool real = false, + bool bcast = false) + : type(type), n_kv(n_kv), n_blk(n_blk), n_pick(n_blk ? n_pick : n_kv), n_top(n_top), n_extra(n_extra), score(score), + nb(nb), nh(nh), ns(ns), real(real), bcast(bcast) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * m = bcast ? ggml_new_tensor_1d(ctx, type, n_kv) : ggml_new_tensor_4d(ctx, type, n_kv, nb, 1, ns); + ggml_set_name(m, "m"); + + ggml_tensor * top_k = ggml_new_tensor_4d(ctx, GGML_TYPE_I32, n_top, nb, nh, ns); + ggml_set_name(top_k, "top_k"); + + ggml_tensor * s = nullptr; + if (score) { + s = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_pick, nb, nh, ns); + ggml_set_name(s, "score"); + } + + ggml_tensor * blocks = nullptr; + if (n_blk > 0) { + blocks = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_blk, n_pick); + ggml_set_name(blocks, "blocks"); + } + + ggml_tensor * extra = nullptr; + if (n_extra > 0) { + extra = ggml_new_tensor_4d(ctx, GGML_TYPE_I32, n_extra, nb, nh, ns); + ggml_set_name(extra, "extra"); + } + + ggml_tensor * out = ggml_top_k_mask(ctx, m, top_k, s, blocks, extra); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + std::mt19937 gen(0x4B50); + + // cells and picks sometimes fall out of range, the op skips them + auto init_cells = [&](ggml_tensor * t) { + std::vector<int32_t> data(ggml_nelements(t)); + if (real) { + // block p holds cells p*n_blk..p*n_blk + n_blk - 1, the extra cells are the last cells + const bool is_blocks = strcmp(t->name, "blocks") == 0; + for (int64_t i = 0; i < (int64_t) data.size(); ++i) { + const int64_t c = is_blocks ? i : n_kv - 1 - i % t->ne[0]; + data[i] = c < n_kv ? (int32_t) c : (int32_t) n_kv; + } + } else { + std::uniform_int_distribution<int32_t> cell(0, n_kv - 1); + for (auto & c : data) { + const uint32_t r = gen() % 16; + c = r == 0 ? (int32_t) n_kv : r == 1 ? -1 : cell(gen); + } + } + ggml_backend_tensor_set(t, data.data(), 0, data.size()*sizeof(int32_t)); + }; + + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "m") == 0) { + if (type == GGML_TYPE_F16) { + init_tensor_kq_mask(t); + } else { + std::vector<float> data(ggml_nelements(t)); + std::uniform_real_distribution<float> dis(-1.0f, 1.0f); + for (auto & v : data) { + v = gen() % 4 == 0 ? -INFINITY : dis(gen); + } + ggml_backend_tensor_set(t, data.data(), 0, data.size()*sizeof(float)); + } + } else if (strcmp(t->name, "score") == 0) { + // -inf marks the picks a row cannot see + std::vector<float> data(ggml_nelements(t)); + std::uniform_real_distribution<float> dis(-1.0f, 1.0f); + for (auto & v : data) { + v = !real && gen() % 4 == 0 ? -INFINITY : dis(gen); + } + ggml_backend_tensor_set(t, data.data(), 0, data.size()*sizeof(float)); + } else if (strcmp(t->name, "top_k") == 0) { + std::vector<int32_t> data(ggml_nelements(t)); + std::vector<int32_t> order(n_pick); + for (int64_t r = 0; r < ggml_nrows(t); ++r) { + for (int64_t i = 0; i < n_pick; ++i) { + order[i] = i; + } + std::shuffle(order.begin(), order.end(), gen); + for (int64_t j = 0; j < n_top; ++j) { + const bool out = !real && j == 0 && r % 5 == 0; + data[r*n_top + j] = out ? (r % 2 ? -1 : (int32_t) n_pick) : order[j % n_pick]; + } + } + ggml_backend_tensor_set(t, data.data(), 0, data.size()*sizeof(int32_t)); + } else if (strcmp(t->name, "blocks") == 0 || strcmp(t->name, "extra") == 0) { + init_cells(t); + } else { + // the harness sentinels and the output, the callback checks the sentinels for NaN + init_tensor_uniform(t); + } + } + } +}; + // Deserializable generic test case struct input_tensor { ggml_type type; @@ -9280,6 +10493,7 @@ static const ggml_type all_types[] = { GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, + GGML_TYPE_F8_E4M3, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, @@ -9299,6 +10513,7 @@ static const ggml_type base_types[] = { GGML_TYPE_Q4_1, // for I8MM tests GGML_TYPE_Q4_K, GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, // TODO: or "other" + GGML_TYPE_F8_E4M3, GGML_TYPE_IQ2_XXS }; @@ -9330,6 +10545,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { std::default_random_engine rng(0); // unary ops + // TODO: add GGML_UNARY_OP_SOFTPLUS test that is sensitive to using `log1p(exp(x))` vs `log(exp(x) + 1)` for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { for (int v : {0, 1}) { for (int op = 0; op < GGML_UNARY_OP_COUNT; op++) { @@ -9349,7 +10565,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } // fused unary + mul (gated activations that are not expressed as GGML_OP_GLU) - for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) { + for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_RELU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) { for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16 }) { for (bool swap : { false, true }) { test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, swap)); @@ -9370,6 +10586,24 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } + // fused add + unary + mul: the delta-net alpha gate, bias and scale broadcast over dim 0 + for (ggml_unary_op op : { GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) { + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 32, 7, 1, 1 })); + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 32, 512, 1, 1 })); + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 5, 7, 11, 13 })); + // one token: no broadcast left, and the unary result may be either MUL operand + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 32, 1, 1, 1 }, false, "same_shape")); + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 32, 1, 1, 1 }, true, "same_shape")); + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 32, 7, 1, 1 }, false, "bcast", "consumer")); + // must not fuse + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 32, 7, 1, 1 }, false, "same_shape")); + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 32, 7, 1, 1 }, false, "rep_ne0")); + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F32, { 32, 7, 1, 1 }, false, "bcast", "reuse")); + test_cases.emplace_back(new test_add_unary_mul(op, GGML_TYPE_F16, { 32, 7, 1, 1 })); + } + // a unary op with no fused kernel must fall back to the three-op chain + test_cases.emplace_back(new test_add_unary_mul(GGML_UNARY_OP_GELU, GGML_TYPE_F32, { 32, 7, 1, 1 })); + // SNAKE activation fusion: x + sin(a*x)^2 * inv_b for (ggml_type type : { GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16 }) { test_cases.emplace_back(new test_snake_fuse(type, { 5, 7, 1, 1})); // primes sub-block @@ -9413,6 +10647,37 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_dsv4_hc_post(31, 17, true, true)); test_cases.emplace_back(new test_dsv4_hc_post(2560, 21, true, true)); test_cases.emplace_back(new test_dsv4_hc_post(31, 17, false, true)); + test_cases.emplace_back(new test_dsv4_hc_post(4096, 1)); // DeepSeek-V4-Flash decode + test_cases.emplace_back(new test_dsv4_hc_post(2560, 1, true, true)); // Qwen4-Exp decode + + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + // blocks with extra cells and scores (k-pool) + test_cases.emplace_back(new test_top_k_mask(type)); + test_cases.emplace_back(new test_top_k_mask(type, 1001, 4, 70, 16, 3, true, 7)); + test_cases.emplace_back(new test_top_k_mask(type, 256, 4, 64, 64, 0, true, 33)); + test_cases.emplace_back(new test_top_k_mask(type, 512, 8, 16, 32, 7, true, 5, 1, 4)); + test_cases.emplace_back(new test_top_k_mask(type, 4096, 4, 1024, 512, 3, true, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 4096, 4, 1024, 512, 3, true, 64, 1, 2)); + // cells without scores (DSA) + test_cases.emplace_back(new test_top_k_mask(type, 256, 0, 0, 16, 0, false, 32)); + test_cases.emplace_back(new test_top_k_mask(type, 1001, 0, 0, 64, 0, false, 7, 1, 3)); + test_cases.emplace_back(new test_top_k_mask(type, 4096, 0, 0, 2048, 0, false, 5)); + // one row, no blocks, fewer than 128 picks: no read past the pick list + test_cases.emplace_back(new test_top_k_mask(type, 1001, 0, 0, 16, 3, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 256, 0, 0, 40, 0, true, 1)); + // one row (single-row kernel) with each optional input combination: no score and no extra, blocks only, blocks and extra without score + test_cases.emplace_back(new test_top_k_mask(type, 512, 0, 0, 16, 0, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 1024, 4, 200, 32, 0, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 1024, 4, 200, 32, 5, false, 1)); + // cells with scores and extra cells + test_cases.emplace_back(new test_top_k_mask(type, 512, 0, 0, 32, 5, true, 9)); + // blocks picked per head, the mask broadcast over the heads + test_cases.emplace_back(new test_top_k_mask(type, 512, 16, 32, 4, 0, true, 6, 4, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 1024, 8, 128, 16, 8, false, 3, 2, 2)); + // one mask row for all rows and streams (QSA) + test_cases.emplace_back(new test_top_k_mask(type, 1001, 4, 70, 16, 3, true, 7, 1, 1, false, true)); + test_cases.emplace_back(new test_top_k_mask(type, 512, 8, 16, 32, 7, true, 5, 1, 4, false, true)); + } // glu ops for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { @@ -9433,10 +10698,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } - for (int v : {0, 1}) { - for (float alpha : {.5f, 1.702f}) { - for (float limit : {2.0f, 7.0f}) { - test_cases.emplace_back(new test_swiglu_oai(GGML_TYPE_F32, { 128, 2, 2, 2 }, v, alpha, limit)); + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int v : {0, 1}) { + for (float alpha : {.5f, 1.702f}) { + for (float limit : {2.0f, 7.0f}) { + test_cases.emplace_back(new test_swiglu_oai(type, { 128, 2, 2, 2 }, v, alpha, limit)); + } } } } @@ -9456,6 +10723,16 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 1, 8, 2, 1, 1, false)); + for (ggml_type type : {GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, + GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S}) { + test_cases.emplace_back(new test_get_rows_after_mul_mat(type)); + } + for (ggml_type type : {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S}) { + test_cases.emplace_back(new test_mul_mat_reorder_then_big_batch(type)); + } + for (int n : {1, 3, 257}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, n, 5, 3, 1, 1, false)); + } for (ggml_type type : all_types) { for (int b : {1, 7}) { for (bool v : {false, true}) { @@ -9519,6 +10796,20 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 128, 32, 8, 1 }, GGML_ROPE_TYPE_IMROPE)); + // partial rotation + for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE }) { + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 256, 4, 8, 1 }, mode, 64)); + } + } + + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (bool truncate : {false, true}) { + test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, {128, 2, 3, 2}, mode, 128, true, truncate)); + } + } + } for (ggml_type type_input : {GGML_TYPE_F32}) { for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { @@ -9535,6 +10826,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } } + // MSA indexer block-max shape: wide 1-D window along ne0, no padding, non-divisible width + for (int64_t iw : {4096, 4160}) { + for (int blk : {32, 64, 128}) { + test_cases.emplace_back(new test_pool2d(pool_type, type_input, {iw, 2, 1, 1}, blk, 1, blk, 1, 0, 0)); + } + } + test_cases.emplace_back(new test_pool2d(pool_type, type_input, {259, 5, 3, 2}, 128, 3, 64, 2, 64, 1)); + test_cases.emplace_back(new test_pool2d(pool_type, type_input, {193, 2, 1, 1}, 128, 1, 128, 1, 128, 2)); } } @@ -9561,9 +10860,25 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // im2col 1D test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); + // deepseek4v / deepseek41v aligner: 3x3 stride-3 unfold (F.unfold == im2col) + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {48, 33, 1024, 1}, {3, 3, 1024, 1}, 3, 3, 0, 0, 1, 1, true)); test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 384, 1, 1}, {3, 384, 384, 1}, 1, 0, 1, 0, 1, 0, false)); + // 1D convs of an audio decoder: long sequences, dilated taps, pointwise kernels + for (int d0 : {1, 3, 9}) { + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {2000 + 6 * d0, 96, 1, 1}, {7, 96, 96, 1}, 1, 0, 0, 0, d0, 0, false)); + } + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {1500, 192, 1, 1}, {1, 192, 192, 1}, 1, 0, 0, 0, 1, 0, false)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {1001, 64, 1, 1}, {5, 64, 64, 1}, 2, 0, 2, 0, 1, 0, false)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {777, 48, 2, 1}, {3, 48, 48, 1}, 1, 0, 4, 0, 3, 0, false)); + // several rows of a few output-col blocks each: a thread's range of blocks can start mid-row + for (int64_t n : {5, 7}) { + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {600, 8, n, 1}, {3, 8, 8, 1}, 1, 0, 1, 0, 1, 0, false)); + } + // 2D with a kernel one row tall: with p1 > 0 some output rows fall outside the input height + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {40, 6, 8, 1}, {3, 1, 8, 4}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {40, 6, 8, 1}, {3, 1, 8, 4}, 1, 1, 1, 0, 1, 1, true)); for (int s0 : {1, 3}) { for (int p0 : {0, 3}) { for (int d0 : {1, 3}) { @@ -9608,6 +10923,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 1536, 729}, {2, 2, 1536, 4096}, 1, 1, 0, 0, 1, 1, true)); test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 1, 2}, {32, 33, 1, 2}, 1, 1, 1, 1, 1, 1, true)); test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 2, 1}, {33, 34, 2, 1}, 1, 1, 1, 1, 1, 1, true)); + // non-overlapping (stride == kernel, no pad/dilation) with IC*KH well above 256, e.g. SD1.5 1x1 convs + for (ggml_type dst_type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {64, 64, 320, 1}, {1, 1, 320, 1}, 1, 1, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {16, 16, 1280, 2}, {1, 1, 1280, 1}, 1, 1, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {8, 8, 2560, 1}, {1, 1, 2560, 1}, 1, 1, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {32, 32, 192, 1}, {2, 2, 192, 1}, 2, 2, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {64, 512, 1, 1}, {2, 512, 1, 1}, 2, 0, 0, 0, 1, 0, false)); + } // im2col 3D test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32)); @@ -9761,6 +11084,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { }; for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_conv_3d(1, 4, 6, 78, 54, 512, 3, 3, 3, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(2, 16, 4, 64, 64, 256, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); for (int N : {1, 2}) { for (int IC : {1, 3}) { for (int OC : {1, 4}) { @@ -9817,6 +11142,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + // input and output channel strides just over 16 MB, beyond the 24-bit DMA stride field + test_cases.emplace_back(new test_conv_3d(1, 2, 65, 256, 256, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + // KD * KH = 127, 128, 255, 256 around the HTP DMA queue limits + test_cases.emplace_back(new test_conv_3d(1, 2, 127, 1, 40, 3, 127, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 64, 2, 40, 3, 64, 2, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 85, 3, 40, 3, 85, 3, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 16, 16, 40, 3, 16, 16, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); } for(uint32_t Cout : {1, 9}){ @@ -9862,8 +11194,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 2}, {3, 3, 1, 9}, 2, kernel_type)); // for multiple batches } - test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1})); - test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 5000, 1, 1})); + for (bool same_src : {false, true}) { + test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1}, same_src)); + test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 5000, 1, 1}, same_src)); + } test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 1, 1, 1})); test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 513, 1, 1})); @@ -9942,6 +11276,19 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); // cpy not-contiguous } } + for (ggml_type type_src : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { + test_cases.emplace_back(new test_cpy(type_src, GGML_TYPE_F8_E4M3, {32, 2, 3, 4}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {64, 2, 3, 4})); + test_cases.emplace_back(new test_cpy(type_src, GGML_TYPE_F8_E4M3, {32, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(type_src, GGML_TYPE_F8_E4M3, {32, 2, 3, 4}, {16, 4, 3, 4}, {1, 0, 2, 3}, {0, 2, 1, 3})); + } + + for (ggml_type type_dst : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { + test_cases.emplace_back(new test_cpy(GGML_TYPE_F8_E4M3, type_dst, {33, 2, 3, 4})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F8_E4M3, type_dst, {32, 2, 3, 4}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {64, 2, 3, 4})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F8_E4M3, type_dst, {32, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F8_E4M3, type_dst, {32, 2, 3, 4}, {16, 4, 3, 4}, {1, 0, 2, 3}, {0, 2, 1, 3})); + } + // quant block count not a multiple of the kernel block size test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {96, 1, 1, 1})); test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {96, 1, 1, 1})); @@ -9960,6 +11307,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2097121, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {1, 64, 48, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // ne[0] == 1 permute + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {1, 64, 48, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // ne[0] == 1 permute test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 524281, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst @@ -10022,6 +11371,35 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } + // source strides over 16 MB (24-bit DMA descriptor limit) + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {512, 8193, 2, 1}, false, {2, 1, 0, 3})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {256, 16385, 2, 1}, false, {0, 2, 1, 3})); + + // channel swap with sizes not multiples of 32/64 tile + for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_F16 }) { + test_cases.emplace_back(new test_cont(type_dst, {33, 17, 5, 19}, false, {1, 2, 3, 0})); + } + + // small channel dims and extent-1 dims + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {8, 104, 60, 13}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F16, {8, 104, 60, 13}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {16, 1, 50, 7}, false, {1, 2, 3, 0})); + + // full reversal: contiguous src dim lands on dst dim 3 + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {33, 5, 7, 9}, false, {3, 2, 1, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {33, 64, 32, 16}, false, {3, 2, 1, 0})); + + // permutes that move dense blocks (ne[0] stays dim 0) + for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_F16 }) { + test_cases.emplace_back(new test_cont(type_dst, {4, 65, 9, 2}, false, {0, 2, 1, 3})); + } + test_cases.emplace_back(new test_cont(GGML_TYPE_F16, {64, 9, 7, 1}, false, {0, 2, 1, 3})); + + // transpose with i32 and CPY with reshape + test_cases.emplace_back(new test_cont(GGML_TYPE_I32, {33, 17, 5, 19}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {33, 17, 5, 1}, {17, 33, 5, 1}, {0, 0, 0, 0}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {8, 52, 39, 1}, {8, 39, 52, 1}, {0, 0, 0, 0}, {0, 2, 1, 3})); + auto add_test_bin_bcast = [&](ggml_type type, std::array<int64_t, 4> ne, std::array<int, 4> nr, bool perm1 = false, bool src_overlap = false) { for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) { test_cases.emplace_back(new test_bin_bcast(op, type, ne, nr, 1, perm1, src_overlap)); @@ -10073,6 +11451,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { //add_test_bin_bcast(type, {3, 3, 2560, 1280}, {2, 1, 1, 1}); } + for (auto op : {ggml_add, ggml_add_inplace}) { + for (int64_t n : {1, 7, 33}) { + test_cases.emplace_back(new test_bin_bcast(op, GGML_TYPE_F8_E4M3, {n, 5, 2, 1}, {1, 1, 1, 1}, 1, false, false, GGML_TYPE_F32)); + } + } + + for (auto op : {ggml_add, ggml_mul}) { + test_cases.emplace_back(new test_bin_bcast(op, GGML_TYPE_F32, {1, 2, 2, 2}, {17, 1, 1, 1}, 1, false, false, GGML_TYPE_COUNT, true)); + } + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {2, 2, 2, 2}, {17, 1, 1, 1}, 1, false, false, GGML_TYPE_COUNT, true)); + // single inplace tests, especially important for WebGPU backend since kernels for inplace vs. not are different test_cases.emplace_back(new test_bin_bcast(ggml_add_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); test_cases.emplace_back(new test_bin_bcast(ggml_mul_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); @@ -10107,6 +11496,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f)); test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {100, 10, 10, 10}, 0.5f, 0.0f)); + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 0.0f, false, 1)); // view test + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace + test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace test_cases.emplace_back(new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f)); test_cases.emplace_back(new test_silu_back()); @@ -10139,10 +11532,27 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // in-place tests test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, false, 1e-6f, true)); + // shapes below exceed the CUDA gridDim.y/gridDim.z limit of 65535 (#27901) + test_cases.emplace_back(new test_norm (GGML_TYPE_F32, {4, 1, 65536, 1}, false, 1e-6f)); + test_cases.emplace_back(new test_rms_norm (GGML_TYPE_F32, {4, 1, 65536, 1}, false, 1e-6f, false)); + test_cases.emplace_back(new test_rms_norm (GGML_TYPE_F32, {4, 1, 1, 65536}, false, 1e-6f, false)); + test_cases.emplace_back(new test_l2_norm (GGML_TYPE_F32, {4, 1, 65536, 1}, 1e-12f, false, false)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {4, 1, 65536, 1}, 1e-6f, false, false)); + + for (uint32_t n : {1, 127, 129, 511, 513, 2816, 4096, 16385}) { + test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {n, 3, 2, 1})); + test_cases.emplace_back(new test_rms_norm_mul_rope({n, 3, 2, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NORMAL, true, false)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {n, 3, 2, 1}, 1e-6f, false, false, false, false, true)); + } for (ggml_type set_rows_type : { GGML_TYPE_F32, GGML_TYPE_F16 }) { test_cases.emplace_back(new test_rms_norm_mul_rope({ 256, 1, 1, 1 }, 1e-6f, false, true, false, GGML_ROPE_TYPE_NORMAL, false, false, set_rows_type)); test_cases.emplace_back(new test_rms_norm_mul_rope({ 128, 4, 3, 1 }, 1e-6f, false, true, false, GGML_ROPE_TYPE_NORMAL, false, false, set_rows_type)); + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + for (int64_t n : {64, 256, 512, 1024}) { + test_cases.emplace_back(new test_rms_norm_mul_rope({n, 4, 3, 1}, 1e-6f, false, true, true, mode, true, true, set_rows_type)); + } + } } for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 1.0f }) { @@ -10197,7 +11607,22 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } } - for (int64_t d_conv : {3, 4, 9}) { + // ne[2] > 65535 exceeds the CUDA gridDim.y limit (#27901) + test_cases.emplace_back(new test_rms_norm_mul_rope({4, 1, 65536, 1}, 1e-6f, false, false, false, GGML_ROPE_TYPE_NORMAL)); + test_cases.emplace_back(new test_rms_norm_mul_rope({4, 1, 65536, 1}, 1e-6f, false, true, false, GGML_ROPE_TYPE_NEOX)); + + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + for (int64_t n : {128, 1024}) { + for (bool truncate : {false, true}) { + test_cases.emplace_back(new test_rms_norm_mul_rope({n, 2, 3, 1}, 1e-6f, false, false, true, mode, true, true, GGML_TYPE_F16, true, truncate)); + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_rms_norm_mul_rope({n, 2, 3, 1}, 1e-6f, false, true, true, mode, true, true, type, true, truncate)); + } + } + } + } + + for (int64_t d_conv : {3, 4, 7, 9}) { for (int64_t d_inner: {1024, 1536, 2048}) { test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {2 * d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); @@ -10210,7 +11635,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // fused ssm_conv + (optional) bias_add + silu. The bias-only graph (no silu) is intentionally // not tested since there's no fusion for that pattern in ggml_cuda_can_fuse. - for (int64_t d_conv : {3, 4, 9}) { + for (int64_t d_conv : {3, 4, 7, 9}) { for (int64_t d_inner : {1024, 1536, 2048}) { for (bool fuse_bias : {false, true}) { // short token path (n_t <= 32) @@ -10230,6 +11655,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 16, 1, 1024, 1, 32, 4)); // Mamba-1 + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 64, 64, 112, 2, 32, 4)); // Zamba2 (real hparams: d_inner=7168, dt_rank=112, n_group=2 -> head_dim=7168/112=64) + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 64, 64, 112, 2, 1, 1)); // Zamba2 decode (single-token, matches real per-token decode workload) test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 96, 64, 128, 8, 1, 1)); // Nemotron-3-Puzzle decode (scan path, unused warp in last block) test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 96, 64, 16, 8, 300, 2, false, /*K=*/1, /*weak_decay=*/true)); // d_state=96 SSD multi-chunk (partial 2nd chunk, 2 seqs) test_cases.emplace_back(new test_ssm_scan_rollback(GGML_TYPE_F32, 96, 64, 16, 8, 8, 2, /*K=*/3)); // d_state=96 rollback snapshots match prefix states @@ -10247,6 +11674,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 16, 2, 128, 2)); // SSD multi-chunk, no tail (exercises the chunk-to-chunk state handoff) test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 16, 2, 128, 2, false, /*K=*/1, /*weak_decay=*/true)); // SSD multi-chunk, carried state not numerically negligible + // ssm_scan + cache cpy fusion + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 4, 1, 4)); + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 1, 1, 4)); // n_seq_tokens < K + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 8, 1, 3)); // n_seq_tokens > K + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 64, 64, 112, 2, 4, 1, 4)); // Zamba2 + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 96, 64, 16, 2, 4, 1, 4)); + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 256, 64, 8, 2, 4, 1, 4)); + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 1, 1, 1)); // K == 1, final state only + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 4, 1, 1)); + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 300, 1, 1)); // K == 1, SSD path over two chunks + test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 1, 1)); test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 1)); test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 4)); @@ -10298,6 +11736,36 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 8192, 1, 8192)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 1024, 7, 1024)); // many rows + // src1 precision: few-row paths, aligned tiles, partial tiles (K and output), broadcast batches + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q8_0}) { + // Q8_0 requires K to be a multiple of its block size (32) + const int64_t k_partial = type_a == GGML_TYPE_Q8_0 ? 96 : 100; + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 1, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 8, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 32, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 128, 512, 512)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 67, 33, k_partial)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 32, 256, {2, 3}, {2, 1})); + } + + // src1 precision: same shapes with GGML_PREC_F32, which the backends must not round down to F16 or TF32 + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 1, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 8, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 32, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 128, 512, 512)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 67, 33, 100)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 32, 256, {2, 3}, {2, 1})); + } + + // src0 range: an F32 src0 must not be downcast to F16 when src1 requests BF16 + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 1, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 8, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 32, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 128, 512, 512)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 67, 33, 100)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 32, 256, {2, 3}, {2, 1})); + // FP4 activation precision (default = native W4A4, src1 GGML_PREC_Q8 = W4A8) test_cases.emplace_back(new test_mul_mat_w4a8(GGML_TYPE_NVFP4, GGML_TYPE_F32, 32, 1, 256)); test_cases.emplace_back(new test_mul_mat_w4a8(GGML_TYPE_NVFP4, GGML_TYPE_F32, 32, 32, 256)); @@ -10314,6 +11782,26 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_id_w4a8(GGML_TYPE_MXFP4, GGML_TYPE_F32, 8, 2, false, 32, 32, 256)); test_cases.emplace_back(new test_mul_mat_id_w4a4(GGML_TYPE_MXFP4, GGML_TYPE_F32, 8, 2, false, 32, 32, 256)); + // small K at batch > 8, e.g. LoRA B matrices with K = rank + for (ggml_type type_a : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { + for (int64_t k : {16, 32, 48}) { + for (int64_t n : {9, 33, 128}) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 100, n, k, {1, 1}, {1, 1})); + } + } + } + +#if 0 + // > 2^31-element dst (8.9 GB), about 45 GB peak memory. Too large to be enabled by default. + // kernel_mul_mm dst offset of the last batch is past INT32_MAX: rows 0-31 use full tiles, rows 32-39 use edge tiles. + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1048576, 40, 64, {1, 1}, {53, 1})); + + // kernel_mul_mm_id: the index of tokens 32768-33791 does not fit in 16 bits (about 0.3 GB peak memory). + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 64, 33792, 64)); + // kernel_mul_mm_id: > 2^31-element dst (8.7 GB), about 45 GB peak memory. Tokens 32514+ have a dst offset past INT32_MAX, tokens 32768+ also need a 32-bit token index. + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 16, 8, false, 8256, 32832, 64)); +#endif + #if 0 // > 4GB A matrix. Too slow to be enabled by default. test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 900000, 3, 2592, {1, 1}, {1, 1})); @@ -10370,6 +11858,28 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 512, 2048, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 509, 2051, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 509, 2051, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_BF16, 16, 4, 3, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_BF16, 16, 4, 256, {1, 1}, {1, 1})); + // FP8 views with an odd byte offset must use the cuBLAS fallback instead of MMVF. + for (int n : {1, 2}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 2, n, 64, {1, 1}, {1, 1}, {0, 1, 2, 3}, 66, 1, false, 0, 0, 1)); + } + // Native FP8 cuBLASLt requires 16-byte alignment, including each batch pointer. + for (size_t offset_a : {1, 2, 4, 16}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 16, 16, 64, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 17, 0, offset_a)); + } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 16, 16, 64, {2, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 17, 4)); + + // testing very large N + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 16, 2105344, 32, {1, 1}, {1, 1})); + +#if 0 + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 16, 8400000, 32, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 16, 8400000, 32, {1, 1}, {1, 1})); + // same shapes with a repeat in dim 3 + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 16, 8400000, 32, {1, 1}, {2, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 16, 8400000, 32, {1, 1}, {2, 1})); +#endif test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 31, 509, 2051, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1})); @@ -10381,8 +11891,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, m, n, k, {1, 1}, {1, 1})); } test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 64, n, 256, {3, 2}, {2, 1})); - for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, - GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}) { + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, + GGML_TYPE_Q8_0, GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_MXFP4, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, + GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_TQ2_0, + GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, + GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 48, n, 2560, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, n, 1024, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 3000, n, 512, {1, 1}, {1, 1})); @@ -10394,7 +11907,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } for (ggml_type type_a : {GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q5_K, GGML_TYPE_F16}) { - for (mul_mat_add_mode mode : {MUL_MAT_ADD_MM_RES, MUL_MAT_ADD_RES_MM, MUL_MAT_ADD_ROW, MUL_MAT_ADD_RES_INPLACE}) { + for (mul_mat_add_mode mode : {MUL_MAT_ADD_MM_RES, MUL_MAT_ADD_RES_MM, MUL_MAT_ADD_ROW, MUL_MAT_ADD_RES_INPLACE, MUL_MAT_ADD_MM_MM}) { test_cases.emplace_back(new test_mul_mat_add(type_a, 1000, n, 1024, mode)); } test_cases.emplace_back(new test_mul_mat_add(type_a, 2048, n, 2048, MUL_MAT_ADD_B_INPLACE)); @@ -10405,10 +11918,32 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, n, 1024, {1, 1}, {1, 1}, {0, 1, 2, 3}, 1280)); } // a src1 with a nonzero mean, for the zero points and mins of quantized src0 types - for (ggml_type type_a : {GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K}) { + for (ggml_type type_a : {GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K}) { test_cases.emplace_back(new test_mul_mat_pos(type_a, GGML_TYPE_F32, 1000, n, 1024, {1, 1}, {1, 1})); } } + // CUDA few-row MMQ: 1-3 column groups, 32-row tiles (6170 rows), K = 18 units (uneven over the warps) and 9 units (one unit per ring stage) + for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, + GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) { + for (int64_t k : {4608, 2304}) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, 8, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 6170, 13, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, 21, k, {1, 1}, {1, 1})); + } + } + // permuted and broadcast src0/src1 for the 2 byte aligned types + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 64, 13, 2304, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 63, 5, 2304, {3, 2}, {2, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q3_K, GGML_TYPE_F32, 64, 13, 4608, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_IQ3_S, GGML_TYPE_F32, 63, 5, 4608, {3, 2}, {2, 2})); + // partial column group, broadcast, padded rows, src1 in [0, 1], src0 in a weights buffer + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 5, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 64, 13, 4608, {3, 2}, {2, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 5120)); + test_cases.emplace_back(new test_mul_mat_pos(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat_pos(GGML_TYPE_Q5_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 6170, 13, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true)); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 1000, 21, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true)); #if 0 { @@ -10466,6 +12001,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 2, 1, 3})); test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 1, 3, 2})); test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); + + // test cases with permutation and a broadcast of a in dim 3 (e.g. a weight against a per-stream attention output) + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {1, 2}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, k, {1, 1}, {1, 2}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, k, {1, 1}, {4, 2}, {0, 2, 1, 3})); } // test cases with large ne00/ne10 to cover stream-k fixup @@ -10478,6 +12018,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } + // BF16 matrices of few rows and a long K at small batch sizes (MMVF instead of MMF) + for (int n : {2, 3, 4, 5, 8}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 320, n, 10240, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 512, n, 2560, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 128, n, 2560, {1, 1}, {1, 1})); + } + // BF16 is absent from base_types: add the 3 standard non-contig permutations explicitly test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 2, 1, 3})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); @@ -10557,9 +12104,26 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // the batches of a are padded, nb[2] is not a multiple of nb[1] test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 8, 1, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 16, 16)); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 8, 16, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 16, 16)); + // Ragged expert slices, strided routing IDs and broadcast activations. + for (ggml_type type : {GGML_TYPE_Q8_0, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K}) { + for (int n : {3, 17, 33}) { + for (bool broadcast : {false, true}) { + test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 37, 2, broadcast, 65, n, 256)); + } + test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 37, 2, false, 65, n, 256, 0, false, true)); + } + } + + // Both experts receive every token + for (ggml_type type : {GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K}) { + for (int n : {64, 65}) { + test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 2, 2, false, 33, n, 512)); + } + } + // as is a view whose experts are strided by more rows than it uses - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 8, 1, 64, 1.0f, 64)); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 4, 2, false, 8, 16, 64, 1.0f, 64)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 8, 1, 64, 64)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 4, 2, false, 8, 16, 64, 64)); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 576, 512, 576, {1,1}, {1,1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1, 2048, 8192, {1, 1}, {1, 1})); @@ -10663,6 +12227,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 28, 10, false, 1024, 1, 4096)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 128, 8, false, 1024, 1, 2048)); + // TODO: disabled until CUDA backend support is added [TAG_MUL_MAT_ID_DUPLICATE_ID] +#if 0 + // repeated expert ids within a row + for (ggml_type ta : {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { + for (int n : {9, 16, 33, 64}) { + test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 28, 10, false, 1024, n, 256)); + } + } + + for (ggml_type ta : {GGML_TYPE_F16, GGML_TYPE_Q4_0}) { + test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 1025, 10, false, 64, 33, 256, 0, true)); + test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 2050, 10, true, 64, 33, 256)); + } +#endif + for (ggml_type type_a : all_types) { test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a))); } @@ -10673,12 +12252,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 4, false, 16, 10, 256)); } - // test src1 f16 overflow - for (int n : {16, 32, 64}) { - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 4, false, 4096, n, 2048, 1e5f)); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8, 2, false, 512, n, 256, 1e5f)); - } - for (ggml_type type_a : base_types) { for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) { for (int n_mats : {4, 8}) { @@ -10711,6 +12284,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } + // skipped slots (negative ids) + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_K, GGML_TYPE_Q8_0, GGML_TYPE_MXFP4}) { + for (int n_used : {1, 2, 4}) { + for (bool b : {false, true}) { + for (int n : {1, 4, 33, 129}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 8, n_used, b, 512, n, 256, 0, false, true)); + } + } + } + } + // n_used == n_mats, so every expert can be skipped + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 4, 4, false, 512, 64, 256, 0, false, true)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, 1, 0, false, true)); + test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3, false, true)); + for (int bs : {1, 4, 512}) { for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_K}) { for (ggml_type type_b : {GGML_TYPE_F32}) { @@ -10758,6 +12346,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { for (int n_embd : {32, 129}) { for (int n_token : {1, 32, 129}) { test_cases.emplace_back(new test_add_id(type_a, type_b, n_embd, n_mats, n_used, n_token)); + test_cases.emplace_back(new test_add_id(type_a, type_b, n_embd, n_mats, n_used, n_token, true)); } } } @@ -10772,6 +12361,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_sin (type)); test_cases.emplace_back(new test_cos (type)); test_cases.emplace_back(new test_clamp (type)); + test_cases.emplace_back(new test_clamp (type, {10, 5, 4, 3}, -0.5f, 0.5f, 1)); test_cases.emplace_back(new test_leaky_relu(type)); test_cases.emplace_back(new test_floor (type)); test_cases.emplace_back(new test_ceil (type)); @@ -10801,10 +12391,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_trunc (type, {1024, 1024, 1, 1})); } - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 1, 1}, 5)); - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 1}, 5)); - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 2}, 5)); - #if 0 std::uniform_int_distribution<> dist_ne1(1, 50); int exponent = 1; @@ -10969,13 +12555,24 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true, 32)); } + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION}) { + for (bool truncate : {false, true}) { + for (bool fw : {false, true}) { + test_cases.emplace_back(new test_rope(type, {128, 2, 3, 1}, 128, mode, 8192, 1.0f/256, 1.0f, 1.0f, false, 0, fw, false, 0, 4096, 1000000.0f, 32.0f, truncate)); + } + test_cases.emplace_back(new test_rope(type, {128, 2, 3, 1}, 128, mode, 8192, 1.0f/256, 1.0f, 1.0f, true, 1, true, true, 0, 4096, 1000000.0f, 32.0f, truncate)); + } + } + } + // Real-model RoPE: F32 forward, packed Q, 512-token prefill. test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global - for (int v : { 0, 1, 2, 3 }) { + for (int v : { 0, 1, 2, 3, 16 }) { for (int dim : { 0, 1, 2, 3, }) { test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v)); test_cases.emplace_back(new test_concat(GGML_TYPE_F16, {11, 12, 13, 14}, 7, dim, v)); @@ -11002,6 +12599,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_concat(GGML_TYPE_Q8_0, {8192, 2, 1, 1}, dim == 0 ? 4096 : 1, dim, 0)); } + // GDN conv_input + for (int64_t ne_b_d : { 31, 32, 33, 64 }) { + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 10240, 1, 1}, ne_b_d, 0, 16)); + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {1, 37, 1, 1}, ne_b_d, 0, 16)); + } + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {64, 37, 1, 1}, 32, 0, 16)); + // Empty src0 + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {0, 64, 1, 1}, 33, 0, 16)); + for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) { for (uint32_t i = 4; i <= 1024*1024; i *= 2) { test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i-1, 1, 1, 1})); @@ -11025,6 +12631,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {n, 2, 1, 3}, k, true)); } } + for (int k : {1, 10, 40}) { + test_cases.emplace_back(new test_top_k_inf({4096, 2, 1, 1}, k)); + test_cases.emplace_back(new test_top_k_inf({248320, 1, 1, 1}, k)); + } + // a selection of the indexer of sparse attention, with more ties at the threshold than slots + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {1088, 2048, 1, 1}, 512, true)); + for (int i = 0; i < 20; ++i) { for (int k : {1, 2, 3, 7, 15, 100, 500, 1023, 9999}) { if (k <= 1<<i) { @@ -11067,6 +12680,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, { 8192, 2, 1, 1 }, 2051, true)); test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, { 33024, 4, 1, 1 }, 2051, true)); + // rows that CUDA processes in several chunks (bitonic, radix) + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, { 500, 40000, 1, 1 }, 16)); + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, { 1100, 33000, 1, 1 }, 16)); + // qwen4exp QSA indexer top-k fusion (get_rows + f16 mask + top_k) test_cases.emplace_back(new test_topk_qsa(512, 2048, 1, 1, 1500)); test_cases.emplace_back(new test_topk_qsa(512, 2048, 2, 1, 1500)); @@ -11088,6 +12705,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {2, 5, 7, 11}, {5, 7, 11, 13}, (ggml_scale_mode)(mode | GGML_SCALE_FLAG_ALIGN_CORNERS))); test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {1, 4, 3, 2}, {2, 8, 3, 2}, (ggml_scale_mode)(mode | GGML_SCALE_FLAG_ALIGN_CORNERS))); test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {4, 1, 3, 2}, {1, 1, 3, 2}, (ggml_scale_mode)(mode | GGML_SCALE_FLAG_ALIGN_CORNERS))); + test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {64, 48, 48, 1}, {156, 104, 64, 1}, (ggml_scale_mode)(mode | GGML_SCALE_FLAG_ALIGN_CORNERS), {2, 0, 1, 3})); } test_cases.emplace_back(new test_sum()); @@ -11104,10 +12722,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false)); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true)); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, true)); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 1, 7, 3, 2 })); + for (int64_t n : {1023, 1024, 1025}) { + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { n, 5, 1, 1 })); + } + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 17, 65537, 1, 1 })); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 512, 1, 1, 1 })); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 281, 1536, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }, { 1, 0, 2, 3 })); // sum dst not-contiguous + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 10, 5, 4, 3 }, { 1, 0, 2, 3 })); // non-contiguous rows, 4D + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 11, 5, 6, 3 }, { 2, 0, 1, 3 })); // rotated dims, nb[0] != sizeof(float) + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 0, 0, 0}, true)); // sliced view (gaps) + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {1, 0, 2, 3}, true)); // sliced + non-contiguous rows test_cases.emplace_back(new test_sum_rows()); test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false)); test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true)); @@ -11118,8 +12747,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 33, 1, 1, 1 })); test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 33, 1024, 1, 1 })); test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 33, 256, 1, 1 })); + test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 17, 65537, 1, 1 })); test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {64, 64, 320, 1})); test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {9, 9, 1280, 1})); + test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {3, 1, 1, 65537}, 1)); test_cases.emplace_back(new test_group_norm_mul_add(GGML_TYPE_F32, {64, 64, 320, 1})); test_cases.emplace_back(new test_group_norm_mul_add(GGML_TYPE_F32, {9, 9, 1280, 1})); test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 1, 1}, {256, 16, 1, 1}, -1)); @@ -11145,12 +12776,26 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {100, 1, 1, 1}, 100, 0, false)); test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {100, 1, 1, 1}, 0, 100, false)); test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {100, 100, 1, 1}, 50, 50, false)); + // more than 65535 rows or slices, beyond the CUDA grid.y/grid.z limit + test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {4, 70000, 1, 1}, 1, 1, false)); + test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {4, 70000, 1, 1}, 1, 1, true)); + test_cases.emplace_back(new test_pad_ext(GGML_TYPE_F32, {4, 2, 300, 300}, 1, 1, 0, 0, 0, 0, 0, 0, 0, false)); test_cases.emplace_back(new test_pad_reflect_1d()); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {1, 3, 2, 2}, 0, 0)); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {5, 3, 2, 2}, 4, 1, true)); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {281, 128, 1, 1}, 2, 2)); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {257, 7, 1, 1}, 0, 256)); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {2, 16777217, 1, 1}, 1, 1)); test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1})); test_cases.emplace_back(new test_roll()); test_cases.emplace_back(new test_roll(3, -2, 1, -1, true)); test_cases.emplace_back(new test_arange()); + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 3.0f, 3.5f, 1.0f)); + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, -10.0f, 10.0f, 0.3f)); + for (float n : {255.0f, 256.0f, 257.0f, 1023.0f, 1024.0f, 1025.0f}) { + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, -1.0f, -1.0f + n * 0.25f, 0.25f)); + } test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, 1048576.0f, 1.0f)); test_cases.emplace_back(new test_timestep_embedding()); test_cases.emplace_back(new test_leaky_relu()); @@ -11173,13 +12818,19 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_xielu()); test_cases.emplace_back(new test_xielu(GGML_TYPE_F16)); + test_cases.emplace_back(new test_xielu(GGML_TYPE_BF16)); test_cases.emplace_back(new test_xielu(GGML_TYPE_F32, { 512, 16, 1, 1 })); test_cases.emplace_back(new test_xielu(GGML_TYPE_F16, { 512, 16, 1, 1 })); + test_cases.emplace_back(new test_xielu(GGML_TYPE_BF16, { 512, 16, 1, 1 })); test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER)); test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG)); test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER)); test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG)); + test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER)); + test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER_DIAG)); + test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_UPPER)); + test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_UPPER_DIAG)); test_cases.emplace_back(new test_fill(0.0f)); test_cases.emplace_back(new test_fill(2.0f, GGML_TYPE_F32, { 303, 207, 11, 3 })); @@ -11217,6 +12868,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 1 }, { 32, 128, 4, 1 })); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 200, 64, 4, 4 })); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 384, 64, 4, 4 })); + // n % 32 != 0, output prefilled with NaN + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 34, 34, 1, 1 }, { 256, 34, 1, 1 }, true)); + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 79, 79, 5, 3 }, { 417, 79, 5, 3 }, true)); for (int tfrm : {0, 1, 2}) { for (bool circular : {false, true}) { @@ -11238,6 +12892,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } + // KV-head counts that do not divide the device count + for (int hs : { 128, 256, 512, }) { + for (int nh : { 1, 2, 6, }) { + for (int nr2 : { 4, 8, }) { + for (int nb : { 75, 1024, }) { + test_cases.emplace_back(new test_flash_attn_ext(hs, hs, nh, {nr2, 1}, 1024, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + } + } + } + } + for (int hsk : { 40, 64, 72, 80, 96, 128, 192, 256, 320, 512, 576 }) { for (int hsv : { 40, 64, 72, 80, 96, 128, 192, 256, 512 }) { if (hsk != 96 && hsk != 192 && hsk != 320 && hsk != 576 && hsk != hsv) continue; @@ -11266,8 +12931,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { for (int kv : { 113, 512, 1024, }) { if (nr2 != 1 && kv != 512) continue; for (int nb : { 1, 3, 32, 75, }) { - for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_DEFAULT}) { - if (hsk != 128 && prec == GGML_PREC_DEFAULT) continue; + for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_BF16, GGML_PREC_UNDEFINED}) { + if (hsk != 128 && prec != GGML_PREC_F32) continue; for (ggml_type type_KV : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { if (type_KV != GGML_TYPE_F16 && hsk != 64 && hsk != 72) continue; // DeepSeek MLA: the V cache is a sub-view of the K cache @@ -11314,6 +12979,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0)); test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {1, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F8_E4M3)); + + for (int hs : {64, 128, 256}) { + for (ggml_type type_K : {GGML_TYPE_F16, GGML_TYPE_F8_E4M3}) { + test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 4, {1, 1}, 256, 1, true, false, 0, 0, GGML_PREC_F32, type_K, GGML_TYPE_F8_E4M3)); + } + } // q8_0 KV cases: decode and prompt batches, KV pad, permuted KV, feature flags, and long context test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); @@ -11323,10 +12999,24 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1025, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3})); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + // Dense Q8 KV pre-dequantization: both query thresholds and a partial final tile. + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 1, {8, 1}, 4096, 1024, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 1, {8, 1}, 8192, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 1, {8, 1}, 8193, 513, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + + // Grouped decode uses Q's head stride, which is not eight-element aligned here. + for (int nb : {1, 3}) { + for (ggml_type type_KV : {GGML_TYPE_F16, GGML_TYPE_Q8_0}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, {2, 1}, 512, nb, true, false, 0, 0, GGML_PREC_F32, type_KV, type_KV, {0, 2, 1, 3}, false, false, 0, 4)); + } + } + // MLA shape: the V cache is a sub-view of the K cache, with quantized KV test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 4096, 24, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {8, 1}, 4096, 24, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false)); // Sparse mask hint: supported decode/prefill layouts and dense fallbacks. test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); @@ -11339,6 +13029,18 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 4, true, false, 8.0f, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); + // flash attention over a list of K/V rows per slice (unified KV cache) + for (int64_t nb : {1, 3, 8, 64, 512}) { + for (int64_t n_slices : {1, 4}) { + test_cases.emplace_back(new test_flash_attn_ext_kv_rows(256, 2, 8, 4096, 1024, nb, n_slices)); + test_cases.emplace_back(new test_flash_attn_ext_kv_rows(128, 8, 4, 4096, 512, nb, n_slices)); + test_cases.emplace_back(new test_flash_attn_ext_kv_rows(128, 4, 1, 1000, 256, nb, n_slices)); + test_cases.emplace_back(new test_flash_attn_ext_kv_rows( 64, 4, 2, 3000, 768, nb, n_slices, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext_kv_rows(512, 2, 8, 4096, 1024, nb, n_slices)); + test_cases.emplace_back(new test_flash_attn_ext_kv_rows(256, 8, 2, 4096, 1280, nb, n_slices)); + } + } + // sparse mask with large batch size test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); @@ -11368,6 +13070,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // KV not a multiple of the compaction workgroup size. test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 5003, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); + // Sparse gather: native block sizes, padded slots, and head/batch strides. + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_IQ4_NL}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 96, 2, {8, 2}, 5003, 3, true, false, 0, 0, GGML_PREC_F32, type, type, {0, 1, 2, 3}, true, false, 257)); + } + test_cases.emplace_back(new test_flash_attn_ext(128, 96, 2, {8, 2}, 5003, 3, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_F16, {0, 2, 1, 3}, true, false, 257)); + test_cases.emplace_back(new test_flash_attn_ext(128, 96, 2, {8, 2}, 5003, 3, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, {0, 2, 1, 3}, true, false, 257)); + test_cases.emplace_back(new test_flash_attn_ext(128, 96, 2, {8, 2}, 5003, 3, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_F32, {0, 1, 2, 3}, true, false, 257)); + test_cases.emplace_back(new test_flash_attn_ext(128, 96, 2, {8, 2}, 5003, 3, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F32, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 257)); + // more V-is-sub-view-of-K cases: other head shapes, and full views with equal head sizes test_cases.emplace_back(new test_flash_attn_ext(320, 256, 1, {32, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, {8, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); @@ -11396,6 +13107,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {8, 1}, 1024, 320, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(96, 96, 8, {4, 1}, 512, 256, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {8, 1}, 512, 259, true, false, 8.0f, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {8, 1}, 512, 288, true, false, 0, 30.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {8, 1}, 1024, 320, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {8, 1}, 512, 259, true, false, 8.0f, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16, {0, 2, 1, 3}, false)); + // FLASH_ATTN_EXT MMA: non-pow2 head size and MLA K/V view. test_cases.emplace_back(new test_flash_attn_ext(192, 128, 8, {8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); @@ -11406,11 +13124,18 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {2, 1}, 1024, 32, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(512, 512, 4, {2, 1}, 1024, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // BF16 K/V for the other head sizes of the Metal tensor API kernel + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {12, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16)); + test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, { 8, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16)); + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16, {0, 1, 2, 3}, true, true)); + // FLASH_ATTN_EXT: large logits test_cases.emplace_back(new test_flash_attn_ext_large_logits( 64, 64, 16, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext_large_logits(128, 128, 8, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256, 4, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256, 4, {4, 1}, 1024, 75, true, false, 0, 10.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256, 4, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16)); test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3})); test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1})); @@ -11420,6 +13145,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_opt_step_adamw(GGML_TYPE_F32, {10, 5, 4, 3})); test_cases.emplace_back(new test_opt_step_sgd(GGML_TYPE_F32, {10, 5, 4, 3})); + for (bool with_bias : {false, true}) { + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 32, 256, + false, 1, 1, false, with_bias, false, true, {1, 1})); + } + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 32, 256, + false, 1, 1, false, false, false, true, {2, 1})); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 31, 257, + false, 1, 1, false, false, false, true, {1, 1})); + for (ggml_type type : base_types) { for (bool with_gate : {false, true}) { for (bool use_id : {false, true}) { @@ -11439,18 +13173,23 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { continue; } for (bool with_lane_scale : {false, true}) { - if (with_lane_scale && type != GGML_TYPE_NVFP4) { + if (with_lane_scale && type != GGML_TYPE_NVFP4 && type != GGML_TYPE_F8_E4M3) { continue; } - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, 1, 32, 256, use_id, 16, 8, b, with_bias, with_gate, with_lane_scale)); - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, 1, 32, 256, use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1})); // multi-token batches (spec decoding) for (int64_t m_batch : { 2, 4, 8 }) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, m_batch, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, m_batch, 32, 256, use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1})); } + // the lane scale does a GET_ROWS on the ids, which does not accept -1 + if (use_id && !with_lane_scale) { + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, 8, 32, 256, + use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1}, true)); + } } } } @@ -11460,16 +13199,35 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } for (bool b : {false, true}) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_IQ2_S, GGML_GLU_OP_SWIGLU_CLAMP, 1, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_IQ2_S, GGML_GLU_OP_SWIGLU_CLAMP, 1, 32, 256, true, 16, 8, b, false, true, false)); } + // Gate and up on the SYCL XMX kernels: rows long enough for 2D block loads, a last tile of 8 rows, and + // columns on both sides of the 8 and 16 column tiles + for (ggml_type type : {GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, + GGML_TYPE_Q8_0}) { + for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) { + for (int64_t m_batch : {1, 2, 3, 5, 8, 9, 16, 17}) { + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, m_batch, 1000, 1024, + false, 16, 8, false, false, true, false, { 1, 1 })); + } + } + } + + for (int64_t k : {96, 896, 1024, 2880}) { + for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) { + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q8_0, glu_op, 1, 32, k, + false, 16, 8, false, false, true, false, {1, 1})); + } + } + // Fused row-pair coverage: minimum rows, an even pair, and an odd tail. // TODO: the max_nmse_err() for these cases is not estimated correctly causing sporadic false failures. //for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) { // for (int64_t m_batch : { 2, 3, 4 }) { // for (int64_t rows : { 1, 2, 3 }) { - // test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, glu_op, m_batch, rows, 256, + // test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, glu_op, m_batch, rows, 256, // false, 16, 8, false, false, true, false, { 1, 1 })); // } // } @@ -11477,10 +13235,43 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // Both sides of the same row-count boundary as above, on the fused path. for (int64_t rows : {6271, 6272, 6273}) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 2, rows, 256, + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 2, rows, 256, false, 16, 8, false, false, true, false, { 1, 1 })); } + // Fused gate/up + GLU on the MMQ path, only used if its tiling is efficient: ~1280 tiles per case, one case per type and GLU op. + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 17, 81920, 256, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q5_K, GGML_GLU_OP_GEGLU, 150, 40960, 256, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q6_K, GGML_GLU_OP_SWIGLU_OAI, 150, 40960, 256, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q8_0, GGML_GLU_OP_SWIGLU_CLAMP, 150, 6912, 256, false, 1, 1, false, false, true, false, { 2, 3 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_IQ4_XS, GGML_GLU_OP_SWIGLU, 150, 40960, 512, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 6912, 256, false, 1, 1, false, false, true, true, { 2, 3 })); + // NVFP4 weight scale without gate/up: MMQ applies it in the write-back, the trailing bias add stays a separate node + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, false, true)); + // Same pattern with broadcast input, 4 of 16 experts used, 8-bit activations (W4A8), and Q4_K, which keeps the scale as a separate node + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, true, false, false, true)); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 4, false, false, false, true)); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, false, true)); + // Few columns and tiles with a long K: split-K through the stream-k fixup, the scale is applied to the partial sums + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 11, 64, 2048, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 11, 64, 2048, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 150, 2560, 256, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, false, true)); + // The same with experts (~1280 tiles each): 17 tokens with broadcast input (one MMA tile per warp), 2048 tokens without (two, several column tiles per expert). + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 17, 5120, 256, true, 16, 8, true)); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q8_0, GGML_GLU_OP_SWIGLU, 2048, 128, 256, true, 40, 8, false)); + // NVFP4 with experts, without and with per-expert weight scales. + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false)); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 17, 5120, 256, true, 16, 8, true, false, true, true)); + // NVFP4 with 8-bit activations requested (W4A16 checkpoints), without and with weight scales. + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false)); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 40960, 256, false, 1, 1, false, false, true, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, true, true)); + // K = 2048: few-row MMQ, where enabled, takes the gate and up matmuls instead of the fused MMQ path + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 9, 128, 2048, false, 1, 1, false, false, true, false, { 1, 1 })); + for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) { for (bool with_norm : {false, true}) { for (bool bias_probs : {false, true}) { @@ -11495,15 +13286,41 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_topk_moe({160, 4, 1, 1}, 160, with_norm, bias_probs, gate, scale_w)); test_cases.emplace_back(new test_topk_moe({256, 22, 1, 1}, 6, with_norm, bias_probs, gate, scale_w)); // Used by DeepSeek-V4 test_cases.emplace_back(new test_topk_moe({288, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); // Used by StepFun 3.7 + test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, with_norm, bias_probs, gate, scale_w)); // Used by DeepSeek-V4.1 + test_cases.emplace_back(new test_topk_moe({384, 8, 1, 1}, 6, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({384, 22, 1, 1}, 6, with_norm, bias_probs, gate, scale_w)); // rows at and just past the limit where one block still covers all rows test_cases.emplace_back(new test_topk_moe({32, 8, 1, 1}, 4, with_norm, bias_probs, gate, scale_w)); test_cases.emplace_back(new test_topk_moe({32, 8, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); test_cases.emplace_back(new test_topk_moe({32, 9, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); + + // grouped experts + test_cases.emplace_back(new test_topk_moe({128, 1, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 16)); + test_cases.emplace_back(new test_topk_moe({256, 22, 1, 1}, 6, with_norm, bias_probs, gate, scale_w, 32)); + test_cases.emplace_back(new test_topk_moe({256, 7, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 8)); // DeepSeek V3 + test_cases.emplace_back(new test_topk_moe({128, 5, 1, 1}, 6, with_norm, bias_probs, gate, scale_w, 8)); + test_cases.emplace_back(new test_topk_moe({512, 3, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 4)); } } } } + // the sum is adjusted with an additive bias instead of a lower clamp (DeepSeek-V4.1) + for (bool bias_probs : {false, true}) { + for (float scale_w : {0.0f, 2.0f}) { + test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, bias_probs, GATING_FUNC_SOFTMAX, scale_w, 1, true)); + test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, bias_probs, GATING_FUNC_SOFTMAX, scale_w, 1, true)); + test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, bias_probs, GATING_FUNC_SQRT_SOFTPLUS, scale_w, 1, true)); + test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, bias_probs, GATING_FUNC_SQRT_SOFTPLUS, scale_w, 1, true)); + } + } + + // sigmoid scoring always ranks a selection bias (GLM, DeepSeek-V2/V3.2) + for (float scale_w : {0.0f, 2.0f}) { + test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, true, GATING_FUNC_SIGMOID, scale_w, 1, true)); + test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, true, GATING_FUNC_SIGMOID, scale_w, 1, true)); + } + // Cover the supported boundaries, common k = 8 shapes, interleaved views and adds, and k = 16 fallback. test_cases.emplace_back(new test_moe_reduce(63, 2, 17)); test_cases.emplace_back(new test_moe_reduce(2048, 8, 128)); @@ -11543,6 +13360,18 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true)); + // MMA-eligible GDN shapes: whole heads, value slices, tails, and the HIP prefill threshold. + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 1, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 3, 4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 1, 4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 257, 1, 3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 513, 1, 3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 513, 2, 2, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1, 2)); + // Keep the WebGPU dispatch below 65535 workgroups (32 per token for this shape). + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2047, 2, 3)); + // K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first // (slot 0 = final state, slot s = state s tokens back). // exact-match cases (K == n_seq_tokens): @@ -11552,9 +13381,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 128, 4, 1, 1, false, false, /*K=*/4)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, false, true, /*K=*/4)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 32, 4, 2, 2, false, true, /*K=*/4)); - // overflow: n_tokens > K — only the last K snapshots kept. - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 8, 1, 1, false, false, /*K=*/3)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 16, 2, 1, false, false, /*K=*/4)); // gdn + cache cpy fusion (K > 1) test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 2, 1, 2)); @@ -11601,6 +13427,33 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { std::vector<std::unique_ptr<test_case>> test_cases; + for (float n : {128.0f, 4096.0f, 32768.0f, 1048576.0f}) { + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, n, 1.0f)); + } + + // LoRA at prefill: B is [rank, n_out], so K = rank + for (int64_t k : {16, 32, 48, 64}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 2048, 512, k, {1, 1}, {1, 1})); + } + + // DSV4_HC_PRE: n_tokens 1 is decode, 128 and 512 are prompt batches + for (int64_t n_tokens : {1, 128, 512}) { + test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, n_tokens)); // DeepSeek-V4-Flash + test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, n_tokens, true)); // synthetic, no gated model uses this width + test_cases.emplace_back(new test_dsv4_hc_pre(2560, 4, n_tokens, true)); // Qwen4-Exp (Qwen3.8-Flash-Next) + } + + // DSV4_HC_POST: n_tokens 1 is decode, 128 and 512 are prompt batches + for (int64_t n_tokens : {1, 128, 512}) { + test_cases.emplace_back(new test_dsv4_hc_post(4096, n_tokens)); // DeepSeek-V4-Flash + test_cases.emplace_back(new test_dsv4_hc_post(7168, n_tokens)); // DeepSeek-V4-Pro + test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true, true)); // Qwen4-Exp, with the post gate chain + test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true)); // synthetic, Qwen4-Exp kernel without the gate chain + } + + // Qwen3.8-Flash-Next pooled indexer prefill. + test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_F32)); + test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_Q4_0)); // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands // note: same bytes either way, so a backend that indexes them differently shows it here for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { @@ -11610,6 +13463,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { } } + // GDN conv_input shapes + for (int64_t channels : {8192, 10240}) { + for (int64_t n_tokens : {512, 2048, 4096}) { + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, channels, 1, 1}, n_tokens, 0, 16)); + } + } + // CONT of a 0<->2 permute at DeepSeek-V4 lightning-indexer shapes: // indexer_kq is [n_kv, n_tokens, n_head=64] and gets ggml_cont(ggml_permute(.., 2,1,0,3)). for (int64_t n_kv : { 1024, 1280, 2048, 2304 }) { @@ -11621,6 +13481,35 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { GGML_TYPE_F32, {n_kv, 512, 64, 1}, false, {2, 1, 0, 3})); } + // POOL_2D max over MiniMax-M3 indexer block scores: + // sc is [n_ps, n_head=4, n_tokens] and gets ggml_pool_2d(.., MAX, blk=128, 1, blk=128, 1, 0, 0). + for (int64_t n_ps : { 8192, 32768 }) { + for (int64_t n_tokens : { 1, 512 }) { + test_cases.emplace_back(new test_pool2d( + GGML_OP_POOL_MAX, GGML_TYPE_F32, {n_ps, 4, n_tokens, 1}, 128, 1, 128, 1, 0, 0)); + } + } + + // RMS_NORM and fused variants at decode and prefill shapes + for (int64_t n_tokens : {1, 512}) { + test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {4096, n_tokens, 1, 1})); + test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {128, 32*n_tokens, 1, 1})); + test_cases.emplace_back(new test_rms_norm_mul_rope({4096, n_tokens, 1, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NORMAL, true, false)); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 32, n_tokens, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NEOX)); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 8, n_tokens, 1}, 1e-6f, false, true, true, GGML_ROPE_TYPE_NEOX)); + } + + // POOL_1D max over MoE down-projection amax shapes: + // amax is [1, n_expert, n_tokens] after reducing the whole intermediate dim. + for (int64_t n_ff : { 2048, 4096, 8192 }) { + for (int64_t n_expert : { 8, 16 }) { + for (int64_t n_tokens : { 1, 512 }) { + test_cases.emplace_back(new test_pool1d( + GGML_OP_POOL_MAX, GGML_TYPE_F32, {n_ff, n_expert, n_tokens, 1}, n_ff, n_ff, 0)); + } + } + } + // LEAKY_RELU at FFN activation width, for direct comparison with RELU for (int64_t n_tokens : {512, 2048}) { test_cases.emplace_back(new test_leaky_relu(GGML_TYPE_F32, { 17408, n_tokens, 1, 1 }, 0.1f)); @@ -11682,6 +13571,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { {1, 320, 8, 76, 52, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, {1, 1280, 8, 76, 52, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, {1, 320, 8, 152, 104, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + // Wan VAE pads spatial borders and temporal history before conv3d; operator padding is zero. + {1, 384, 3, 40, 28, 384, 3, 3, 3, 1, 1, 1, 0, 0, 0, 1, 1, 1}, + {1, 192, 4, 78, 54, 192, 3, 3, 3, 1, 1, 1, 0, 0, 0, 1, 1, 1}, + {1, 96, 6, 154, 106, 96, 3, 3, 3, 1, 1, 1, 0, 0, 0, 1, 1, 1}, + {1, 96, 6, 306, 210, 3, 3, 3, 3, 1, 1, 1, 0, 0, 0, 1, 1, 1}, + {1, 1024, 3, 40, 28, 1024, 3, 3, 3, 1, 1, 1, 0, 0, 0, 1, 1, 1}, + {1, 256, 6, 154, 106, 256, 3, 3, 3, 1, 1, 1, 0, 0, 0, 1, 1, 1}, + {1, 384, 3, 38, 26, 768, 3, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1}, + {1, 192, 2, 76, 52, 96, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1}, #if 0 // too slow on some devices {1, 1280, 8, 152, 104, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, @@ -11729,6 +13627,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {64, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {77, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + for (int64_t n : {256, 1024}) { + test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, n, 1, 1})); + } + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 10, 1, 1})); test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {1024, 10, 1, 1})); test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32000, 512, 1, 1})); @@ -11739,6 +13641,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 1, 1})); test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1})); + test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {64, 64, 320, 1})); + test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {9, 9, 1280, 1})); + test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {3, 1, 1, 65537}, 1)); + // SNAKE activation fusion at BigVGAN scale (T=7680 = 24 kHz x 320 ms, C=192) test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_F32, {7680, 192, 1, 1})); test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_F16, {7680, 192, 1, 1})); @@ -11756,6 +13662,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 2048, 128)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 2048, 256)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 512, 2048, 512)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 1, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 64, 1, 64)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 256, 1, 256)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 32, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 64, 2048, 64)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 2048, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 256, 2048, 256)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 512, 2048, 512)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 1024, 2048, 1024)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 4096, 2048, 4096)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 8192, 2048, 8192)); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 32, 64, 4, 4 })); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 2 }, { 32, 128, 4, 2 })); @@ -11789,7 +13706,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { - for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) { + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) { for (ggml_type type_b : {GGML_TYPE_F32}) { test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 128, 8, false, 768, bs, 2048)); test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 128, 8, false, 768, bs, 2048, 1)); @@ -11831,6 +13748,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // Same non-causal vision-encoder shape for the other head size padded in-kernel by the CUDA MMA kernels + test_cases.emplace_back(new test_flash_attn_ext(40, 40, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); // Sparse flash attention (n_kv_max hint) decode across KV depths. // Shapes: 576/512 DeepSeek MLA, 512/512 DeepSeek-V4/GLM-5.2, 256/256 gqa12 Qwen QSA. @@ -11839,6 +13758,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 512)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 2048)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 1, 2, 3}, true, false, 2048)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16, {0, 1, 2, 3}, true, false, 2048)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 0)); } @@ -11862,6 +13783,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 8192, 32, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 8192, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(96, 96, 2, {8, 1}, 8192, 128, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + for (int64_t kv : {8192, 32768}) { + for (int64_t nq : {128, 256, 512, 1024}) { + test_cases.emplace_back(new test_flash_attn_ext(192, 192, 2, {8, 1}, kv, nq, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + } + } + // MLA uses V as a view of the compressed K cache. + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 32768, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false, true)); + for (int64_t nq : {256, 512, 1024, 2048}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {8, 1}, 32768, nq, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + } + for (int64_t nq : {512, 1024}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 2, {16, 1}, 4096, nq, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + } + // q8_0 KV cases with long context (decode and prompt) test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); @@ -11923,6 +13859,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { test_cases.emplace_back(new test_col2im_1d(GGML_TYPE_F16, 16, 512, 2048, 8, 0)); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, {256, 256, 3, 1})); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, {512, 1, 1, 1})); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, {281, 1536, 1, 1})); for (int n_token : {1, 512}) { @@ -11990,6 +13928,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, 2048)); } } + // bitonic vs radix crossover + for (auto cols : {520, 1000, 2048, 3000, 4096}) { + for (auto nrows : {2, 16, 32, 48, 64, 128, 256, 512}) { + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, 16)); + } + } // backend sampler: one row of the vocab (llama-sampler.cpp top_k) for (auto k : {20, 40}) { test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {151936, 1, 1, 1}, k)); @@ -12062,6 +14006,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { } } + // top_k_mask, Flash-Next QSA shapes: 512 pools of 4 + a 3-cell tail, and DSA shapes: 2048 cells + for (int64_t n_kv : { 18432, 131072 }) { + for (int64_t nb : { 1, 2048 }) { + test_cases.emplace_back(new test_top_k_mask(GGML_TYPE_F16, n_kv, 4, n_kv/4, 512, 3, true, nb, 1, 1, true)); + test_cases.emplace_back(new test_top_k_mask(GGML_TYPE_F16, n_kv, 0, 0, 2048, 0, false, nb, 1, 1, true)); + } + } + // launch-overhead isolation: single L2_NORM launch vs batched siblings at the GDN // production shape (strided qkv views) -- perf-mode only, the eval list has its own // 2/4-wide coverage @@ -12551,7 +14503,7 @@ int main(int argc, char ** argv) { const char * backend_filter = nullptr; const char * params_filter = nullptr; const char * test_file_path = nullptr; - int parallel_workers = 1; + int parallel_workers = test_parallel_workers(1); for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "test") == 0) { diff --git a/ggml/tests/test-quantize-fns.cpp b/ggml/tests/test-quantize-fns.cpp index 1930a09b2..1542ab656 100644 --- a/ggml/tests/test-quantize-fns.cpp +++ b/ggml/tests/test-quantize-fns.cpp @@ -7,6 +7,7 @@ #include <assert.h> #include <algorithm> #include <cmath> +#include <limits> #include <math.h> #include <stdio.h> #include <string> @@ -36,7 +37,7 @@ static const char* RESULT_STR[] = {"ok", "FAILED"}; // Generate synthetic data static void generate_data(float offset, size_t n, float * dst, float amplitude = 2.0f) { for (size_t i = 0; i < n; i++) { - dst[i] = 0.1 + amplitude*cosf(i + offset); + dst[i] = 0.1 + amplitude*std::cos(i + offset); } } @@ -47,7 +48,7 @@ static float array_rmse(const float * a1, const float * a2, size_t n) { double diff = a1[i] - a2[i]; sum += diff * diff; } - return sqrtf(sum) / n; + return std::sqrt(sum) / n; } // Total quantization error on test data @@ -105,7 +106,7 @@ static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); const float dot_ref = dot_product(test_data1, test_data2, test_size); - return fabsf(result - dot_ref) / test_size; + return std::fabs(result - dot_ref) / test_size; } // nrc == 2: kernel computes a 2x2 dot product matrix @@ -124,7 +125,7 @@ static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type const float ref11 = dot_product(test_data3, test_data4, test_size); const auto err = [test_size](float val, float ref) { - const float e = fabsf(val - ref) / test_size; + const float e = std::fabs(val - ref) / test_size; return std::isfinite(e) ? e : INFINITY; }; @@ -143,7 +144,7 @@ static int test_vec_dot_f32(bool verbose) { float result = 0.0f; f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1); const float ref = dot_product(a.data(), b.data(), n); - const float error = fabsf(result - ref) / n; + const float error = std::fabs(result - ref) / n; const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR); num_failed += failed; @@ -155,6 +156,152 @@ static int test_vec_dot_f32(bool verbose) { return num_failed; } +// Positive finite E4M3 values in encoding order, from OCP FP8 rev. 1.1, section 5.1. +static const float F8_E4M3_VALUES[] = { + 0.0f, 0.001953125f, 0.00390625f, 0.005859375f, 0.0078125f, 0.009765625f, 0.01171875f, 0.013671875f, + 0.015625f, 0.017578125f, 0.01953125f, 0.021484375f, 0.0234375f, 0.025390625f, 0.02734375f, 0.029296875f, + 0.03125f, 0.03515625f, 0.0390625f, 0.04296875f, 0.046875f, 0.05078125f, 0.0546875f, 0.05859375f, + 0.0625f, 0.0703125f, 0.078125f, 0.0859375f, 0.09375f, 0.1015625f, 0.109375f, 0.1171875f, + 0.125f, 0.140625f, 0.15625f, 0.171875f, 0.1875f, 0.203125f, 0.21875f, 0.234375f, + 0.25f, 0.28125f, 0.3125f, 0.34375f, 0.375f, 0.40625f, 0.4375f, 0.46875f, + 0.5f, 0.5625f, 0.625f, 0.6875f, 0.75f, 0.8125f, 0.875f, 0.9375f, + 1.0f, 1.125f, 1.25f, 1.375f, 1.5f, 1.625f, 1.75f, 1.875f, + 2.0f, 2.25f, 2.5f, 2.75f, 3.0f, 3.25f, 3.5f, 3.75f, + 4.0f, 4.5f, 5.0f, 5.5f, 6.0f, 6.5f, 7.0f, 7.5f, + 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, + 16.0f, 18.0f, 20.0f, 22.0f, 24.0f, 26.0f, 28.0f, 30.0f, + 32.0f, 36.0f, 40.0f, 44.0f, 48.0f, 52.0f, 56.0f, 60.0f, + 64.0f, 72.0f, 80.0f, 88.0f, 96.0f, 104.0f, 112.0f, 120.0f, + 128.0f, 144.0f, 160.0f, 176.0f, 192.0f, 208.0f, 224.0f, 240.0f, + 256.0f, 288.0f, 320.0f, 352.0f, 384.0f, 416.0f, 448.0f, +}; + +static uint8_t fp32_to_f8_e4m3_ref(float value) { + if (std::isnan(value)) { + return 0x7F; + } + const uint8_t sign = std::signbit(value) ? 0x80 : 0; + const double magnitude = std::fabs((double) value); + if (magnitude >= F8_E4M3_VALUES[126]) { + return sign | 126; + } + int nearest = 0; + double distance = magnitude; + for (int i = 1; i < 127; ++i) { + const double candidate = std::fabs(magnitude - F8_E4M3_VALUES[i]); + if (candidate < distance || (candidate == distance && (i & 1) == 0)) { + nearest = i; + distance = candidate; + } + } + return sign | (uint8_t) nearest; +} + +static int test_f8_e4m3(bool verbose) { + static_assert(sizeof(ggml_fp8_e4m3_t) == 1); + + const auto * traits = ggml_get_type_traits(GGML_TYPE_F8_E4M3); + const auto * traits_cpu = ggml_get_type_traits_cpu(GGML_TYPE_F8_E4M3); + std::vector<uint8_t> encoded(256); + std::vector<uint8_t> roundtrip(256); + std::vector<float> decoded(256); + int num_failed = 0; + + const bool traits_failed = traits->blck_size != 1 || traits->type_size != sizeof(ggml_fp8_e4m3_t) || traits->is_quantized; + num_failed += traits_failed; + if (verbose || traits_failed) { + printf("f8_e4m3 scalar type traits: %s\n", RESULT_STR[traits_failed]); + } + + for (int i = 0; i < 256; ++i) { + encoded[i] = (uint8_t) i; + } + traits->to_float(encoded.data(), decoded.data(), decoded.size()); + traits_cpu->from_float(decoded.data(), roundtrip.data(), decoded.size()); + + for (int i = 0; i < 256; ++i) { + const bool is_nan = (i & 0x7F) == 0x7F; + const float expected = is_nan ? NAN : std::copysign(F8_E4M3_VALUES[i & 0x7F], i & 0x80 ? -1.0f : 1.0f); + const bool decode_failed = is_nan ? !std::isnan(decoded[i]) + : decoded[i] != expected || (expected == 0.0f && std::signbit(decoded[i]) != std::signbit(expected)); + const bool encode_failed = !is_nan && roundtrip[i] != encoded[i]; + if (decode_failed || encode_failed) { + num_failed++; + if (verbose) { + printf("f8_e4m3 code 0x%02x failed: decoded=%f expected=%f roundtrip=0x%02x\n", + i, decoded[i], expected, roundtrip[i]); + } + } + } + + std::vector<float> inputs; + for (float value : F8_E4M3_VALUES) { + inputs.push_back(value); + inputs.push_back(-value); + } + for (int i = 0; i < 126; ++i) { + const float midpoint = (F8_E4M3_VALUES[i] + F8_E4M3_VALUES[i + 1]) / 2.0f; + for (float value : {std::nextafter(midpoint, -INFINITY), midpoint, std::nextafter(midpoint, INFINITY)}) { + inputs.push_back(value); + inputs.push_back(-value); + } + } + for (float value : {std::numeric_limits<float>::denorm_min(), std::numeric_limits<float>::max(), + std::nextafter(448.0f, INFINITY), std::nextafter(464.0f, -INFINITY), 464.0f, + std::nextafter(464.0f, INFINITY), INFINITY, NAN}) { + inputs.push_back(value); + inputs.push_back(-value); + } + + std::vector<uint8_t> quantized(inputs.size()); + std::vector<uint8_t> quantized_ref(inputs.size()); + traits_cpu->from_float(inputs.data(), quantized.data(), inputs.size()); + traits->from_float_ref(inputs.data(), quantized_ref.data(), inputs.size()); + for (size_t i = 0; i < inputs.size(); ++i) { + const uint8_t expected = fp32_to_f8_e4m3_ref(inputs[i]); + if (quantized[i] != expected || quantized_ref[i] != expected) { + num_failed++; + if (verbose) { + printf("f8_e4m3 input %a failed: encoded=0x%02x reference=0x%02x expected=0x%02x\n", + inputs[i], quantized[i], quantized_ref[i], expected); + } + } + } + + for (int n : {0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 65, 129, 257}) { + std::vector<uint8_t> x(n + 1); + std::vector<float> y(n + 1); + double expected = 0.0; + for (int i = 0; i < n; ++i) { + const uint8_t code = (uint8_t) (i % 254 + (i % 254 >= 127)); + x[i + 1] = code; + y[i + 1] = (i % 7 - 3) * 0.125f; + expected += std::copysign(F8_E4M3_VALUES[code & 0x7F], code & 0x80 ? -1.0f : 1.0f) * y[i + 1]; + } + float actual; + traits_cpu->vec_dot(n, &actual, 0, x.data() + 1, 0, y.data() + 1, 0, 1); + const bool failed = std::fabs(actual - expected) > 1e-5 * std::max(1.0, std::fabs(expected)); + num_failed += failed; + if (verbose || failed) { + printf("f8_e4m3 dot n=%d: %s (ref=%f got=%f)\n", n, RESULT_STR[failed], expected, actual); + } + } + + for (uint8_t code : {0x7F, 0xFF}) { + std::vector<uint8_t> x(65, 0); + std::vector<float> y(65, 1.0f); + x[0] = x.back() = code; + float actual; + traits_cpu->vec_dot(x.size(), &actual, 0, x.data(), 0, y.data(), 0, 1); + num_failed += !std::isnan(actual); + } + + if (verbose || num_failed) { + printf("f8_e4m3 exhaustive conversion: %s (%d failures)\n", RESULT_STR[num_failed != 0], num_failed); + } + return num_failed; +} + static int test_vec_dot_q(bool verbose) { int num_failed = 0; @@ -299,6 +446,7 @@ int main(int argc, char * argv[]) { int num_failed = 0; num_failed += test_vec_dot_f32(verbose); + num_failed += test_f8_e4m3(verbose); num_failed += test_vec_dot_q(verbose); num_failed += test_quantize_imatrix_degenerate(verbose); diff --git a/patches/ggml/0002-backend-reg-filter.patch b/patches/ggml/0002-backend-reg-filter.patch index b52ca4df9..11b687d0f 100644 --- a/patches/ggml/0002-backend-reg-filter.patch +++ b/patches/ggml/0002-backend-reg-filter.patch @@ -1,8 +1,8 @@ diff --git a/include/ggml-backend.h b/include/ggml-backend.h -index cc3f8cd3..764ffe84 100644 +index afb8633..324160f 100644 --- a/include/ggml-backend.h +++ b/include/ggml-backend.h -@@ -259,6 +259,13 @@ extern "C" { +@@ -276,6 +276,13 @@ extern "C" { GGML_API void ggml_backend_load_all(void); GGML_API void ggml_backend_load_all_from_path(const char * dir_path); @@ -17,7 +17,7 @@ index cc3f8cd3..764ffe84 100644 // Backend scheduler // diff --git a/src/ggml-backend-reg.cpp b/src/ggml-backend-reg.cpp -index 1c18b82c..7c076862 100644 +index c3bc926..dc801ba 100644 --- a/src/ggml-backend-reg.cpp +++ b/src/ggml-backend-reg.cpp @@ -3,6 +3,7 @@ @@ -171,8 +171,8 @@ index 1c18b82c..7c076862 100644 // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths const fs::path name_path = fs::u8path(name); const fs::path file_prefix = backend_filename_prefix().native() + name_path.native() + fs::u8path("-").native(); -@@ -599,7 +652,7 @@ void ggml_backend_load_all_from_path(const char * dir_path) { - ggml_backend_load_best("cpu", silent, dir_path); +@@ -593,7 +646,7 @@ void ggml_backend_load_all_from_path(const char * dir_path) { + ggml_backend_load_best("cpu", silent, dir_path); // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend const char * backend_path = std::getenv("GGML_BACKEND_PATH"); - if (backend_path) { @@ -181,7 +181,7 @@ index 1c18b82c..7c076862 100644 } } diff --git a/src/ggml-hip/CMakeLists.txt b/src/ggml-hip/CMakeLists.txt -index a6a6b727..9b963dd5 100644 +index 75d4d60..8dec5ee 100644 --- a/src/ggml-hip/CMakeLists.txt +++ b/src/ggml-hip/CMakeLists.txt @@ -81,6 +81,8 @@ ggml_add_backend_library(ggml-hip @@ -194,10 +194,10 @@ index a6a6b727..9b963dd5 100644 add_compile_definitions(GGML_USE_HIP) diff --git a/src/ggml-musa/CMakeLists.txt b/src/ggml-musa/CMakeLists.txt -index 82b754f4..6cb178eb 100644 +index 721621a..f451b0c 100644 --- a/src/ggml-musa/CMakeLists.txt +++ b/src/ggml-musa/CMakeLists.txt -@@ -63,6 +63,8 @@ if (MUSAToolkit_FOUND) +@@ -66,6 +66,8 @@ if (MUSAToolkit_FOUND) # TODO: do not use CUDA definitions for MUSA if (NOT GGML_BACKEND_DL) target_compile_definitions(ggml PUBLIC GGML_USE_CUDA)